Ë
    ÛÓ^j	ë  ã                   óè  — d dl mZ d dlmZ d dlmZ d dlZd dlmZ ddlm	Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZmZmZ ddlmZmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0 ddl1m2Z2m3Z3m4Z4  e&jj                  e6«      Z7 G d„ dejp                  «      Z9 G d„ dejp                  «      Z:e$ G d„ de«      «       Z;	 dLdejp                  dejx                  dejx                  d ejx                  d!ejx                  dz  d"e=d#e=fd$„Z> G d%„ d&ejp                  «      Z? G d'„ d(ejp                  «      Z@ G d)„ d*e«      ZA G d+„ d,ejp                  «      ZB G d-„ d.e;«      ZC G d/„ d0ejp                  «      ZD G d1„ d2ejp                  «      ZE G d3„ d4ejp                  «      ZF G d5„ d6ejp                  «      ZG G d7„ d8ejp                  «      ZH G d9„ d:e«      ZI G d;„ d<ejp                  «      ZJ G d=„ d>e;«      ZK e$d?¬@«      e G dA„ dBe"«      «       «       ZL e$dC¬@«       G dD„ dEe;«      «       ZMe$e G dF„ dGe«      «       «       ZN e$dH¬@«       G dI„ dJe;e«      «       ZOg dK¢ZPy)Mé    )ÚCallable)Ú	dataclass)ÚAnyN)Únné   )Úinitialization)ÚACT2FN)ÚGenerationMixin)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚSeq2SeqLMOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚ	torch_int)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	AutoModelÚAutoModelForCausalLMÚAutoModelForSeq2SeqLMé   )ÚInstructBlipVideoConfigÚInstructBlipVideoQFormerConfigÚInstructBlipVideoVisionConfigc                   ó¢   ‡ — e Zd Zdefˆ fd„Zdej                  dededej                  fd„Zddej                  d	e
dej                  fd
„Zˆ xZS )Ú!InstructBlipVideoVisionEmbeddingsÚconfigc                 óp  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        t        j                  t        j                  dd| j                  «      «      | _        t        j                  d| j                  | j                  | j                  ¬«      | _        | j
                  | j                  z  dz  | _        | j                  dz   | _        t        j                  t        j                  d| j                  | j                  «      «      | _        y )Nr$   r   )Úin_channelsÚout_channelsÚkernel_sizeÚstrider    )ÚsuperÚ__init__r*   Úhidden_sizeÚ	embed_dimÚ
image_sizeÚ
patch_sizer   Ú	ParameterÚtorchÚrandnÚclass_embeddingÚConv2dÚpatch_embeddingÚnum_patchesÚnum_positionsÚposition_embedding©Úselfr*   Ú	__class__s     €ú‹/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/instructblipvideo/modeling_instructblipvideo.pyr1   z*InstructBlipVideoVisionEmbeddings.__init__<   sá   ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿ|™|¬E¯K©K¸¸1¸d¿n¹nÓ,MÓNˆÔä!Ÿy™yØ¨¯©ÀDÇOÁOÐ\`×\kÑ\kô 
ˆÔð !ŸO™O¨t¯©Ñ>À1ÑDˆÔØ!×-Ñ-°Ñ1ˆÔä"$§,¡,¬u¯{©{¸1¸d×>PÑ>PÐRV×R`ÑR`Ó/aÓ"bˆÕó    Ú
embeddingsÚheightÚwidthÚreturnc                 ó¦  — |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                  «       s||k(  r||k(  r| j                  S | j                  dd…dd…f   }| j                  dd…dd…f   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz  «      }|j                  d|||«      }|j                  dddd«      }t        j                  j                  ||	|
fdd	¬
«      }|j                  dddd«      j                  dd|«      }t        j                  ||fd¬«      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r$   Néÿÿÿÿg      à?r   r   r    ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)Úshaper>   r7   ÚjitÚ
is_tracingr5   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewÚcat)r@   rD   rE   rF   r<   r=   Úclass_pos_embedÚpatch_pos_embedrO   Ú
new_heightÚ	new_widthÚsqrt_num_positionss               rB   Úinterpolate_pos_encodingz:InstructBlipVideoVisionEmbeddings.interpolate_pos_encodingN   s`  € ð !×&Ñ& qÑ)¨AÑ-ˆØ×/Ñ/×5Ñ5°aÑ8¸1Ñ<ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×*Ñ*Ð*à×1Ñ1²!°R°a°R°%Ñ8ˆØ×1Ñ1²!°Q±R°%Ñ8ˆà×Ñ˜rÑ"ˆà˜tŸ™Ñ.ˆ
Ø˜TŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐI[Ð]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØ˜iÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�y‰y˜/¨?Ð;ÀÔCÐCrC   Úpixel_valuesr^   c                 ó  — |j                   \  }}}}| j                  j                  j                  }| j                  |j	                  |¬«      «      }|j                  d«      j                  dd«      }| j                  j                  |dd«      j	                  |«      }	t        j                  |	|gd¬«      }
|r| j                  |
||«      }n| j                  }|
|d d …d |
j                  d«      …d d …f   j	                  |«      z   }
|
S )N)Údtyper    r$   rI   rN   )rP   r;   Úweightra   ÚtoÚflattenÚ	transposer9   Úexpandr7   rX   r^   r>   rK   )r@   r_   r^   Ú
batch_sizeÚ_rE   rF   Útarget_dtypeÚpatch_embedsÚclass_embedsrD   r>   s               rB   Úforwardz)InstructBlipVideoVisionEmbeddings.forwardv   sü   € Ø'3×'9Ñ'9Ñ$ˆ
�A�v˜uØ×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆØ#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆØ×+Ñ+×2Ñ2°:¸qÀ"ÓE×HÑHÈÓVˆÜ—Y‘Y ¨lÐ;ÀÔCˆ
Ù#Ø!%×!>Ñ!>¸zÈ6ÐSXÓ!YÑà!%×!8Ñ!8ÐØÐ"4²QÐ8L¸*¿/¹/È!Ó:LÐ8LÊaÐ5OÑ"P×"SÑ"SÐT`Ó"aÑaˆ
ØÐrC   ©F)Ú__name__Ú
__module__Ú__qualname__r'   r1   r7   ÚTensorÚintr^   ÚFloatTensorÚboolrl   Ú__classcell__©rA   s   @rB   r)   r)   ;   sm   ø„ ðcÐ<õ cð$&D°5·<±<ð &DÈð &DÐUXð &DÐ]b×]iÑ]ió &DñP E×$5Ñ$5ð ÐQUð Ðbg×bnÑbn÷ rC   r)   c                   ó2   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 dd„Zˆ xZS )Ú"InstructBlipVideoQFormerEmbeddingsz;Construct the embeddings from word and position embeddings.c                 ó,  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j
                  «      | _	        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        | j!                  dt#        j$                  |j                  «      j'                  d«      d¬«       || _        y )N)Úpadding_idx©ÚepsÚposition_ids©r$   rI   F)Ú
persistent)r0   r1   r   Ú	EmbeddingÚ
vocab_sizer2   Úpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚ	LayerNormÚlayer_norm_epsÚ	layernormÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferr7   Úarangerf   r*   r?   s     €rB   r1   z+InstructBlipVideoQFormerEmbeddings.__init__ˆ   sË   ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¡<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ äŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒð 	×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	ô 	
ð ˆ�rC   c                 óè  — |�|j                  «       d   }nd}|€&| j                  d d …|||z   …f   j                  «       }|�\| j                  |«      }| j	                  |j                  |j                  «      «      }||z   }|�t        j                  ||fd¬«      }n|}|j                  | j                  j                  j                  «      }| j                  |«      }| j                  |«      }|S )Nr$   r   rN   )rK   r}   Úclonerƒ   r…   rc   Údevicer7   rX   rˆ   rb   ra   r‹   )r@   Ú	input_idsr}   Úquery_embedsÚpast_key_values_lengthÚ
seq_lengthrD   r…   s           rB   rl   z*InstructBlipVideoQFormerEmbeddings.forward—   sñ   € ð Ð Ø"Ÿ™Ó)¨!Ñ,‰JàˆJàÐØ×,Ñ,ªQÐ0FÈÐVlÑIlÐ0lÐ-lÑm×sÑsÓuˆLàÐ Ø×-Ñ-¨iÓ8ˆJà"&×":Ñ":¸<¿?¹?È:×K\ÑK\Ó;]Ó"^ÐØ#Ð&9Ñ9ˆJàÐ'Ü"ŸY™Y¨°jÐ'AÀqÔI‘
à%ˆJà—]‘] 4§>¡>×#8Ñ#8×#>Ñ#>Ó?ˆ
Ø—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐrC   ©NNNr   )rn   ro   rp   Ú__doc__r1   rl   ru   rv   s   @rB   rx   rx   …   s   ø„ ÙEôð" ØØØ ÷rC   rx   c                   óz   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZdZg d¢Z ej                  «       ˆ fd„«       Zˆ xZS )Ú InstructBlipVideoPreTrainedModelr*   Úblip)ÚvideoÚtextT)rx   ÚInstructBlipVideoAttentionÚInstructBlipVideoEncoderLayerÚInstructBlipVideoQFormerLayerÚ*InstructBlipVideoQFormerMultiHeadAttentionÚ"InstructBlipVideoQFormerSelfOutputc                 ó:  •— t         ‰| �  |«       | j                  j                  }t	        |t
        «      rEt        j                  |j                  d|¬«       t        j                  |j                  d|¬«       yt	        |t        t        f«      r t        j                  |j                  «       yt	        |t        «      rZt        j                  |j                   t#        j$                  |j                   j&                  d   «      j)                  d«      «       yy)zInitialize the weightsç        )ÚmeanÚstdrI   r~   N)r0   Ú_init_weightsr*   Úinitializer_rangeÚ
isinstancer)   ÚinitÚtrunc_normal_r>   r9   Ú)InstructBlipVideoForConditionalGenerationÚInstructBlipVideoModelÚzeros_Úquery_tokensrx   Úcopy_r}   r7   r�   rP   rf   )r@   ÚmoduleÚfactorrA   s      €rB   r¥   z.InstructBlipVideoPreTrainedModel._init_weightsÍ   sË   ø€ ô 	‰Ñ˜fÔ%Ø—‘×.Ñ.ˆÜ�fÔ?Ô@Ü×Ñ˜v×8Ñ8¸sÈÕOÜ×Ñ˜v×5Ñ5¸CÀVÖLÜ˜Ô!JÔLbÐ cÔdÜ�K‰K˜×+Ñ+Õ,Ü˜Ô BÔCÜ�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhð DrC   )rn   ro   rp   r%   Ú__annotations__Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_no_split_modulesr7   Úno_gradr¥   ru   rv   s   @rB   r˜   r˜   ·   s^   ø… à#Ó#ØÐØ(ÐØ&*Ð#Ø"&ÐØÐØ€NØÐà!ÐòÐð €U‡]�]ƒ_ó
ió ô
irC   r˜   r¯   ÚqueryÚkeyÚvalueÚattention_maskÚscalingr‹   c                 óp  — t        j                  ||j                  dd«      «      |z  }|�||z   }t        j                  j                  |d¬«      }t        j                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )NrI   éþÿÿÿrN   )ÚpÚtrainingr$   r    )	r7   Úmatmulre   r   rU   Úsoftmaxr‹   rÄ   Ú
contiguous)
r¯   r¼   r½   r¾   r¿   rÀ   r‹   ÚkwargsÚattn_weightsÚattn_outputs
             rB   Úeager_attention_forwardrË   Ü   s¢   € ô —<‘<  s§}¡}°R¸Ó'<Ó=ÀÑG€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$rC   c                   óÈ   ‡ — e Zd ZdZˆ fd„Zdej                  dedefd„Zdej                  de	ej                  ej                  d	z  e	ej                     d	z  f   fd
„Z
ˆ xZS )rœ   z=Multi-headed attention from 'Attention Is All You Need' paperc                 óö  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚| j                  dz  | _	        d| _
        |j                  | _        t        j                  | j                  d| j                  z  d¬«      | _        |j                  ret        j                   t#        j$                  | j                  «      «      }t        j                   t#        j$                  | j                  «      «      }nd }d }|�Qt#        j&                  |t#        j(                  |d¬«      |f«      }t        j                   |«      | j                  _        t        j                  | j                  | j                  «      | _        y )	Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).ç      à¿Fr   )Úbias)Úrequires_grad)r0   r1   r*   r2   r3   Únum_attention_headsÚ	num_headsÚhead_dimÚ
ValueErrorÚscaleÚ	is_causalÚattention_dropoutr   ÚLinearÚqkvÚqkv_biasr6   r7   ÚzerosrX   Ú
zeros_likerÏ   Ú
projection)r@   r*   Úq_biasÚv_biasrÚ   rA   s        €rB   r1   z#InstructBlipVideoAttention.__init__ö   su  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
ØˆŒØ!'×!9Ñ!9ˆÔô —9‘9˜TŸ^™^¨Q°·±Ñ-?ÀeÔLˆŒà�?Š?Ü—\‘\¤%§+¡+¨d¯n©nÓ"=Ó>ˆFÜ—\‘\¤%§+¡+¨d¯n©nÓ"=Ó>‰FàˆFØˆFàÐÜ—y‘y &¬%×*:Ñ*:¸6ÐQVÔ*WÐY_Ð!`ÓaˆHÜŸL™L¨Ó2ˆD�H‰HŒMäŸ)™) D§N¡N°D·N±NÓCˆ�rC   ÚtensorÚseq_lenÚbszc                 óŽ   — |j                  ||| j                  | j                  «      j                  dd«      j	                  «       S )Nr$   r    )rW   rÒ   rÓ   re   rÇ   )r@   rà   rá   râ   s       rB   Ú_shapez!InstructBlipVideoAttention._shape  s7   € Ø�{‰{˜3 ¨¯©¸¿¹ÓG×QÑQÐRSÐUVÓW×bÑbÓdÐdrC   Úhidden_statesrG   Nc                 ó  — |j                  «       \  }}}| j                  |«      }|j                  ||d| j                  || j                  z  «      j	                  ddddd«      }|d   |d   |d   }	}}t        j                  | j                  j                  t        «      }
 |
| |||	fd| j                  sdn| j                  | j                  dœ|¤Ž\  }}|j                  ||d	«      j                  «       }| j                  |«      }||fS )
z#Input shape: Batch x Time x Channelr   r    r   r$   é   Nr¢   )r¿   r‹   rÀ   rI   )rK   rÙ   rS   rÒ   rT   r   Úget_interfacer*   Ú_attn_implementationrË   rÄ   r×   rÕ   rÇ   rÝ   )r@   rå   rÈ   râ   Útgt_lenr3   Ú	mixed_qkvÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacerÊ   rÉ   s                rB   rl   z"InstructBlipVideoAttention.forward  s#  € ð #0×"4Ñ"4Ó"6ÑˆˆW�ià—H‘H˜]Ó+ˆ	à×%Ñ% c¨7°A°t·~±~ÀyÐTX×TbÑTbÑGbÓc×kÑkØˆq�!�Q˜ó
ˆ	ð 2;¸1±¸yÈ¹|ÈYÐWXÉ\ ,�jˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØð		%
ð
  Ø#Ÿ}š}‘C°$×2HÑ2HØ—J‘Jñ	%
ð ñ	%
Ñ!ˆ�\ð "×)Ñ)¨#¨w¸Ó;×FÑFÓHˆØ—o‘o kÓ2ˆà˜LÐ(Ð(rC   )rn   ro   rp   r–   r1   r7   rq   rr   rä   Útuplerl   ru   rv   s   @rB   rœ   rœ   ó   sp   ø„ ÙGôDð>e˜UŸ\™\ð e°Cð e¸có eð")à—|‘|ð")ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷	")rC   rœ   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚInstructBlipVideoMLPc                 ó  •— t         ‰| �  «        || _        t        |j                     | _        t        j                  |j                  |j                  «      | _
        t        j                  |j                  |j                  «      | _        y ©N)r0   r1   r*   r	   Ú
hidden_actÚactivation_fnr   rØ   r2   Úintermediate_sizeÚfc1Úfc2r?   s     €rB   r1   zInstructBlipVideoMLP.__init__>  sd   ø€ Ü‰ÑÔØˆŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�rC   rå   rG   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rô   )rø   rö   rù   ©r@   rå   s     rB   rl   zInstructBlipVideoMLP.forwardE  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐrC   ©rn   ro   rp   r1   r7   rq   rl   ru   rv   s   @rB   rò   rò   =  s$   ø„ ôKð U§\¡\ð °e·l±l÷ rC   rò   c                   óp   ‡ — e Zd Zdefˆ fd„Zedej                  dee	   dej                  fd„«       Zˆ xZS )r�   r*   c                 óD  •— t         ‰| �  «        |j                  | _        t	        |«      | _        t        j                  | j                  |j                  ¬«      | _	        t        |«      | _        t        j                  | j                  |j                  ¬«      | _        y ©Nr{   )r0   r1   r2   r3   rœ   Ú	self_attnr   r†   r‡   Úlayer_norm1rò   ÚmlpÚlayer_norm2r?   s     €rB   r1   z&InstructBlipVideoEncoderLayer.__init__M  sm   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ3°FÓ;ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÔÜ'¨Ó/ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÕrC   rå   rÈ   rG   c                 ó¶   — |}| j                  |«      } | j                  dd|i|¤Ž\  }}||z   }|}| j                  |«      }| j                  |«      }||z   }|S )Nrå   © )r  r   r  r  )r@   rå   rÈ   Úresidualrh   s        rB   rl   z%InstructBlipVideoEncoderLayer.forwardU  s}   € ð !ˆà×(Ñ(¨Ó7ˆØ)˜4Ÿ>™>ñ 
Ø'ð
àñ
Ñˆ�qð &¨Ñ0ˆØ ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆà%¨Ñ0ˆàÐrC   )rn   ro   rp   r%   r1   r   r7   rq   r   r   rs   rl   ru   rv   s   @rB   r�   r�   L  sR   ø„ ðSÐ6õ Sð ðà—|‘|ðð Ð+Ñ,ðð 
×	Ñ	ò	ó ôrC   r�   c                   óN   ‡ — e Zd ZdZdefˆ fd„Zedee   de	e
z  fd„«       Zˆ xZS )ÚInstructBlipVideoEncodera"  
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`InstructBlipVideoEncoderLayer`].

    Args:
        config (`InstructBlipVideoConfig`):
            The corresponding vision configuration for the `InstructBlipVideoEncoder`.
    r*   c                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w ©NF)
r0   r1   r*   r   Ú
ModuleListÚrangeÚnum_hidden_layersr�   ÚlayersÚgradient_checkpointing)r@   r*   rh   rA   s      €rB   r1   z!InstructBlipVideoEncoder.__init__v  sQ   ø€ Ü‰ÑÔØˆŒÜ—m‘mÔTYÐZ`×ZrÑZrÓTsÖ$tÈqÔ%BÀ6Õ%JÒ$tÓuˆŒØ&+ˆÕ#ùò %us   ½A#rÈ   rG   c                 óR   — |}| j                   D ]  } ||fi |¤Ž}Œ t        |¬«      S )N©Úlast_hidden_state)r  r   )r@   Úinputs_embedsrÈ   rå   Úencoder_layers        rB   rl   z InstructBlipVideoEncoder.forward|  s>   € ð &ˆØ!Ÿ[™[ò 	ˆMÙ)Øñàñ‰Mð	ô °Ô?Ð?rC   )rn   ro   rp   r–   r%   r1   r   r   r   rð   r   rl   ru   rv   s   @rB   r  r  l  sL   ø„ ñð,Ð6õ ,ð ð@ð Ð+Ñ,ð@ð 
�Ñ	 ò	@ó ô@rC   r  c                   ó¸   ‡ — e Zd ZU dZdZeed<   eedœZ	defˆ fd„Z
e ed¬«      e	 	 ddej                  dz  d	ed
ee   deez  fd„«       «       «       Zd„ Zˆ xZS )ÚInstructBlipVideoVisionModelr_   rš   r*   )rå   Ú
attentionsc                 óö   •— t         ‰| �  |«       || _        |j                  }t	        |«      | _        t        |«      | _        t        j                  ||j                  ¬«      | _        | j                  «        y rÿ   )r0   r1   r*   r2   r)   rD   r  Úencoderr   r†   r‡   Úpost_layernormÚ	post_init)r@   r*   r3   rA   s      €rB   r1   z%InstructBlipVideoVisionModel.__init__•  s]   ø€ Ü‰Ñ˜Ô ØˆŒØ×&Ñ&ˆ	ä;¸FÓCˆŒÜ/°Ó7ˆŒÜ Ÿl™l¨9¸&×:OÑ:OÔPˆÔà�‰ÕrC   F)Útie_last_hidden_statesNr^   rÈ   rG   c                 óø   — |€t        d«      ‚| j                  ||¬«      } | j                  dd|i|¤Ž}|j                  }| j	                  |«      }|d d …dd d …f   }| j	                  |«      }t        ||¬«      S )Nz You have to specify pixel_values)r^   r  r   ©r  Úpooler_outputr  )rÔ   rD   r  r  r  r   )r@   r_   r^   rÈ   rå   Úencoder_outputsr  Úpooled_outputs           rB   rl   z$InstructBlipVideoVisionModel.forward   sœ   € ð ÐÜÐ?Ó@Ð@àŸ™¨ÐOg˜Óhˆà+7¨4¯<©<ñ ,
Ø'ð,
àñ,
ˆð
 ,×=Ñ=ÐØ ×/Ñ/Ð0AÓBÐà)ª!¨Q²¨'Ñ2ˆØ×+Ñ+¨MÓ:ˆä)Ø/Ø'ô
ð 	
rC   c                 ó   — | j                   S rô   )rD   ©r@   s    rB   Úget_input_embeddingsz1InstructBlipVideoVisionModel.get_input_embeddings¾  s   € Ø�‰ÐrC   r
  )rn   ro   rp   Úmain_input_namer³   r'   r±   r�   rœ   Ú_can_record_outputsr1   r   r   r   r7   rs   rt   r   r   rð   r   rl   r$  ru   rv   s   @rB   r  r  Œ  s¢   ø… Ø$€OØÐØ)Ó)à6Ø0ñÐð
	Ð<õ 	ð  Ù¨EÔ2Øð 26Ø).ñ
à×'Ñ'¨$Ñ.ð
ð #'ð
ð Ð+Ñ,ð	
ð
 
Ð+Ñ	+ò
ó ó 3ó  ð
ö6rC   r  c                   ó:   ‡ — e Zd Zdˆ fd„	Z	 	 	 ddee   fd„Zˆ xZS )rŸ   c                 ó¨  •— t         ‰| �  «        || _        |j                  |j                  z  dk7  r0t        |d«      s$t        d|j                  |j                  fz  «      ‚|j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _	        | j                  dz  | _
        d| _        |j                  | _        t        j                  |j                  | j                  «      | _        |r_t        j                  |j"                  | j                  «      | _        t        j                  |j"                  | j                  «      | _        y t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        y )Nr   Úembedding_sizezLThe hidden size (%d) is not a multiple of the number of attention heads (%d)rÎ   F)r0   r1   r*   r2   rÑ   ÚhasattrrÔ   rr   Úattention_head_sizeÚall_head_sizerÀ   rÖ   Úattention_probs_dropout_probr×   r   rØ   r¼   Úencoder_hidden_sizer½   r¾   ©r@   r*   Úis_cross_attentionrA   s      €rB   r1   z3InstructBlipVideoQFormerMultiHeadAttention.__init__Ã  sj  ø€ Ü‰ÑÔØˆŒØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ^Ø×%Ñ% v×'AÑ'AÐBñCóð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ×/Ñ/°Ñ5ˆŒØˆŒØ!'×!DÑ!DˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ÙÜ—y‘y ×!;Ñ!;¸T×=OÑ=OÓPˆDŒHÜŸ™ 6×#=Ñ#=¸t×?QÑ?QÓRˆD�Jä—y‘y ×!3Ñ!3°T×5GÑ5GÓHˆDŒHÜŸ™ 6×#5Ñ#5°t×7IÑ7IÓJˆD�JrC   rÈ   c                 óì  — |d u}|j                   d d }g |¢d‘| j                  ‘­}|r|}	|}n|}	g |	j                   d d ¢d‘| j                  ‘­}
| j                  |«      j                  |«      j	                  dd«      }| j                  |	«      j                  |
«      j	                  dd«      }| j                  |	«      j                  |
«      j	                  dd«      }t        j                  | j                  j                  t        «      } || ||||f| j                  sdn| j                  | j                  dœ|¤Ž\  }} |j                  g |j                   d d ¢d‘­Ž j!                  «       }||fS )NrI   r$   r    r¢   )r‹   rÀ   )rP   r+  r¼   rW   re   r½   r¾   r   rè   r*   ré   rË   rÄ   r×   rÀ   rS   rÇ   )r@   rå   r¿   Úencoder_hidden_statesÚencoder_attention_maskrÈ   r0  Úinput_shapeÚhidden_shapeÚcurrent_statesÚkv_shapeÚquery_layerÚ	key_layerÚvalue_layerrï   rÊ   rÉ   s                    rB   rl   z2InstructBlipVideoQFormerMultiHeadAttention.forwardÛ  sŽ  € ð 3¸$Ð>Ðà#×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆáØ2ˆNØ3‰Nà*ˆNàM�^×)Ñ)¨#¨2Ð.ÐM°ÐM°D×4LÑ4LÑMˆØ—j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆØ—H‘H˜^Ó,×1Ñ1°(Ó;×EÑEÀaÈÓKˆ	Ø—j‘j Ó0×5Ñ5°hÓ?×IÑIÈ!ÈQÓOˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)ÐE¨;×+<Ñ+<¸R¸aÐ+@ÐEÀ"ÒE×PÑPÓRˆØ˜LÐ(Ð(rC   rm   ©NNN)rn   ro   rp   r1   r   r   rl   ru   rv   s   @rB   rŸ   rŸ   Â  s)   ø„ õKð6 Ø"Ø#ñ,)ð Ð+Ñ,÷,)rC   rŸ   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )r    c                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y rÿ   )r0   r1   r   rØ   r2   Údenser†   r‡   r‰   rŠ   r‹   r?   s     €rB   r1   z+InstructBlipVideoQFormerSelfOutput.__init__  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�rC   rå   Úinput_tensorrG   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S rô   ©r>  r‹   r†   ©r@   rå   r?  s      rB   rl   z*InstructBlipVideoQFormerSelfOutput.forward  ó7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐrC   rü   rv   s   @rB   r    r    
  ó1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ rC   r    c                   óÄ   ‡ — e Zd Zd
ˆ fd„	Z	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dee   dej
                  fd	„Z	ˆ xZ
S )Ú!InstructBlipVideoQFormerAttentionc                 ód   •— t         ‰| �  «        t        ||«      | _        t	        |«      | _        y rô   )r0   r1   rŸ   Ú	attentionr    Úoutputr/  s      €rB   r1   z*InstructBlipVideoQFormerAttention.__init__  s)   ø€ Ü‰ÑÔÜCÀFÐL^Ó_ˆŒÜ8¸Ó@ˆ�rC   Nrå   r¿   r2  r3  rÈ   rG   c                 ó^   —  | j                   d||||dœ|¤Ž\  }}| j                  ||«      }|S )N)rå   r¿   r2  r3  r  )rH  rI  )	r@   rå   r¿   r2  r3  rÈ   rÊ   rh   Úattention_outputs	            rB   rl   z)InstructBlipVideoQFormerAttention.forward  sL   € ð (˜Ÿ™ð 
Ø'Ø)Ø"7Ø#9ñ	
ð
 ñ
‰ˆ�Qð  Ÿ;™; {°MÓBÐØÐrC   rm   r;  )rn   ro   rp   r1   r7   rq   rs   r   r   rl   ru   rv   s   @rB   rF  rF    s‚   ø„ õAð 48Ø:>Ø;?ñ à—|‘|ð ð ×)Ñ)¨DÑ0ð ð  %×0Ñ0°4Ñ7ð	 ð
 !&× 1Ñ 1°DÑ 8ð ð Ð+Ñ,ð ð 
�‰÷ rC   rF  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )Ú$InstructBlipVideoQFormerIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y rô   )r0   r1   r   rØ   r2   r÷   r>  r§   rõ   Ústrr	   Úintermediate_act_fnr?   s     €rB   r1   z-InstructBlipVideoQFormerIntermediate.__init__2  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$rC   rå   rG   c                 óJ   — | j                  |«      }| j                  |«      }|S rô   )r>  rP  rû   s     rB   rl   z,InstructBlipVideoQFormerIntermediate.forward:  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐrC   rü   rv   s   @rB   rM  rM  1  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ rC   rM  c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚInstructBlipVideoQFormerOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y rÿ   )r0   r1   r   rØ   r÷   r2   r>  r†   r‡   r‰   rŠ   r‹   r?   s     €rB   r1   z'InstructBlipVideoQFormerOutput.__init__A  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�rC   rå   r?  rG   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S rô   rA  rB  s      rB   rl   z&InstructBlipVideoQFormerOutput.forwardG  rC  rC   rü   rv   s   @rB   rS  rS  @  rD  rC   rS  c                   óF   ‡ — e Zd Zˆ fd„Z	 	 	 	 ddee   fd„Zd„ Zd„ Zˆ xZ	S )rž   c                 óf  •— t         ‰| �  «        |j                  | _        d| _        t	        |«      | _        || _        ||j                  z  dk(  rt	        |d¬«      | _        d| _	        nd| _	        t        |«      | _        t        |«      | _        t        |«      | _        t        |«      | _        y )Nr$   r   T)r0  F)r0   r1   Úchunk_size_feed_forwardÚseq_len_dimrF  rH  Ú	layer_idxÚcross_attention_frequencyÚcrossattentionÚhas_cross_attentionrM  ÚintermediaterS  rI  Úintermediate_queryÚoutput_query©r@   r*   rZ  rA   s      €rB   r1   z&InstructBlipVideoQFormerLayer.__init__O  s�   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ:¸6ÓBˆŒà"ˆŒà�v×7Ñ7Ñ7¸1Ò<Ü"CÀFÐ_cÔ"dˆDÔØ'+ˆDÕ$à',ˆDÔ$ä@ÀÓHˆÔÜ4°VÓ<ˆŒä"FÀvÓ"NˆÔÜ:¸6ÓBˆÕrC   rÈ   c           
      ób  —  | j                   |fd|i|¤Ž}|dkD  rè|d d …d |…d d …f   }| j                  r$|€t        d«      ‚ | j                  |f|||dœ|¤Ž}t	        | j
                  | j                  | j                  |«      }	|j                  d   |kD  rjt	        | j                  | j                  | j                  |d d …|d …d d …f   «      j                  |	j                  «      }
t        j                  |	|
gd¬«      }	|	S t	        | j                  | j                  | j                  |«      }	|	S )Nr¿   r   z>encoder_hidden_states must be given for cross-attention layers)r¿   r2  r3  r$   rN   )rH  r]  rÔ   r\  r   Úfeed_forward_chunk_queryrX  rY  rP   Úfeed_forward_chunkrc   r�   r7   rX   )r@   rå   r¿   r2  r3  Úquery_lengthrÈ   rK  Úquery_attention_outputÚlayer_outputÚlayer_output_texts              rB   rl   z%InstructBlipVideoQFormerLayer.forwardc  sp  € ð *˜4Ÿ>™>Øñ
à)ð
ð ñ
Ðð ˜!ÒØ%5²a¸¸,¸ÊÐ6IÑ%JÐ"à×'Ò'Ø(Ð0Ü$Ð%eÓfÐfØ)<¨×)<Ñ)<Ø*ð*à#1Ø*?Ø+Añ	*ð
 ñ*Ð&ô 5Ø×-Ñ-Ø×,Ñ,Ø× Ñ Ø&ó	ˆLð  ×%Ñ% aÑ(¨<Ò7Ü$=Ø×+Ñ+Ø×0Ñ0Ø×$Ñ$Ø$¢Q¨©²qÐ%8Ñ9ó	%÷
 ‘"�\×(Ñ(Ó)ð "ô  %Ÿy™y¨,Ð8IÐ)JÐPQÔR�ð Ðô 5Ø×'Ñ'Ø×,Ñ,Ø× Ñ Ø ó	ˆLð ÐrC   c                 óL   — | j                  |«      }| j                  ||«      }|S rô   )r^  rI  ©r@   rK  Úintermediate_outputrg  s       rB   rd  z0InstructBlipVideoQFormerLayer.feed_forward_chunk˜  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐrC   c                 óL   — | j                  |«      }| j                  ||«      }|S rô   )r_  r`  rj  s       rB   rc  z6InstructBlipVideoQFormerLayer.feed_forward_chunk_query�  s.   € Ø"×5Ñ5Ð6FÓGÐØ×(Ñ(Ð)<Ð>NÓOˆØÐrC   r•   )
rn   ro   rp   r1   r   r   rl   rd  rc  ru   rv   s   @rB   rž   rž   N  s7   ø„ ôCð. Ø"Ø#Øñ3ð Ð+Ñ,ó3òjö
rC   rž   c                   óD   ‡ — e Zd Zˆ fd„Ze	 	 	 	 ddee   fd„«       Zˆ xZS )ÚInstructBlipVideoQFormerEncoderc           	      óÒ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        d| _	        y c c}w r
  )
r0   r1   r*   r   r  r  r  rž   Úlayerr  ra  s      €rB   r1   z(InstructBlipVideoQFormerEncoder.__init__¤  sW   ø€ Ü‰ÑÔØˆŒÜ—]‘]ÜOTÐU[×UmÑUmÓOnÖoÀ)Ô*¨6°9Õ=Òoó
ˆŒ
ð ',ˆÕ#ùò ps   ½A$rÈ   c                 óœ   — t        | j                  j                  «      D ]  }| j                  |   } ||||f||dœ|¤Ž}Œ! t	        |¬«      S )N)r3  re  r  )r  r*   r  rp  r   )	r@   rå   r¿   r2  r3  re  rÈ   ÚiÚlayer_modules	            rB   rl   z'InstructBlipVideoQFormerEncoder.forward¬  sk   € ô �t—{‘{×4Ñ4Ó5ò 
	ˆAØŸ:™: a™=ˆLá(ØØØ%ðð (>Ø)ñð ñ‰Mð
	ô 9Ø+ô
ð 	
rC   r•   )	rn   ro   rp   r1   r   r   r   rl   ru   rv   s   @rB   rn  rn  £  s:   ø„ ô,ð ð Ø"Ø#Øñ
ð Ð+Ñ,ò
ó ô
rC   rn  c                   ó†  ‡ — e Zd ZdZdZdZdZdZe e	e
dd¬«      g e	e
dd¬«      gdœZdefˆ fd	„Zd
„ Zd„ Zeee	 	 	 	 	 ddej(                  dej*                  dz  dej(                  dz  dej,                  dz  dej*                  dz  dej*                  dz  dee   deej*                     ez  fd„«       «       «       Zˆ xZS )ÚInstructBlipVideoQFormerModelz’
    Querying Transformer (Q-Former), used in InstructBlipVideo. Slightly modified from BLIP-2 as it also takes the
    instruction as input.
    Tr$   z
.attention)ÚindexÚ
layer_namez.crossattention)rå   r  Úcross_attentionsr*   c                 ó’   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        | j                  «        y rô   )r0   r1   r*   rx   rD   rn  r  r  r?   s     €rB   r1   z&InstructBlipVideoQFormerModel.__init__Ü  s9   ø€ Ü‰Ñ˜Ô ØˆŒä<¸VÓDˆŒä6°vÓ>ˆŒà�‰ÕrC   c                 ó.   — | j                   j                  S rô   ©rD   rƒ   r#  s    rB   r$  z2InstructBlipVideoQFormerModel.get_input_embeddingsæ  s   € Ø�‰×.Ñ.Ð.rC   c                 ó&   — || j                   _        y rô   r{  )r@   r¾   s     rB   Úset_input_embeddingsz2InstructBlipVideoQFormerModel.set_input_embeddingsé  s   € Ø*/ˆ�‰Õ'rC   Nr‘   r¿   r}   r’   r2  r3  rÈ   rG   c                 óx  — |€|€t        d«      ‚|�|j                  d   nd}| j                  |||¬«      }	t        | j                  |	|¬«      }|�.|dkD  r|	dd…d|…dd…f   n|	}
t        | j                  |
||¬«      } | j
                  |	f||||dœ|¤Ž}|j                  }|dd…ddd…f   }t        ||¬	«      S )
a$  
        query_embeds (`torch.FloatTensor`  of shape `(batch_size, sequence_length, hidden_size)`):
            Hidden states to be used in the attention computation. If cross-attention,
            will be used for the query (i.e., key and value will use the encoder_hidden_states).
        Nz7You have to specify query_embeds when input_ids is Noner$   r   )r‘   r}   r’   )r*   r  r¿   )r*   r  r¿   r2  )r¿   r2  r3  re  r  )rÔ   rP   rD   r   r*   r  r  r   )r@   r‘   r¿   r}   r’   r2  r3  rÈ   re  Úembedding_outputÚquery_embedding_outputr   Úsequence_outputr!  s                 rB   rl   z%InstructBlipVideoQFormerModel.forwardì  s  € ð$ Ð Ð!5ÜÐVÓWÐWà0<Ð0H�|×)Ñ)¨!Ò,ÈaˆàŸ?™?ØØ%Ø%ð +ó 
Ðô 3Ø—;‘;Ø*Ø)ô
ˆð "Ð-ð O[Ð]^ÒN^Ð%5²a¸¸,¸ÊÐ6IÒ%JÐdtÐ"Ü%>Ø—{‘{Ø4Ø5Ø&;ô	&Ð"ð ,8¨4¯<©<Øð,
à)Ø"7Ø#9Ø%ñ,
ð ñ,
ˆð *×;Ñ;ˆØ'ª¨1ªa¨Ñ0ˆä;Ø-Ø'ô
ð 	
rC   )NNNNN)rn   ro   rp   r–   rµ   r·   r¶   r¸   rž   r   rŸ   r&  r&   r1   r$  r}  r   r   r   r7   Ú
LongTensorrs   rq   r   r   rð   r   rl   ru   rv   s   @rB   ru  ru  Ç  sB  ø„ ñð
 #'ÐØ€NØÐØÐð 7áÐEÈQÐ[gÔhð
ñ ÐEÈQÐ[lÔmð
ñÐðÐ=õ ò/ò0ð  ØØð 48Ø04Ø,0Ø:>Ø;?ñ9
à×#Ñ#ð9
ð ×)Ñ)¨DÑ0ð9
ð ×&Ñ&¨Ñ-ð	9
ð
 —l‘l TÑ)ð9
ð  %×0Ñ0°4Ñ7ð9
ð !&× 1Ñ 1°DÑ 8ð9
ð Ð+Ñ,ð9
ð 
ˆu× Ñ Ñ	!Ð$PÑ	Pò9
ó ó ó  ô9
rC   ru  zV
    Class defining the outputs of [`InstructBlipVideoForConditionalGeneration`].
    )Úcustom_introc                   óÂ   — e Zd ZU dZdZeej                     dz  ed<   dZ	eej                     dz  ed<   dZ
edz  ed<   dZedz  ed<   dZeez  dz  ed<   dee   fd	„Zy)
Ú4InstructBlipVideoForConditionalGenerationModelOutputaª  
    loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
        Language modeling loss from the language model.
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head of the language model.
    vision_outputs (`BaseModelOutputWithPooling`):
        Outputs of the vision encoder.
    qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`):
        Outputs of the Q-Former (Querying Transformer).
    language_model_outputs (`CausalLMOutputWithPast` or `Seq2SeqLMOutput`):
        Outputs of the language model.
    NÚlossÚlogitsÚvision_outputsÚqformer_outputsÚlanguage_model_outputsrG   c                 óH   ‡ — t        ˆ fd„‰ j                  «       D «       «      S )Nc              3   ód   •K  — | ]'  }|d vr‰|   nt        ‰|«      j                  «       –— Œ) y­w)©rˆ  r‰  rŠ  N)ÚgetattrÚto_tuple)Ú.0Úkr@   s     €rB   ú	<genexpr>zPInstructBlipVideoForConditionalGenerationModelOutput.to_tuple.<locals>.<genexpr>F  sB   øè ø€ ò 
ð ð ÐWÑWð �ŠGä˜˜qÓ!×*Ñ*Ó,ó-ñ
ùs   ƒ-0)rð   Úkeysr#  s   `rB   r�  z=InstructBlipVideoForConditionalGenerationModelOutput.to_tupleE  s%   ø€ Üó 
ð —Y‘Y“[ô	
ó 
ð 	
rC   )rn   ro   rp   r–   r†  rð   r7   rs   r±   r‡  rˆ  r   r‰  r   rŠ  r   r   r   r�  r  rC   rB   r…  r…  +  sƒ   … ñð -1€Dˆ%�×!Ñ!Ñ
" TÑ
)Ó0Ø.2€FˆE�%×#Ñ#Ñ$ tÑ+Ó2Ø8<€NÐ.°Ñ5Ó<ØKO€OÐAÀDÑHÓOØNRÐÐ2°_ÑDÀtÑKÓRð
˜% ™*ô 
rC   r…  z`
    InstructBlipVideo base Model consisting of language model, qformer and vision encoder.
    c                   ó   ‡ — e Zd ZdZdgZdefˆ fd„Zd„ Zdej                  dej                  fd„Zee	 	 	 	 	 	 	 	 ddej                  d
ej                  dej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  deded	z  dee   deez  fd„«       «       Zˆ xZS )r«   r_   r­   r*   c                 ó  •— t         ‰| �  |«       t        |j                  «      | _        t        j                  t        j                  d|j                  |j                  j                  «      «      | _        t        |j                  «      | _        t        j                  |j                  j                  |j                   j                  «      | _        t%        j&                  |j                   «      | _        | j+                  «        y ©Nr$   )r0   r1   r  Úvision_configÚvision_modelr   r6   r7   rÛ   Únum_query_tokensÚqformer_configr2   r­   ru  ÚqformerrØ   Útext_configÚlanguage_projectionr!   Úfrom_configÚlanguage_modelr  r?   s     €rB   r1   zInstructBlipVideoModel.__init__W  s·   ø€ Ü‰Ñ˜Ô ä8¸×9MÑ9MÓNˆÔÜŸL™L¬¯©°Q¸×8OÑ8OÐQW×QfÑQf×QrÑQrÓ)sÓtˆÔÜ4°V×5JÑ5JÓKˆŒä#%§9¡9¨V×-BÑ-B×-NÑ-NÐPV×PbÑPb×PnÑPnÓ#oˆÔ Ü'×3Ñ3°F×4FÑ4FÓGˆÔð 	�‰ÕrC   c                 ó  — | j                   }t        |«      dkD  r:d|vr6t        j                  j	                  «       dkD  rt
        j                  d«       t        | j                  d«      rd| j                  j                  _
        yy©z­
        Some pre-processing hacks to make the model `accelerate` compatible. Check
        https://github.com/huggingface/transformers/pull/21707 for more details.
        r$   rŸ  a   The `language_model` is not in the `hf_device_map` dictionary and you are running your script in a multi-GPU environment. this may lead to unexpected behavior when using `accelerate`. Please pass a `device_map` that contains `language_model` to remove this warning. Please refer to https://github.com/huggingface/blog/blob/main/accelerate-large-models.md for more details on creating a `device_map` for large models.Ú_hf_hookTN©Úhf_device_mapÚlenr7   ÚcudaÚdevice_countÚloggerÚwarningr*  rŸ  r¢  Úio_same_device©r@   r¤  s     rB   Ú_preprocess_acceleratez-InstructBlipVideoModel._preprocess_accelerated  óy   € ð
 ×*Ñ*ˆäˆ}Ó Ò!Ð&6¸mÑ&KÔPU×PZÑPZ×PgÑPgÓPiÐlmÒPmä�N‰NðMôô �4×&Ñ&¨
Ô3Ø:>ˆD×Ñ×(Ñ(Õ7ð 4rC   r‘   r  c                 ój  — |€m| | j                  «       t        j                  | j                  j                  t        j
                  |j                  ¬«      «      k(  }|j                  d«      }n|| j                  j                  k(  }|j                  d«      j                  |j                  «      }|S ©zZ
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`.
        ©ra   r�   rI   )
r$  r7   rà   r*   Úimage_token_idÚlongr�   ÚallÚ	unsqueezerc   ©r@   r‘   r  Úspecial_image_masks       rB   Úget_placeholder_maskz+InstructBlipVideoModel.get_placeholder_maskx  ó�   € ð ÐØ!.Ð2M°$×2KÑ2KÓ2MÜ—‘˜TŸ[™[×7Ñ7¼u¿z¹zÐR_×RfÑRfÔgó3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¨d¯k©k×.HÑ.HÑ!HÐà/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØ!Ð!rC   NÚqformer_input_idsÚqformer_attention_maskr¿   Údecoder_input_idsÚdecoder_attention_maskr^   Ú	use_cacherÈ   rG   c           	      ó�  — |j                   \  }}}}}|j                  ||z  |||«      } | j                  d||	dœ|¤Ž}|d   }t        j                  |j                  «       dd t        j                  |j                  ¬«      }| j                  j                  |j                   d   dd«      }t        j                  |j                  «       dd t        j                  |j                  ¬«      }|€t        j                  |«      }|j                  |d¬«      }|j                  |d¬«      }|j                  |j                  «      }t        j                  ||gd¬«      } | j                  d|||||dœ|¤Ž}|d   dd…d|j                  d«      …dd…f   }| j                  |«      }|j                  || j                   j"                  |z  d«      }|€Q | j$                  j'                  «       |«      }|| j                   j(                  k(  }|€‚t        j                  |«      }nl| | j'                  «       t        j*                  | j                   j(                  t        j                  |j                  ¬«      «      k(  }|j-                  d«      }|j/                  d«      j                  |j                  «      }|j                  |j                  |j0                  «      }|j3                  ||«      }| j                   j4                  r | j$                  d|||
d	œ|¤Ž}n | j$                  d|||||
d
œ|¤Ž}t7        |||¬«      S )aÕ  
        qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
            to serve as text prompt, which the Q-Former model will encode.

            Indices can be obtained using [`InstructBlipVideoProcessor`]. See [`InstructBlipVideoProcessor.__call__`] for
            details.

            [What are input IDs?](../glossary#input-ids)
        qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

            Only relevant in case an encoder-decoder language model (like T5) is used.
        ©r_   r^   r   NrI   r°  rN   r$   ©r‘   r¿   r’   r2  r3  ©r  r¿   r½  )r  r¿   r»  r¼  r½  r�  r  )rP   rS   r˜  r7   ÚonesrK   r²  r�   r­   rf   Ú	ones_likeÚrepeat_interleaverc   rX   r›  r�  r*   r™  rŸ  r$  Úvideo_token_idrà   r³  r´  ra   Úmasked_scatterÚuse_decoder_only_language_modelr…  )r@   r_   r¹  rº  r‘   r¿   r»  r¼  r  r^   r½  rÈ   rg   ÚframesÚchannelrE   rF   rˆ  Úimage_embedsÚimage_attention_maskr­   Úquery_attention_maskÚquery_outputsÚquery_outputÚlanguage_model_inputsr¶  Úoutputss                              rB   rl   zInstructBlipVideoModel.forward‡  sm  € ðP 6B×5GÑ5GÑ2ˆ
�F˜G V¨UØ#×+Ñ+¨J¸Ñ,?ÀÈ&ÐRWÓXˆà*˜×*Ñ*ð 
Ø%Ø%=ñ
ð ñ
ˆð
 & aÑ(ˆô  %Ÿz™z¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÔqÐð ×(Ñ(×/Ñ/°×0BÑ0BÀ1Ñ0EÀrÈ2ÓNˆÜ$Ÿz™z¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÔqÐà!Ð)Ü%*§_¡_Ð5FÓ%GÐ"à-×?Ñ?ÀÈAÐ?ÓNÐØ!7×!IÑ!IÈ&ÐVWÐ!IÓ!XÐØ!7×!:Ñ!:Ð;O×;VÑ;VÓ!WÐÜ!&§¡Ð,@ÐBXÐ+YÐ_`Ô!aÐØ$˜Ÿ™ð 
Ø'Ø1Ø%Ø".Ø#7ñ
ð ñ
ˆð % QÑ'ªÐ+A¨\×->Ñ->¸qÓ-AÐ+AÂ1Ð(DÑEˆð !%× 8Ñ 8¸Ó FÐð !6× =Ñ =¸jÈ$Ï+É+×JfÑJfÐioÑJoÐqsÓ tÐØÐ ØF˜D×/Ñ/×DÑDÓFÀyÓQˆMØ!*¨d¯k©k×.HÑ.HÑ!HÐØÐ%Ü!&§¡°Ó!;‘à!.Ð2M°$×2KÑ2KÓ2MÜ—‘˜TŸ[™[×7Ñ7¼u¿z¹zÐR_×RfÑRfÔgó3ñ "Ðð "4×!7Ñ!7¸Ó!;Ðà/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØ 5× 8Ñ 8¸×9MÑ9MÈ}×ObÑObÓ cÐØ%×4Ñ4Ð5GÐI^Ó_ˆà�;‰;×6Ò6Ø)�d×)Ñ)ð Ø+Ø-Ø#ñð ñ	‰Gð *�d×)Ñ)ð Ø+Ø-Ø"3Ø'=Ø#ñð ñˆGô DØ)Ø)Ø#*ô
ð 	
rC   )NNNNNNFN)rn   ro   rp   r%  Ú_keep_in_fp32_modulesr%   r1   r¬  r7   r‚  rs   r·  r   r   rq   rt   r   r   rð   r…  rl   ru   rv   s   @rB   r«   r«   N  sZ  ø„ ð %€OØ+Ð,ÐðÐ6õ ò?ð("¨e×.>Ñ.>ð "Èu×O`ÑO`ó "ð Øð
 ;?Ø.2Ø26Ø59Ø:>Ø-1Ø).Ø!%ñq
à×'Ñ'ðq
ð !×,Ñ,ðq
ð !&× 0Ñ 0°4Ñ 7ð	q
ð
 ×$Ñ$ tÑ+ðq
ð ×(Ñ(¨4Ñ/ðq
ð !×+Ñ+¨dÑ2ðq
ð !&× 0Ñ 0°4Ñ 7ðq
ð —|‘| dÑ*ðq
ð #'ðq
ð ˜$‘;ðq
ð Ð+Ñ,ðq
ð 
ÐEÑ	Eòq
ó ó ôq
rC   r«   c                   ó:   — e Zd ZU dZdZedz  ed<   dZedz  ed<   y)Ú'BaseModelOutputWithVisionQformerOutputszÝ
    vision_outputs (`BaseModelOutputWithPooling`):
        Outputs of the vision encoder.
    qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`):
        Outputs of the Q-Former (Querying Transformer).
    Nrˆ  r‰  )	rn   ro   rp   r–   rˆ  r   r±   r‰  r   r  rC   rB   rÓ  rÓ  ý  s)   … ñð 9=€NÐ.°Ñ5Ó<ØKO€OÐAÀDÑHÔOrC   rÓ  a¯  
    InstructBlipVideo Model for generating text given an image and an optional text prompt. The model consists of a vision
    encoder, Querying Transformer (Q-Former) and a language model.

    One can optionally pass `input_ids` to the model, which serve as a text prompt, to make the language model continue
    the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token.
    c                   óž  ‡ — e Zd ZU eed<   dZdZdgZdefˆ fd„Zd„ Z	de
j                  fd„Zdˆ fd
„	Zd„ Zd„ Zdej"                  dej$                  fd„Zee	 	 	 	 	 	 	 	 	 ddej$                  dej$                  dej"                  d	z  dej$                  d	z  dej"                  d	z  dej"                  d	z  dej"                  d	z  dej$                  d	z  dej"                  d	z  deded	z  dee   deez  fd„«       «       Z ej8                  «       	 	 	 	 	 	 ddej$                  dej"                  d	z  dej"                  d	z  dej"                  d	z  dej"                  d	z  dej$                  d	z  dedej"                  fd„«       Zee	 	 ddej$                  dej"                  dej"                  d	z  ded	z  dee   deez  fd„«       «       Zˆ xZ S ) rª   r*   r_   Tr­   c                 ó˜  •— t         ‰| �  |«       t        j                  |j                  «      | _        t        j                  t        j                  d|j                  |j                  j                  «      «      | _        t        j                  |j                  «      | _        t        j                   |j                  j                  |j"                  j                  «      | _        |j&                  r t)        j*                  |j"                  «      }nt-        j*                  |j"                  «      }|| _        | j1                  «        y r–  )r0   r1   r  Ú_from_configr—  r˜  r   r6   r7   rÛ   r™  rš  r2   r­   ru  r›  rØ   rœ  r�  rÇ  r"   rž  r#   rŸ  r  )r@   r*   rŸ  rA   s      €rB   r1   z2InstructBlipVideoForConditionalGeneration.__init__  sç   ø€ Ü‰Ñ˜Ô ä8×EÑEÀf×FZÑFZÓ[ˆÔäŸL™L¬¯©°Q¸×8OÑ8OÐQW×QfÑQf×QrÑQrÓ)sÓtˆÔÜ4×AÑAÀ&×BWÑBWÓXˆŒä#%§9¡9¨V×-BÑ-B×-NÑ-NÐPV×PbÑPb×PnÑPnÓ#oˆÔ à×1Ò1Ü1×=Ñ=¸f×>PÑ>PÓQ‰Nä2×>Ñ>¸v×?QÑ?QÓRˆNà,ˆÔð 	�‰ÕrC   c                 ó:   — | j                   j                  |«       y rô   )rŸ  Úset_output_embeddings)r@   Únew_embeddingss     rB   rØ  z?InstructBlipVideoForConditionalGeneration.set_output_embeddings/  s   € Ø×Ñ×1Ñ1°.ÕArC   rG   c                 ó6   — | j                   j                  «       S rô   )rŸ  Úget_output_embeddingsr#  s    rB   rÛ  z?InstructBlipVideoForConditionalGeneration.get_output_embeddings2  s   € Ø×"Ñ"×8Ñ8Ó:Ð:rC   Nc                 ó\   •— |€| j                   j                  «       S t        ‰| �  |¬«      S )N)Úmodality)rŸ  Úget_encoderr0   )r@   rÝ  rA   s     €rB   rÞ  z5InstructBlipVideoForConditionalGeneration.get_encoder5  s1   ø€ ØÐØ×&Ñ&×2Ñ2Ó4Ð4ä‘7Ñ&°Ð&Ó9Ð9rC   c                 ó6   — | j                   j                  «       S rô   )rŸ  Úget_decoderr#  s    rB   rà  z5InstructBlipVideoForConditionalGeneration.get_decoder;  s   € Ø×"Ñ"×.Ñ.Ó0Ð0rC   c                 ó  — | j                   }t        |«      dkD  r:d|vr6t        j                  j	                  «       dkD  rt
        j                  d«       t        | j                  d«      rd| j                  j                  _
        yyr¡  r£  r«  s     rB   r¬  z@InstructBlipVideoForConditionalGeneration._preprocess_accelerate>  r­  rC   r‘   r  c                 ój  — |€m| | j                  «       t        j                  | j                  j                  t        j
                  |j                  ¬«      «      k(  }|j                  d«      }n|| j                  j                  k(  }|j                  d«      j                  |j                  «      }|S r¯  )
r$  r7   rà   r*   rÅ  r²  r�   r³  r´  rc   rµ  s       rB   r·  z>InstructBlipVideoForConditionalGeneration.get_placeholder_maskR  r¸  rC   r¹  rº  r¿   r»  r¼  Úlabelsr^   r½  rÈ   c           
      óÂ  —  | j                   |f|||
dœ|¤Ž}|j                  }|j                  }|j                  }|€ | j	                  «       |«      }|€t        j                  |«      }|j                  |j                  |j                  «      }| j                  ||¬«      }|j                  ||«      }| j                  j                  rT | j                  d	|||dœ|¤Ž}|d   }d}|	�f | j                  d	||	| j                  j                   j"                  dœ|¤Ž}n1 | j                  d	|||||	|dœ|¤Ž}|j$                  }|j&                  }t)        |||||¬«      S )
a˜  
        qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)):
            The sequence used as a prompt to be fed to the Q-Former module.
        qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
            Mask to avoid performing attention on padding token indices.

        Examples:

        ```python
        >>> from transformers import InstructBlipVideoProcessor, InstructBlipVideoForConditionalGeneration
        >>> import torch
        >>> from huggingface_hub import hf_hub_download
        >>> import av
        >>> import numpy as np

        >>> def read_video_pyav(container, indices):
        ...     '''
        ...     Decode the video with PyAV decoder.
        ...     Args:
        ...         container (`av.container.input.InputContainer`): PyAV container.
        ...         indices (`list[int]`): List of frame indices to decode.
        ...     Returns:
        ...         result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
        ...     '''
        ...     frames = []
        ...     container.seek(0)
        ...     start_index = indices[0]
        ...     end_index = indices[-1]
        ...     for i, frame in enumerate(container.decode(video=0)):
        ...         if i > end_index:
        ...             break
        ...         if i >= start_index and i in indices:
        ...             frames.append(frame)
        ...     return np.stack([x.to_ndarray(format="rgb24") for x in frames])

        >>> model = InstructBlipVideoForConditionalGeneration.from_pretrained("Salesforce/instructblip-vicuna-7b", device_map="auto")
        >>> processor = InstructBlipVideoProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b")

        >>> file_path = hf_hub_download(
        ...       repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
        ... )
        >>> container = av.open(file_path)

        >>> # sample uniformly 4 frames from the videWhy is this video funny?o
        >>> total_frames = container.streams.video[0].frames
        >>> indices = np.arange(0, total_frames, total_frames / 4).astype(int)
        >>> clip = read_video_pyav(container, indices)

        >>> prompt = "What is happening in the video?"
        >>> inputs = processor(text=prompt, images=clip, return_tensors="pt").to(model.device)

        >>> outputs = model.generate(
        ...     **inputs,
        ...     do_sample=False,
        ...     num_beams=5,
        ...     max_length=256,
        ...     repetition_penalty=1.5,
        ...     length_penalty=1.0,
        ... )
        >>> generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip()
        >>> print(generated_text)
        "A person is eating a bowl of pasta, and they are using a fork to eat it. The person is sitting at a table, and the plate of pasta is on the table in front"
        ```©r¹  rº  r^   N©r  rÁ  r   )r‡  rã  r�   )r  r¿   r»  r¼  rã  r½  )r†  r‡  rˆ  r‰  rŠ  r  )Úget_video_featuresr  r‰  rˆ  r$  r7   rÃ  rc   r�   ra   r·  rÆ  r*   rÇ  rŸ  Úloss_functionrœ  r�   r†  r‡  r…  )r@   r_   r¹  rº  r‘   r¿   r»  r¼  r  rã  r^   r½  rÈ   Úvideo_featuresrÏ  r‰  rˆ  r¶  rÐ  r‡  r†  s                        rB   rl   z1InstructBlipVideoForConditionalGeneration.forwarda  s¼  € ð` CZÀ$×BYÑBYØðC
à/Ø#9Ø%=ñ	C
ð
 ñC
ˆð !/× <Ñ <ÐØ(×8Ñ8ˆØ'×6Ñ6ˆàÐ Ø7˜D×5Ñ5Ó7¸	ÓBˆMàÐ!Ü"Ÿ_™_¨YÓ7ˆNà 5× 8Ñ 8¸×9MÑ9MÈ}×ObÑObÓ cÐØ!×6Ñ6°yÐP]Ð6Ó^ÐØ%×4Ñ4Ð5GÐI^Ó_ˆà�;‰;×6Ò6Ø)�d×)Ñ)ð Ø+Ø-Ø#ñð ñ	ˆGð ˜Q‘ZˆFØˆDØÐ!Ø)�t×)Ñ)ð Ø!¨&¸T¿[¹[×=TÑ=T×=_Ñ=_ñØciñ‘ð
 *�d×)Ñ)ð Ø+Ø-Ø"3Ø'=ØØ#ñð ñˆGð —<‘<ˆDØ—^‘^ˆFäCØØØ)Ø+Ø#*ô
ð 	
rC   c                 óV  — t        | d«      r| j                  «        |j                  d   }	| j                  ||||¬«      }
|
j                  }|€°|€˜| j
                  j                  g| j
                  j                  z  dz  }|| j
                  j                  j                  gz   }t        j                  |gt        j                  |j                  ¬«      }|j                  |	d«      } | j                  «       |«      }|€t        j                   |«      }|j#                  |j                  |j$                  «      }| j'                  ||¬«      }|j)                  ||«      }||dœ}| j*                  j
                  j,                  s||d	<    | j*                  j.                  d
i |¤|¤Ž}|S )aÙ  
        Overrides `generate` function to be able to use the model as a conditional generator.

        Args:
            pixel_values (`torch.FloatTensor` of shape (batch_size, num_channels, height, width) or
                (batch_size, num_frames, num_channels, height, width)): Input images or videos to be processed.
            qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
                The sequence used as a prompt to be fed to the Q-Former module.
            qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
                Mask to avoid performing attention on padding token indices.
            input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
                The sequence used as a prompt for the generation.
            attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
                Mask to avoid performing attention on padding token indices.
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Embedded representation of the inputs. Should be float, not int tokens.
            interpolate_pos_encoding (`bool`, *optional*, defaults to `False`):
                Whether to interpolate the positional encoding of the image embeddings.

        Returns:
            captions (list): A list of strings of length batch_size * num_captions.
        r¤  r   rå  rç   r°  r$   ræ  )r  r¿   r‘   r  )r*  r¬  rP   rç  r  r*   Úvideo_token_indexr™  rœ  Úbos_token_idr7   rà   r²  r�   Úrepeatr$  rÃ  rc   ra   r·  rÆ  rŸ  Úis_encoder_decoderÚgenerate)r@   r_   r¹  rº  r‘   r¿   r  r^   Úgenerate_kwargsrg   ré  rÏ  Úvideo_tokensÚstart_tokensr¶  ÚinputsrÐ  s                    rB   rï  z2InstructBlipVideoForConditionalGeneration.generateé  sŸ  € ôD �4˜Ô)à×'Ñ'Ô)à!×'Ñ'¨Ñ*ˆ
ØBF×BYÑBYØØ/Ø#9Ø%=ð	 CZó C
ˆð !/× <Ñ <ÐàÐ ØÐ Ø $§¡× =Ñ =Ð>ÀÇÁ×A]ÑA]Ñ]Ð`aÑa�Ø+¨t¯{©{×/FÑ/F×/SÑ/SÐ.TÑT�Ü!ŸL™L¨,¨¼u¿z¹zÐR^×ReÑReÔf�	Ø%×,Ñ,¨Z¸Ó;�	Ø7˜D×5Ñ5Ó7¸	ÓBˆMàÐ!Ü"Ÿ_™_¨YÓ7ˆNà 5× 8Ñ 8¸×9MÑ9MÈ}×ObÑObÓ cÐØ!×6Ñ6°yÐP]Ð6Ó^ÐØ%×4Ñ4Ð5GÐI^Ó_ˆà#0ÀNÑSˆØ×"Ñ"×)Ñ)×<Ò<Ø"+ˆF�;Ñà.�$×%Ñ%×.Ñ.ÑK°ÐK¸?ÑKˆàˆrC   c           	      ó:  — |j                   \  }}}}	}
|j                  ||z  ||	|
«      } | j                  d
||dœ|¤Ž}t        |j                  |j
                  |j                  |j                  |d¬«      }|d   }t        j                  |j                  «       dd t        j                  |j                  ¬«      }| j                  j                  |j                   d   dd«      }t        j                  |j                  «       dd t        j                  |j                  ¬«      }|€t        j                  |«      }|j!                  |d¬«      }|j!                  |d¬«      }|j#                  |j                  «      }t        j$                  ||gd¬«      } | j&                  d
|||||d	œ|¤Ž}||_        |d   dd…d|j                  d«      …dd…f   }| j+                  |«      }|j                  || j,                  j.                  |z  d«      }||_        |S )a  
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The tensors corresponding to the input images.
        qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)):
            The sequence used as a prompt to be fed to the Q-Former module.
        qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
            Mask to avoid performing attention on padding token indices.
        r¿  N)r  r  rå   r  rˆ  r‰  r   rI   r°  rN   r$   rÀ  r  )rP   rS   r˜  rÓ  r  r  rå   r  r7   rÂ  rK   r²  r�   r­   rf   rÃ  rÄ  rc   rX   r›  r‰  r�  r*   r™  )r@   r_   r¹  rº  r^   rÈ   rg   rÈ  rÉ  rE   rF   rˆ  rÊ  rË  r­   rÌ  r‰  rÎ  ré  s                      rB   rç  z<InstructBlipVideoForConditionalGeneration.get_video_features/  s@  € ð( 6B×5GÑ5GÑ2ˆ
�F˜G V¨UØ#×+Ñ+¨J¸Ñ,?ÀÈ&ÐRWÓXˆà5F°T×5FÑ5Fð 6
Ø%Ø%=ñ6
ð ñ6
ˆô
 AØ,×>Ñ>Ø(×6Ñ6Ø(×6Ñ6Ø%×0Ñ0Ø)Ø ô
ˆð & aÑ(ˆô  %Ÿz™z¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÔqÐð ×(Ñ(×/Ñ/°×0BÑ0BÀ1Ñ0EÀrÈ2ÓNˆÜ$Ÿz™z¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÔqÐà!Ð)Ü%*§_¡_Ð5FÓ%GÐ"à-×?Ñ?ÀÈAÐ?ÓNÐØ!7×!IÑ!IÈ&ÐVWÐ!IÓ!XÐØ!7×!:Ñ!:Ð;O×;VÑ;VÓ!WÐÜ!&§¡Ð,@ÐBXÐ+YÐ_`Ô!aÐØ&˜$Ÿ,™,ð 
Ø'Ø1Ø%Ø".Ø#7ñ
ð ñ
ˆð *9ˆÔ&Ø& qÑ)ª!Ð-C¨|×/@Ñ/@ÀÓ/CÐ-CÂQÐ*FÑGˆð ×1Ñ1°,Ó?ˆð (×/Ñ/°
¸D¿K¹K×<XÑ<XÐ[aÑ<aÐceÓfˆØ'5ˆÔ$àÐrC   rô   )	NNNNNNNFN)NNNNNFr
  )!rn   ro   rp   r%   r±   r%  r¹   rÑ  r1   rØ  r   ÚModulerÛ  rÞ  rà  r¬  r7   r‚  rs   r·  r   r   rt   r   r   rð   r…  rl   r»   rï  rÓ  rç  ru   rv   s   @rB   rª   rª     sû  ø… ð $Ó#Ø$€Oà!ÐØ+Ð,ÐðÐ6õ ò(Bð; r§y¡yó ;õ:ò1ò?ð("¨e×.>Ñ.>ð "Èu×O`ÑO`ó "ð Øð
 ;?Ø.2Ø26Ø59Ø:>Ø26Ø*.Ø).Ø!%ñD
à×'Ñ'ðD
ð !×,Ñ,ðD
ð !&× 0Ñ 0°4Ñ 7ð	D
ð
 ×$Ñ$ tÑ+ðD
ð ×(Ñ(¨4Ñ/ðD
ð !×+Ñ+¨dÑ2ðD
ð !&× 0Ñ 0°4Ñ 7ðD
ð ×(Ñ(¨4Ñ/ðD
ð × Ñ  4Ñ'ðD
ð #'ðD
ð ˜$‘;ðD
ð Ð+Ñ,ðD
ð 
ÐEÑ	EòD
ó ó ðD
ðL €U‡]�]ƒ_ð 6:Ø:>Ø-1Ø26Ø26Ø).ñCà×'Ñ'ðCð !×+Ñ+¨dÑ2ðCð !&× 0Ñ 0°4Ñ 7ð	Cð
 ×#Ñ# dÑ*ðCð ×(Ñ(¨4Ñ/ðCð ×(Ñ(¨4Ñ/ðCð #'ðCð 
×	Ñ	òCó ðCðJ Øð
 ;?Ø05ñDà×'Ñ'ðDð !×+Ñ+ðDð !&× 0Ñ 0°4Ñ 7ð	Dð
 #'¨¡+ðDð Ð+Ñ,ðDð 
Ð8Ñ	8òDó ó ôDrC   rª   )r  r˜   ru  r«   rª   )r¢   )QÚcollections.abcr   Údataclassesr   Útypingr   r7   r   Ú r   r¨   Úactivationsr	   Ú
generationr
   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úautor!   r"   r#   Úconfiguration_instructblipvideor%   r&   r'   Ú
get_loggerrn   r¨  rõ  r)   rx   r˜   rq   ÚfloatrË   rœ   rò   r�   r  r  rŸ   r    rF  rM  rS  rž   rn  ru  r…  r«   rÓ  rª   Ú__all__r  rC   rB   ú<module>r
     s�  ðõ, %Ý !Ý ã Ý å &Ý !Ý )Ý 6Ý 9÷÷ ÷ GÝ &Ý 6ß j× jÝ 7ß Eß IÑ I÷ñ ð 
ˆ×	Ñ	˜HÓ	%€ôG¨¯	©	ô GôT/¨¯©ô /ðd ô i ó  ió ð iðV ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ó%ô.G) §¡ô G)ôT˜2Ÿ9™9ô ôÐ$>ô ô@@˜rŸy™yô @ô@3Ð#Cô 3ôlE)°·±ô E)ôP¨¯©ô ô ¨¯	©	ô  ô2¨2¯9©9ô ô R§Y¡Yô ôRÐ$>ô Rôj!
 b§i¡iô !
ôHa
Ð$Dô a
ñH ðôð
 ô
¸;ó 
ó óð
ñ: ðôô
g
Ð=ó g
óð
g
ðT Ø
ô	PÐ.Hó 	Pó ó ð	Pñ ðôôaÐ0PÐRaó aóðaòH�rC   