^j8JddlZddlZddlmZddlmZmZddl m Z m Z m Z m Z ddlmZmZddlmZmZmZddlmZdd lmZmZmZerddlmZej:eZd Z d Z!d Z"d Z#dZ$de%dde%e&fdZ'de&de(e&e&ffdZ)GddedZ*GddeZ+dgZ,y)N) functional) BatchFeature get_size_dict)IMAGENET_STANDARD_MEANIMAGENET_STANDARD_STDPILImageResamplingSizeDict)Unpack VideosKwargs) TensorTypeis_torchvision_availablelogging)BaseVideoProcessor) VideoMetadatagroup_videos_by_shapereorder_videoszYou are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.zgYou are provided the following series of {frame_count} frames from a {video_duration} [H:MM:SS] video. z z Frame from {timestamp}:ivideos torch.Tensorreturnctdx}}|D]0}|jdd\}}t||}t||}2||fS)zH Get the maximum height and width across all videos in a batch. z-infN)floatsizemax)r max_height max_widthvideoheightwidths /var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/smolvlm/video_processing_smolvlm.pyget_max_height_widthr"0s\#6]*J* RS) , y) *  ""resolution_max_sidec:|jdd\}}tt|}| t||n|}||z }||k\r|}t ||z }|dzdk7r(|dz }n"||kDr|}t ||z}|dzdk7r|dz }t|d}t|d}||fS)a Get the output size of the video after resizing given a dictionary specifying the max and min sizes. Args: video (`np.ndarray`): Video to resize. resolution_max_side (`int`): The longest edge of the video will be resized to this value. The shortest edge will be resized to keep the input aspect ratio. Returns: The output size of the video after resizing. rNr)rminMAX_IMAGE_SIZErint)rr$rr aspect_ratios r!get_resize_output_image_sizer,<sJJL%MFEn.AB0C0K#fe,Qd6>L #U\)* A:? aKF %$F\)* 19> QJE ^F qME 5=r#c"eZdZUeeefed<y)SmolVLMVideoProcessorInitKwargsmax_image_sizeN)__name__ __module__ __qualname__dictstrr*__annotations__r#r!r.r.dscN"r#r.F)totalczeZdZejZddiZddiZeZ e Z dZ dZ dZdZdZdZeZddgZdeeffd Z d(d d d edddedd f fd Z d)d d deeefdededef dZ d*deded zdeezd zded zfdZ d+de d deded eddd ed!ed"ed#ed$ee ezd zd%ee ezd zd&e!e"zd zfd'Z#xZ$S),SmolVLMVideoProcessor longest_edgeiilTF pixel_valuespixel_attention_maskkwargsc t|di|d|vrd|vr |d|dd<d|vr=|dd|_|dd|_t |dd|j |_yy)Nrvideo_sampling video_size max_framesfps)default_to_squarer6)super__init__ num_framesrBrrCr)selfr= __class__s r!rEzSmolVLMVideoProcessor.__init__ws "6" V  0F :5;F^F# $\ 2 v %$%56|DDO./6DH%f-=&>|&L`d`v`vwDI &r#Nrrrresamplez7PILImageResampling | tvF.InterpolationMode | int | None antialiasrc |jrt||j}n@|jr%|jr|j|jf}nt d|dt ||t|d|d||}t|jd|jd}t |||||}|S) a' Resize an video to `(size["height"], size["width"])`. Args: video (`torch.Tensor`): Video to resize. size (`SizeDict`): Dictionary in the format `{"height": int, "width": int}` specifying the size of the output video. resample (`PILImageResampling` or `InterpolationMode`, *optional*, defaults to `InterpolationMode.BILINEAR`): Resampling filter to use when resizing the video. Returns: `torch.Tensor`: The resized video. )r$zHSize must contain 'height' and 'width' keys, or 'longest_edge' key. Got .rr')rr )rIrJr:) r:r,rr ValueErrorrDresizer r/) rGrrrIrJr=new_sizemax_sizerHs r!rNzSmolVLMVideoProcessor.resizes(   4$($5$5H[[TZZ TZZ0Hghlgmmnop p 88A;hqkBXaj  4#6#6~#FdNaNabpNqruhYW r# padded_sizemax_num_framesfillreturn_pixel_maskc |jdd}|jdk(r|jdn|jd}|d|dz }|d|dz } ||z } | dks |dks| dkrtd|d|d|d |d ||k7s| dkDr"d| d|ddd| g} t j || | }d} |rBt j|d dddddft j } d| d d|dd|df<|| fS)a*Pads the sample with empty video to the padded_size Args: video (`torch.Tensor`): Batched video to pad. padded_size (`tuple[int, int]`): Height and width to pad. max_num_frames (`int`): The maximum number of frames to which video will be padded. fill (`int`, *optional*): The value to use for the padding. return_pixel_mask (`bool`, *optional*, defaults to `True`): Whether to return a pixel mask. rNr'rzPadding dimensions are negative. Please make sure that the padded size is larger than the original size. Got padded max number of frames z and padded size: z, original number of frames z and size: rL)value.dtype) rndimshaperMFpadtorch zeros_likeint64) rGrrQrRrSrT original_sizerFpadding_height padding_width padding_framepadding pixel_masks r!r]zSmolVLMVideoProcessor.padsQ* RS) ',zzQU[[^EKKN $Q-*::#Aq)99 &3 1  2ma6GBBPAQQcdocpq--7L M?RSU  K '=1+<-NAq!]SGEE%5E ))%Q1 *=U[[QJFGJs.mA..0B-2B0BB Cj  r#metadatarFrB skip_secsc "| t|dd td||n |j}||n |j}|j}t t ||dz}t||}|dkrd}d} |dz } |dkDr6|dd|zz ||zkDr%t ||dz} t |||dzz } td| } t| |dz } | | k\rd|dz } } tj| | |t } tj| } | S) a! Video sampling function which: - Uses `num_frames` (if provided) or calculates it from `fps` and metadata. - Applies a basic center-skip if fewer frames than available, otherwise optionally skips `skip_secs` from both the start and end. - Uniformly samples the desired number of frames between the start and end indices. Args: metadata (`VideoMetadata`): Metadata of the video containing information about total duration, fps and total number of frames. num_frames (`int`, *optional*): Maximum number of frames to sample. Defaults to `self.num_frames`. fps (`int` or `float`, *optional*): Target frames to sample per second. Defaults to `self.fps`. skip_secs (`float`, *optional*, defaults to `1`): Number of seconds to skip from the start and end if the video is long enough. Returns: np.ndarray: Indices to sample video frames. NrBzAsked to sample frames per second but no video metadata was provided which is required when sampling in SmolVLM. Please pass in `VideoMetadata` object or set `do_sample_frames=False`durationr'rr&rX) getattrrMrFrBtotal_num_framesr*roundr(rnplinspaceunique) rGrgrFrBrhr=rlestimated_framesdesired_frames start_idxend_idxindicess r! sample_framesz#SmolVLMVideoProcessor.sample_framessK:  wx=EX  $.#9Zt _c$((#44uS8J+?%?@A-z: A N "Q& q=hz2Q]BzTWGWXI78I*Y%-HHIG9% g/!34  !"$4q$8wI++i.L))G$r#rdo_convert_rgb do_resize do_rescalerescale_factor do_normalizedo_pad image_mean image_stdreturn_tensorsc t|\}}i}|jD]3\}}|r|j|}|r|j|||}|||<5t ||}t|\}}i}|jD] \}}|j ||||| | }|||<"t ||}| rt |}td|D}t|\}}i}i}|jD]&\}}|j|||\}}|||<|||<(t ||}t ||}d|i}| r | r| tjdn|d<t|| S) N)rrIc32K|]}t|ywN)len).0rs r! z4SmolVLMVideoProcessor._preprocess..As JU Js)rQrRr;r)dimr<) tensor_type) ritemsconvert_to_rgbrNrrescale_and_normalizer"rr]r^stackr)rGrrwrxrrIryrzr{r|r}r~rr=grouped_videosgrouped_videos_indexresized_videos_groupedr[stacked_videosresized_videosprocessed_videos_groupedprocessed_videospad_sizerRprocessed_padded_mask_grouped padded_masksr<datas r! _preprocessz!SmolVLMVideoProcessor._preprocesss 0EV/L,,!#%3%9%9%; ; !E>!%!4!4^!D!%^$QY!Z,: "5 )  ; ((>@TU/D^/T,,#% %3%9%9%; = !E>!77 NL*V_N/= $U +  = **BDXY +,<=H J9I JJN3HIY3Z 0N0,. )') $)7)=)=)? D%~/3xx"080, 3A(/7C-e4  D ..FH\] #12OQe#f  01 n8 0a8) ' ( Dn==r#)NT)rT)NNr'r)%r0r1r2r LANCZOSrIrr/rr}rr~rxryr{rwr|do_sample_framesr. valid_kwargsmodel_input_namesr rEr boolrNtupler*r]rrrvlistr4r r __classcell__)rHs@r!r9r9hs!))H G $D$c*N'J%IIJLN F2L')?@ x(G!H x OS (((L (  ( (^"& +!+!38_+! +!  +!  +!`"&"& ! ??$J?5[4  ? : ?\37=>^$=>=> =>  => L =>=>=>=>=>DK'$.=>4;&-=>j(4/=>r#r9)-numpyrnr^torch.nnrr\image_processing_utilsrr image_utilsrrr r processing_utilsr r utilsr rrvideo_processing_utilsr video_utilsrrrtorchvision.transforms.v2tvF get_loggerr0loggerDEFAULT_SYSTEM_MESSAGEDEFAULT_VIDEO_INTRODEFAULT_MEDIA_OUTTROFRAME_TIMESTAMP_MESSAGEr)rr*r"rr,r.r9__all__r6r#r!rs  $A 5BB8OO;   H %Vn5 #n!5 #$s) #%% 38_%P#l%#p>.p>f # #r#