^j8ddlZddlmZddlmZddlmZmZddlmZddl m Z ddl m Z dd l mZdd lmZmZmZmZmZdd lmZd d lmZmZd dlmZd dlmZd dlmZm Z m!Z!m"Z"m#Z#ejHe%Z&edeGddeZ'Gdde"Z(Gdde#Z)edeGddeZ*GddeZ+GddeZ,Gdd eZ-Gd!d"e!Z.Gd#d$e Z/gd%Z0y)&N)strict)nn)Cache DynamicCache)GenerationConfig)FlashAttentionKwargs)BaseModelOutputWithPooling)Unpack)TransformersKwargsauto_docstringcan_return_tupleloggingtorch_compilable_check)merge_with_config_defaults)Idefics3ConfigIdefics3VisionConfig)Idefics3ImageProcessor)Idefics3ImageProcessorPil)Idefics3BaseModelOutputWithPast Idefics3ForConditionalGeneration Idefics3ModelIdefics3PreTrainedModelIdefics3VisionTransformerz$HuggingFaceTB/SmolVLM2-2.2B-Instruct) checkpointceZdZdZdZy)SmolVLMVisionConfiga Example: ```python >>> from transformers.models.smolvlm.modeling_smolvlm import SmolVLMVisionTransformer >>> from transformers.models.smolvlm.configuration_smolvlm import SmolVLMVisionConfig >>> # Initializing a SmolVLMVisionConfig with google/siglip-so400m-patch14-384 style configuration >>> configuration = SmolVLMVisionConfig() >>> # Initializing a SmolVLMVisionTransformer (with random weights) from the google/siglip-so400m-patch14-384 style configuration >>> model = SmolVLMVisionTransformer(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```smolvlm_visionN__name__ __module__ __qualname____doc__ model_typev/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/smolvlm/modular_smolvlm.pyrr+s ""Jr'rc eZdZy)SmolVLMPreTrainedModelNr!r"r#r&r'r(r*r*Br'r*c eZdZy)SmolVLMVisionTransformerNr+r&r'r(r.r.Fr,r'r.ceZdZdZdZy) SmolVLMConfiga scale_factor (`int`, *optional*, defaults to 2): The scale factor for the image encoder. Example: ```python >>> from transformers import SmolVLMModel, SmolVLMConfig >>> # Initializing configuration >>> configuration = SmolVLMConfig() >>> # Initializing a model from the configuration >>> model = SmolVLMModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```smolvlmNr r&r'r(r0r0Js  Jr'r0c eZdZy)SmolVLMImageProcessorNr+r&r'r(r3r3_r,r'r3c eZdZy)SmolVLMImageProcessorPilNr+r&r'r(r5r5cr,r'r5c eZdZy)SmolVLMBaseModelOutputWithPastNr+r&r'r(r7r7gr,r'r7ceZdZdZdej dej dej fdZee d dd ejd ej dzd e e d e ezfd Zeee d ddej dzdej dzdej dzdedzdejdzd ejdzd ej$dzdejdzdedzd e ed e ezfdZy) SmolVLMModelz A subclass of Idefics3Model. We do *not* remove or block the call to inputs_merger in forward. Instead, we override inputs_merger here with custom logic. input_ids inputs_embedsimage_hidden_statesc |j\}}}|a||jtj|jj tj |jk(}|d}n||jj k(}|jd}ttj||zdk(d||z}tjjj|jddd} | dd } |jd } | dz |z} | dz |z} | jd| z}tj |}|||| |ddf||<tj"|jd ||}|S) Ndtypedevice).rdimrzCAt least one sample has tokens not divisible by patch_size.)rAr)value)shapeget_input_embeddingstorchtensorconfigimage_token_idlongr@sumrallr functionalpadcumsum unsqueeze zeros_likewhere)selfr:r;r<_ patch_size image_masknum_image_tokensblocks_per_sampleoffsets block_offsetrow_cum chunk_idx local_idx block_idx image_embeds merged_embedss r( inputs_mergerzSmolVLMModel.inputs_mergerqs/44:q  &*E$*C*C*E T[[77uzzR_RfRfg+J$F+J"dkk&@&@@J%>>a>0 II&3q8 9 Q - :((%%))*;*B*Bq*B*I6YZ)[s| ###+q[Z/ q[J.  **1- 9 '' 6 #6y7LiXbNcef7f#g Z  J$8$8$P>P; J fe#TZZ8 (|((j)@Z?l((+?jj#**$ $=#7#<# (batch_size, max_num_images, 3, max_heights, max_widths) where max_num_images is the maximum number of images among the batch_size samples in the batch. Padding images are not needed beyond padding the pixel_values at the entrance of the model. For efficiency, we only pass through the vision_model's forward the real images by discarding the padding images i.e. pixel_values of size (image_batch_size, 3, height, width) where image_batch_size would be 7 when num_images_per_sample=[1, 3, 1, 2] and max_num_images would be 3. attention_mask position_idspast_key_values use_cachec ||j\} } n||j\} } } n td| r|t|j}|9|jj |j |j}| | td|:|j||dj}|j |j}n)|'|j |j|j}||j|||}|jd ||||| d| }t|j|j|j|j | S) Nz5You have to specify either input_ids or inputs_embeds)rJzMYou cannot specify both pixel_values and image_hidden_states at the same timeT)rpr>)r:r;r<)r;rrrr)r{r hidden_states attentionsr<r&)rF ValueErrorrrJ text_modelrGrqr@rr}r?rcr7r{rrr)rUr:rrrr;rerfr<rrgr~ seq_lengthrVoutputss r(forwardzSmolVLMModel.forwards4  %.__ "J  &(5(;(; %J ATU U 0*$++>O  BDOO@@B9MPPQZQaQabM  #(;(Glm m  #"&"9"92#:#m #6"8"89M9M"N  ,"5"8"8tzzR_RfRf"8"g   * ..#+$7/M "$// ')%+    .%77#33!//)) 3   r')N) NNNNNNNNN)r!r"r#r$rH LongTensorTensorrcrr FloatTensorr r tupler rrr BoolTensorrwr r7rr&r'r(r9r9ks )):?,,]b]i]iBm 9=2''2$..52+, 2 + + 22h   .2.204(,26158<8>> import httpx >>> from io import BytesIO >>> import torch >>> from PIL import Image >>> from io import BytesIO >>> from transformers import AutoProcessor, AutoModelForImageTextToText >>> from transformers.image_utils import load_image >>> # Note that passing the image urls (instead of the actual pil images) to the processor is also possible >>> image1 = load_image("https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg") >>> image2 = load_image("https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg") >>> image3 = load_image("https://cdn.britannica.com/68/170868-050-8DDE8263/Golden-Gate-Bridge-San-Francisco.jpg") >>> processor = AutoProcessor.from_pretrained("HuggingFaceTB/SmolVLM2-2.2B-Instruct") >>> model = AutoModelForImageTextToText.from_pretrained("HuggingFaceTB/SmolVLM2-2.2B-Instruct", dtype=torch.bfloat16, device_map="auto") >>> # Create inputs >>> messages = [ ... { ... "role": "user", ... "content": [ ... {"type": "video", "path": path/to/video}, ... {"type": "text", "text": "What is happening in this video?"}, ... ] ... } ... ] >>> inputs = processor.apply_chat_template([messages], add_generation_prompt=True) >>> # Generate >>> generated_ids = model.generate(**inputs, max_new_tokens=256) >>> generated_texts = processor.batch_decode(generated_ids, skip_special_tokens=True) >>> print(generated_texts) ```Nr&)rr)rU super_kwargsrs r(rz'SmolVLMForConditionalGeneration.forwardsd ','r')r!r"r#_tied_weights_keysrr __classcell__)rs@r(rrs*,RS2(2(r'r)rr0r3r5rr*r9r.)1rHhuggingface_hub.dataclassesrr cache_utilsrr generationrmodeling_flash_attention_utilsr modeling_outputsr processing_utilsr utilsr r rrr utils.genericridefics3.configuration_idefics3rr"idefics3.image_processing_idefics3r&idefics3.image_processing_pil_idefics3ridefics3.modeling_idefics3rrrrr get_loggerr!loggerrr*r.r0r3r5r7r9r__all__r&r'r(rs " ..*B:&jj7RGN   H %AB"."C"* 4  8 ABNC& 2  8  %D g =g T<(&F<(~ r'