
    ^j:8                         d Z ddlZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlmZmZ ddlmZmZmZmZ dd	lmZmZmZ dd
lmZmZ erddlmZ  ej6                  e      Z G d ded      Ze G d de             ZdgZ y)z
Processor class for Idefics3.
    N)
accumulate)TYPE_CHECKINGUnion   )BatchFeature)
ImageInputis_valid_image)MultiModalDataProcessingKwargsProcessorMixinUnpack)
AddedTokenBatchEncoding	TextInput)auto_docstringlogging)PreTokenizedInputc                   $    e Zd ZdddddddidZy)Idefics3ProcessorKwargsTF)add_special_tokenspaddingis_split_into_wordsreturn_mm_token_type_idsreturn_row_col_info)text_kwargsimages_kwargsN)__name__
__module____qualname__	_defaults     {/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/idefics3/processing_idefics3.pyr   r   %   s(     #'#((-	
 "4

Ir"   r   F)totalc                   n    e Zd ZeZ	 ddededz  f fdZe	 	 	 dde	e
e	   z  e
e
e	      z  deede
e   e
d   f   dedz  dee   d	ef
d
       Z	 	 dde	dz  deede
e   e
d   f   dee   fdZ	 	 dde	dz  deede
e   e
d   f   dee   f fdZdeded	efdZde
de
e   d	e
e
e      fdZddZ xZS )Idefics3ProcessorNimage_seq_lenchat_templatec                 :   t        ddd      j                  | _        t        ddd      j                  | _        t        ddd      j                  | _        d| _        || _        |j                  | j                        | _        |j                  | j                        | _	        |j                  | j
                        | _
        t        d      D cg c]0  }t        d      D ]   }|j                  d	|d
z    d|d
z    d      " 2 c}}| _        t        j                  d      | _        d| j                  | j                  | j                  gi}|j!                  |       |j                  | j                        | _        t#        	| H  ||fd|i| yc c}}w )a  
        image_seq_len (`int`, *optional*, defaults to 169):
            The length of the image sequence i.e. the number of <image> tokens per image in the input.
            This parameter is used to build the string from the input prompt and image tokens and should match the
            value the model used. It is computed as: image_seq_len = int(((image_size // patch_size) ** 2) / (scale_factor**2))
        z<fake_token_around_image>FT)
normalizedspecialz<image>z<end_of_utterance>z<global-img>   <row_   _col_>z*(\n?<global-img>\n?|<row_\d+_col_\d+>\n?)+additional_special_tokensr(   N)r   contentfake_image_tokenimage_tokenend_of_utterance_tokenglobal_image_tagr'   convert_tokens_to_idsimage_token_idfake_image_token_idglobal_image_token_idrangerow_col_idsrecompile%_regex_to_remove_extra_special_tokensr   super__init__)
selfimage_processor	tokenizerr'   r(   kwargsijtokens_to_add	__class__s
            r#   rA   zIdefics3Processor.__init__7   s    !++FSXbf g o o%iE4PXX&01ERWae&f&n&n# .*'==d>N>NO#,#B#B4CXCX#Y %.%D%DTEZEZ%["SXYZS[
NOejklem
`aI++eAE7%Awa,HI
I
 68ZZ@m5n2 (%%  ++*
 	$$]3'==d>N>NO)[=[TZ[%
s   5Fimagestextr   rE   returnc                     | j                   d||d|\  }} | j                  d||d|  | j                  t        fd| j                  j
                  i|}||n| j                  }|d   j                  dd      }|d   j                  dd      }|d   j                  dd      }i x}	}
| | j                  |fi |d	   \  }	}|	j                  d
d       |	j                  dd       || j                  ||      \  }} | j                  |fi |d   }
|r||
d<   g }t        |      D ]e  \  }}g }|D ]H  }|d   \  }}|
j                  ||      }|
j                  ||dz
        }|j                  ||z
  dz          J |j                  |       g |r| j                  |
d   |      |
d<   | j                  ||
dg       n| | j                  dd|i|d   }
t        i |
|	|      S )a  
        image_seq_len (`int`, *optional*):
            The length of the image sequence. If not provided, the default value of self.image_seq_len is used.
            image_seq_len should be equal to int(((image_size // patch_size) ** 2) / (scale_factor**2))
        )rJ   rK   tokenizer_init_kwargsNr   return_text_replacement_offsetsFr   return_tensorsr   rowscols)images_replacementstext_replacement_offsetsnew_spanr.   	input_idsmm_token_type_idsimage)
modalitiesrK   )datatensor_typer!   )prepare_inputs_layoutvalidate_inputs_merge_kwargsr   rD   init_kwargsr'   pop_process_imagesget_text_with_replacements	enumeratechar_to_tokenappendcreate_mm_token_type_ids_check_special_mm_tokensr   )rB   rJ   rK   r'   rE   output_kwargsrO   r   rP   image_inputstext_inputsrS   rT   batch_image_seq_lengthsbatch_idtext_replacement_offsetimage_seq_lensrZ   startendstart_id_pos
end_id_poss                         r#   __call__zIdefics3Processor.__call__\   s    2t11UdUfU@F@@***#
"&.."<"<
 
 *7)BHZHZ*7*F*J*JKlns*t'#0#?#C#CD^`e#f &}599:JDQ%''{0D0D0DV0n}]lOm0n-L- VT*VT*151P1P.A 2Q 2.. -dnnTR]=5QR2>VK :;*,'9BC[9\ C5H5%'N 7 M%)*%5
s'2'@'@5'Q%0%>%>xq%Q
&--j<.G!.KLM ,22>BC ,7;7T7T#K02I8K 34 --dKWI-V($..SdSmM6RSK!@K!@<!@n]]r"   c                 B   |#t        |t              r|g}|j                         }|| j                  j	                  |      }t        |      r|gg}||fS t        |t        t        f      rt        |d         r||D cg c]  }|j                  | j                         }}dgt        t        |            z   }t        t        |            D cg c]  }|||   ||dz        }}t        |      |d   kD  r|||d   d  gz   }||fS |}||fS |g}||fS c c}w c c}w )Nr   r.   )
isinstancestrcopyrC   fetch_imagesr	   listtuplecountr4   r   r;   len)	rB   rJ   rK   rE   samplen_images_in_textcumsum_images_in_textrF   split_imagess	            r#   r\   z'Idefics3Processor.prepare_inputs_layout   sk    $$v99;D))66v>Ff%!($ t|# FT5M2~fQi7P#UY'Z6T5E5E(F'Z$'Z-.C$zBR7S2T,T) "'s+;'<!=$ 4Q7:OPQTUPU:VW$L $
 6{%:2%>>!-8Mb8Q8S1T0U!U t|	 ". t| %XFt| ([$s    "DDc                    t        |   ||fi | ||t        d      ||D cg c]  }|j                  | j                         }}|J|D cg c]  }t        |       }}||k7  r,t        d| j                   d| d| j                   d| d	      y |1t        |      r%t        dt        |       d| j                   d      y y y c c}w c c}w )	Nz+You must provide either `text` or `images`.zThe total number of zP tokens in the prompts should be the same as the number of images passed. Found  z tokens and z images per sample.zFound z. tokens in the text but no images were passed.)r@   r]   
ValueErrorr|   r4   r}   anysum)	rB   rJ   rK   rE   r~   r   sublistn_images_in_imagesrI   s	           r#   r]   z!Idefics3Processor.validate_inputs   s    	77<FNJKKMQR6T-=-= >RR!BH%Iwc'l%I"%I#'99$.t/?/?.@ A""2!31T5E5E4FlSeRffy{  :
 C(8$9 S!1231T5E5E4FFtu  %: R%Is   "CCri   	image_idxc           	         |d   D cg c]  }|D ]  }|  c}}|   }|d   D cg c]  }|D ]  }|  c}}|   }|dk(  rI|dk(  rD| j                    | j                   z   | j                   | j                  z  z   | j                    z   S d}	t	        |      D ]R  }
t	        |      D ]=  }|	| j                    d|
dz    d|dz    dz   | j                   | j                  z  z   z  }	? |	d	z  }	T |	d	| j                    | j                   z   | j                   | j                  z  z   | j                    z   z  }	|	S c c}}w c c}}w )
NrQ   rR   r    r-   r.   r/   r0   
)r3   r6   r4   r'   r;   )rB   ri   r   row_listrow
image_rowscol_listcol
image_colstext_split_imagesn_hn_ws               r#   replace_image_tokenz%Idefics3Processor.replace_image_token   s   *6v*>Sh(S3cScST]^
*6v*>Sh(S3cScST]^
?zQ(()**+-%%&$*<*<<= **+- !#Z( * , C%001!#'%ay:;!--.$2D2DDE% "T)!* T**+,**+-%%&$*<*<<= **+- %$5 TSs
   D:E rV   rk   c                    g }t        |      D ]  \  }}t        j                  ||         }t        j                  |      }t        j                  || j
                  k(        d   }d}	|D ]7  }
|	t        |      k\  r n'||	   }||
z   }d||| t        j                  ||      }	9 |j                  |j                                 |S )Nr   r.   )
rc   nparray
zeros_likewherer9   r}   searchsortedre   tolist)rB   rV   rk   rW   rF   seq_lengths	array_idsmm_token_typesimage_start_positionsrG   seq_lenro   rp   s                r#   rf   z*Idefics3Processor.create_mm_token_type_ids   s     '(?@ 	>NA{1.I]]95N$&HHY$:R:R-R$STU$V!A& @122-a0go,-uS)OO$93?@ $$^%:%:%<=	> ! r"   c                     i }|t         j                  j                  di       }|j                  |       |D cg c]   } | j                  j
                  g || " }}| j                  dz   }| j                  dz   }t        | j                  dd      d         dz
  }	g }
g }|D ]B  \  }}}||z  dz   }|d	kD  r|	nd	}|
j                  ||z   ||z  z          |j                  |       D |j                  |
|d
       t        di |S c c}w )a  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.

        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.

        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        r   r      z

F)r   rV   r.   r   )num_image_tokensnum_image_patchesr!   )r   r    getupdaterC   get_number_of_image_patchesr'   r}   rD   re   r
   )rB   image_sizesrE   vision_datar   
image_sizenum_image_row_colsbase_image_length
col_lengthextra_split_newliner   r   num_patchesnum_rowsnum_cols
row_lengthsplit_extras                    r#   _get_num_multimodal_tokensz,Idefics3Processor._get_num_multimodal_tokens
  sR    "3==AA/SUVM  ( #." A$$@@\*\m\" "
 !% 2 2Q 6++a/J
 #&dnnVPUn&VWb&c"dgh"h! "3E 6/Xx'(2Q6
5=\1q ''(9K(G:X`K`(ab!((5	6 4D[lmn,,,/"s   %D)N   N)NNN)NN)N)r   r   r   r   valid_processor_kwargsintrw   rA   r   r   rz   r   r   r   r   rs   r\   r   r]   dictr   rf   r   __classcell__)rI   s   @r#   r&   r&   3   s   4 fj#\>A#\X[^bXb#\J  JNbf$(	>^T*--T*5E0FF>^ I2DOTJ]E^^_>^ Tz	>^
 01>^ 
>^ >^D %)bf T!  I2DOTJ]E^^_  01	 H %)bfT! I2DOTJ]E^^_ )*	2% % % %:!$ !QUVYQZ !_cdhildm_n !*)-r"   r&   )!__doc__r=   	itertoolsr   typingr   r   numpyr   feature_extraction_utilsr   image_utilsr   r	   processing_utilsr
   r   r   r   tokenization_utils_baser   r   r   utilsr   r   r   
get_loggerr   loggerr   r&   __all__r!   r"   r#   <module>r      s    
   '  4 5 X X K K , <			H	%.e  - - -D 
r"   