Ë
    ÛÓ^j:8  ã                   ó   — d Z ddlZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlmZmZ ddlmZmZmZmZ dd	lmZmZmZ dd
lmZmZ erddlmZ  ej6                  e«      Z G d„ ded¬«      Ze G d„ de«      «       ZdgZ y)z
Processor class for Idefics3.
é    N)Ú
accumulate)ÚTYPE_CHECKINGÚUnioné   )ÚBatchFeature)Ú
ImageInputÚis_valid_image)ÚMultiModalDataÚProcessingKwargsÚProcessorMixinÚUnpack)Ú
AddedTokenÚBatchEncodingÚ	TextInput)Úauto_docstringÚlogging)ÚPreTokenizedInputc                   ó$   — e Zd ZdddddœddidœZy)ÚIdefics3ProcessorKwargsTF)Úadd_special_tokensÚpaddingÚis_split_into_wordsÚreturn_mm_token_type_idsÚreturn_row_col_info)Útext_kwargsÚimages_kwargsN)Ú__name__Ú
__module__Ú__qualname__Ú	_defaults© ó    ú{/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/idefics3/processing_idefics3.pyr   r   %   s(   „ ð #'ØØ#(Ø(-ñ	
ð " 4ð
ñ
�Ir"   r   F)Útotalc                   ón  ‡ — e Zd ZeZ	 ddededz  fˆ fd„Ze	 	 	 dde	e
e	   z  e
e
e	      z  deede
e   e
d   f   dedz  dee   d	ef
d
„«       Z	 	 dde	dz  deede
e   e
d   f   dee   fd„Z	 	 dde	dz  deede
e   e
d   f   dee   fˆ fd„Zdeded	efd„Zde
de
e   d	e
e
e      fd„Zdd„Zˆ xZS )ÚIdefics3ProcessorNÚimage_seq_lenÚchat_templatec                 ó:  •— t        ddd¬«      j                  | _        t        ddd¬«      j                  | _        t        ddd¬«      j                  | _        d| _        || _        |j                  | j                  «      | _        |j                  | j                  «      | _	        |j                  | j
                  «      | _
        t        d«      D ��cg c]0  }t        d«      D ]   }|j                  d	|d
z   › d|d
z   › d�«      ‘Œ" Œ2 c}}| _        t        j                  d«      | _        d| j                  | j                  | j                  gi}|j!                  |«       |j                  | j                  «      | _        t#        ‰	| �H  ||fd|i|¤Ž yc c}}w )aŸ  
        image_seq_len (`int`, *optional*, defaults to 169):
            The length of the image sequence i.e. the number of <image> tokens per image in the input.
            This parameter is used to build the string from the input prompt and image tokens and should match the
            value the model used. It is computed as: image_seq_len = int(((image_size // patch_size) ** 2) / (scale_factor**2))
        z<fake_token_around_image>FT)Ú
normalizedÚspecialz<image>z<end_of_utterance>z<global-img>é   ú<row_é   Ú_col_ú>z*(\n?<global-img>\n?|<row_\d+_col_\d+>\n?)+Úadditional_special_tokensr(   N)r   ÚcontentÚfake_image_tokenÚimage_tokenÚend_of_utterance_tokenÚglobal_image_tagr'   Úconvert_tokens_to_idsÚimage_token_idÚfake_image_token_idÚglobal_image_token_idÚrangeÚrow_col_idsÚreÚcompileÚ%_regex_to_remove_extra_special_tokensr   ÚsuperÚ__init__)
ÚselfÚimage_processorÚ	tokenizerr'   r(   ÚkwargsÚiÚjÚtokens_to_addÚ	__class__s
            €r#   rA   zIdefics3Processor.__init__7   sŒ  ø€ ô !+Ð+FÐSXÐbfÔ g× oÑ oˆÔÜ% i¸EÈ4ÔP×XÑXˆÔÜ&0Ð1EÐRWÐaeÔ&f×&nÑ&nˆÔ#Ø .ˆÔØ*ˆÔØ'×=Ñ=¸d×>NÑ>NÓOˆÔØ#,×#BÑ#BÀ4×CXÑCXÓ#YˆÔ Ø%.×%DÑ%DÀT×EZÑEZÓ%[ˆÔ"äSXÐYZÓS[÷
ØNOÔejÐklÓemò
Ø`aˆI×+Ñ+¨e°A¸±E°7¸%ÀÀAÁ¸wÀaÐ,HÕIð
ØIó
ˆÔô 68·Z±ZÐ@mÓ5nˆÔ2ð (Ø×%Ñ%Ø× Ñ Ø×+Ñ+ð*ð
ˆð 	×$Ñ$ ]Ô3Ø'×=Ñ=¸d×>NÑ>NÓOˆÔä‰Ñ˜¨)Ñ[À=Ð[ÐTZÓ[ùó%
s   Ã5FÚimagesÚtextr   rE   Úreturnc                 óä  —  | j                   d||dœ|¤Ž\  }} | j                  d||dœ|¤Ž  | j                  t        fd| j                  j
                  i|¤Ž}|�|n| j                  }|d   j                  dd«      }|d   j                  dd«      }|d   j                  dd«      }i x}	}
|�� | j                  |fi |d	   ¤Ž\  }	}|	j                  d
d«       |	j                  dd«       |�ñ| j                  ||¬«      \  }} | j                  |fi |d   ¤Ž}
|r||
d<   g }t        |«      D ]e  \  }}g }|D ]H  }|d   \  }}|
j                  ||«      }|
j                  ||dz
  «      }|j                  ||z
  dz   «       ŒJ |j                  |«       Œg |r| j                  |
d   |«      |
d<   | j                  ||
dg¬«       n|� | j                  dd|i|d   ¤Ž}
t        i |
¥|	¥|¬«      S )a  
        image_seq_len (`int`, *optional*):
            The length of the image sequence. If not provided, the default value of self.image_seq_len is used.
            image_seq_len should be equal to int(((image_size // patch_size) ** 2) / (scale_factor**2))
        )rJ   rK   Útokenizer_init_kwargsNr   Úreturn_text_replacement_offsetsFr   Úreturn_tensorsr   ÚrowsÚcols)Úimages_replacementsÚtext_replacement_offsetsÚnew_spanr.   Ú	input_idsÚmm_token_type_idsÚimage)Ú
modalitiesrK   )ÚdataÚtensor_typer!   )Úprepare_inputs_layoutÚvalidate_inputsÚ_merge_kwargsr   rD   Úinit_kwargsr'   ÚpopÚ_process_imagesÚget_text_with_replacementsÚ	enumerateÚchar_to_tokenÚappendÚcreate_mm_token_type_idsÚ_check_special_mm_tokensr   )rB   rJ   rK   r'   rE   Úoutput_kwargsrO   r   rP   Úimage_inputsÚtext_inputsrS   rT   Úbatch_image_seq_lengthsÚbatch_idÚtext_replacement_offsetÚimage_seq_lensrZ   ÚstartÚendÚstart_id_posÚ
end_id_poss                         r#   Ú__call__zIdefics3Processor.__call__\   s‡  € ð 2�t×1Ñ1ÐU¸ÀdÑUÈfÑU‰ˆ�Øˆ×ÑÐ@ F°Ñ@¸Ò@à*˜×*Ñ*Ü#ñ
à"&§.¡.×"<Ñ"<ð
ð ñ
ˆð *7Ð)B™È×HZÑHZˆØ*7¸Ñ*F×*JÑ*JÐKlÐnsÓ*tÐ'Ø#0°Ñ#?×#CÑ#CÐD^Ð`eÓ#fÐ Ø& }Ñ5×9Ñ9Ð:JÈDÓQˆà%'Ð'ˆ�{ØÑØ0D°×0DÑ0DÀVÑ0nÈ}Ð]lÑOmÑ0nÑ-ˆLÐ-ð ×Ñ˜V TÔ*Ø×Ñ˜V TÔ*àÐØ15×1PÑ1PØÐ.Að 2Qó 2Ñ.�Ð.ð -˜dŸn™n¨TÑR°]À=Ñ5QÑR�Ù2Ø>V�KÐ :Ñ;à*,Ð'Ü9BÐC[Ó9\ò CÑ5�HÐ5Ø%'�NØ 7ò M˜Ø%)¨*Ñ%5™
˜˜sØ'2×'@Ñ'@ÀÈ5Ó'Q˜Ø%0×%>Ñ%>¸xÈÈqÉÓ%Q˜
à&×-Ñ-¨j¸<Ñ.GÈ!Ñ.KÕLðMð ,×2Ñ2°>ÕBðCñ ,Ø7;×7TÑ7TØ# KÑ0Ð2Ió8�KÐ 3Ñ4ð ×-Ñ-¨d°KÈWÈIÐ-ÕVàÐØ(˜$Ÿ.™.ÑS¨dÐS°mÀMÑ6RÑSˆKäÐ!@ KÐ!@°<Ð!@ÈnÔ]Ð]r"   c                 óB  — |�#t        |t        «      r|g}|j                  «       }|�ë| j                  j	                  |«      }t        |«      r|gg}||fS t        |t        t        f«      r§t        |d   «      r™|�”|D �cg c]  }|j                  | j                  «      ‘Œ }}dgt        t        |«      «      z   }t        t        |«      «      D �cg c]  }|||   ||dz       ‘Œ }}t        |«      |d   kD  r|||d   d  gz   }||fS |}||fS |g}||fS c c}w c c}w )Nr   r.   éÿÿÿÿ)Ú
isinstanceÚstrÚcopyrC   Úfetch_imagesr	   ÚlistÚtupleÚcountr4   r   r;   Úlen)	rB   rJ   rK   rE   ÚsampleÚn_images_in_textÚcumsum_images_in_textrF   Úsplit_imagess	            r#   r\   z'Idefics3Processor.prepare_inputs_layout�   sk  € ð ÐÜ˜$¤Ô$Ø�v�Ø—9‘9“;ˆDàÐØ×)Ñ)×6Ñ6°vÓ>ˆFÜ˜fÔ%Ø!˜(˜�ð$ �tˆ|Ðô# ˜F¤T¬5 MÔ2´~ÀfÈQÁiÔ7PØÐ#àUYÖ'ZÈ6¨¯©°T×5EÑ5EÕ(FÐ'ZÐ$Ð'ZØ-.¨C´$´zÐBRÓ7SÓ2TÑ,TÐ)ô "'¤sÐ+;Ó'<Ó!=ö$àð Ð4°QÑ7Ð:OÐPQÐTUÑPUÑ:VÒWð$�Lð $ô
 ˜6“{Ð%:¸2Ñ%>Ò>Ø!-°Ð8MÈbÑ8QÐ8SÐ1TÐ0UÑ!U˜ð �tˆ|Ðð	 ".˜ð �tˆ|Ðð %˜X�Fà�tˆ|Ðùò ([ùò$s   Â "DÃDc                 ó®  •— t        ‰| �  ||fi |¤Ž |€|€t        d«      ‚|�¨|D �cg c]  }|j                  | j                  «      ‘Œ }}|�J|D �cg c]  }t        |«      ‘Œ }}||k7  r,t        d| j                  › d|› d| j                  › d|› d�	«      ‚y |€1t        |«      r%t        dt        |«      › d| j                  › d�«      ‚y y y c c}w c c}w )	Nz+You must provide either `text` or `images`.zThe total number of zP tokens in the prompts should be the same as the number of images passed. Found ú z tokens and z images per sample.zFound z. tokens in the text but no images were passed.)r@   r]   Ú
ValueErrorr|   r4   r}   ÚanyÚsum)	rB   rJ   rK   rE   r~   r   ÚsublistÚn_images_in_imagesrI   s	           €r#   r]   z!Idefics3Processor.validate_inputs¿   s  ø€ ô 	‰Ñ ¨Ñ7°Ò7àˆ<˜F˜NÜÐJÓKÐKàÐØMQÖRÀ6 §¡¨T×-=Ñ-=Õ >ÐRÐÐRØÐ!ØBHÖ%I°w¤c¨'¥lÐ%IÐ"Ð%IØ#Ð'9Ò9Ü$Ø.¨t×/?Ñ/?Ð.@ð A"Ø"2Ð!3°1°T×5EÑ5EÐ4FÀlÐSeÐRfÐfyð{óð ð :ð
 �¤CÐ(8Ô$9Ü ØœSÐ!1Ó2Ð3°1°T×5EÑ5EÐ4FÐFtÐuóð ð %:�ð ùÚRùâ%Is   ¨"CÁCri   Ú	image_idxc           	      óŒ  — |d   D ��cg c]  }|D ]  }|‘Œ Œ c}}|   }|d   D ��cg c]  }|D ]  }|‘Œ Œ c}}|   }|dk(  rI|dk(  rD| j                   › | j                  › z   | j                  › | j                  z  z   | j                   › z   S d}	t	        |«      D ]R  }
t	        |«      D ]=  }|	| j                   › d|
dz   › d|dz   › d�z   | j                  › | j                  z  z   z  }	Œ? |	d	z  }	ŒT |	d	| j                   › �| j                  › z   | j                  › | j                  z  z   | j                   › z   z  }	|	S c c}}w c c}}w )
NrQ   rR   r   Ú r-   r.   r/   r0   ú
)r3   r6   r4   r'   r;   )rB   ri   r‰   Úrow_listÚrowÚ
image_rowsÚcol_listÚcolÚ
image_colsÚtext_split_imagesÚn_hÚn_ws               r#   Úreplace_image_tokenz%Idefics3Processor.replace_image_tokenØ   s­  € Ø*6°vÑ*>×S˜hÈ(ÒSÀ3’cÐS�cÓSÐT]Ñ^ˆ
Ø*6°vÑ*>×S˜hÈ(ÒSÀ3’cÐS�cÓSÐT]Ñ^ˆ
Ø˜Š?˜z¨Qšà×(Ñ(Ð)Ø×*Ñ*Ð+ñ-à×%Ñ%Ð&¨$×*<Ñ*<Ñ<ñ=ð ×*Ñ*Ð+ñ-ðð !#ÐÜ˜ZÓ(ò *�Ü  Ó,ò �CØ%Ø×0Ñ0Ð1Ø! #¨¡' ¨%°°a±¨y¸Ð:ñ;à!×-Ñ-Ð.°$×2DÑ2DÑDñEñÑ%ðð " TÑ)Ñ!ð*ð Ø�T×*Ñ*Ð+Ð,Ø×*Ñ*Ð+ñ-à×%Ñ%Ð&¨$×*<Ñ*<Ñ<ñ=ð ×*Ñ*Ð+ñ-ñÐð %Ð$ùó5 TùÛSs
   ‰D:¨E rV   rk   c                 óŽ  — g }t        |«      D ]´  \  }}t        j                  ||   «      }t        j                  |«      }t        j                  || j
                  k(  «      d   }d}	|D ]7  }
|	t        |«      k\  r n'||	   }||
z   }d||| t        j                  ||«      }	Œ9 |j                  |j                  «       «       Œ¶ |S )Nr   r.   )
rc   ÚnpÚarrayÚ
zeros_likeÚwherer9   r}   Úsearchsortedre   Útolist)rB   rV   rk   rW   rF   Úseq_lengthsÚ	array_idsÚmm_token_typesÚimage_start_positionsrG   Úseq_lenro   rp   s                r#   rf   z*Idefics3Processor.create_mm_token_type_idsõ   sÛ   € ð ÐÜ'Ð(?Ó@ò 	>‰NˆAˆ{ÜŸ™ ¨1¡Ó.ˆIÜŸ]™]¨9Ó5ˆNÜ$&§H¡H¨Y¸$×:RÑ:RÑ-RÓ$SÐTUÑ$VÐ!ØˆAØ&ò @�ØœÐ1Ó2Ò2ÙØ-¨aÑ0�Ø˜g‘o�Ø,-�˜u SÐ)Ü—O‘OÐ$9¸3Ó?‘ð@ð ×$Ñ$ ^×%:Ñ%:Ó%<Õ=ð	>ð !Ð r"   c                 ó   — i }|�ût         j                  j                  di «      }|j                  |«       |D �cg c]   } | j                  j
                  g |¢|‘­Ž ‘Œ" }}| j                  dz   }| j                  dz   }t        | j                  dd¬«      d   «      dz
  }	g }
g }|D ]B  \  }}}||z  dz   }|d	kD  r|	nd	}|
j                  ||z   ||z  z   «       |j                  |«       ŒD |j                  |
|d
œ«       t        di |¤ŽS c c}w )a»  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.

        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.

        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        r   r   é   z

F)r   rV   r.   r   )Únum_image_tokensÚnum_image_patchesr!   )r   r    ÚgetÚupdaterC   Úget_number_of_image_patchesr'   r}   rD   re   r
   )rB   Úimage_sizesrE   Úvision_datar   Ú
image_sizeÚnum_image_row_colsÚbase_image_lengthÚ
col_lengthÚextra_split_newliner¥   r¦   Únum_patchesÚnum_rowsÚnum_colsÚ
row_lengthÚsplit_extras                    r#   Ú_get_num_multimodal_tokensz,Idefics3Processor._get_num_multimodal_tokens
  sR  € ð ˆØÐ"Ü3×=Ñ=×AÑAÀ/ÐSUÓVˆMØ× Ñ  Ô(ð #.ö"àð A�×$Ñ$×@Ñ@Ð\À*Ð\ÈmÔ\ð"Ðð "ð
 !%× 2Ñ 2°QÑ 6ÐØ×+Ñ+¨aÑ/ˆJô
 #& d§n¡n°VÐPU nÓ&VÐWbÑ&cÓ"dÐghÑ"hÐØ!ÐØ "Ðà3Eò 6Ñ/�˜X xØ'¨(Ñ2°QÑ6�
Ø5=À²\Ñ1Àq�Ø ×'Ñ'Ð(9¸KÑ(GÈ:ÐX`ÑK`Ñ(aÔbØ!×(Ñ(¨Õ5ð	6ð ×ÑÐ4DÐ[lÑmÔnäÑ, Ñ,Ð,ùò/"s   º%D)Né©   N)NNN)NN)N)r   r   r   r   Úvalid_processor_kwargsÚintrw   rA   r   r   rz   r   r   r   r   rs   r\   r   r]   Údictr–   rf   r¶   Ú__classcell__)rI   s   @r#   r&   r&   3   s¬  ø„ à4Ðð fjñ#\Ø>Að#\ØX[Ð^bÑXbõ#\ðJ ð JNØbfØ$(ñ	>^à˜T *Ñ-Ñ-°°T¸*Ñ5EÑ0FÑFð>^ð �IÐ2°D¸±OÀTÐJ]ÑE^Ð^Ñ_ð>^ð ˜T‘zð	>^ð
 Ð0Ñ1ð>^ð 
ò>^ó ð>^ðD %)Øbfñ à˜TÑ!ð ð �IÐ2°D¸±OÀTÐJ]ÑE^Ð^Ñ_ð ð Ð0Ñ1ó	 ðH %)Øbfñà˜TÑ!ðð �IÐ2°D¸±OÀTÐJ]ÑE^Ð^Ñ_ðð Ð)Ñ*õ	ð2%°ð %Àð %Èó %ð:!°$ð !ÐQUÐVYÑQZð !Ð_cÐdhÐilÑdmÑ_nó !÷*)-r"   r&   )!Ú__doc__r=   Ú	itertoolsr   Útypingr   r   Únumpyr˜   Úfeature_extraction_utilsr   Úimage_utilsr   r	   Úprocessing_utilsr
   r   r   r   Útokenization_utils_baser   r   r   Úutilsr   r   r   Ú
get_loggerr   Úloggerr   r&   Ú__all__r!   r"   r#   ú<module>rÈ      s   ðñó 
Ý  ß 'ã å 4ß 5ß XÓ Xß KÑ Kß ,ñ Ý<à	ˆ×	Ñ	˜HÓ	%€ôÐ.°eõ ð ô-˜ó -ó ð-ðD Ð
�r"   