
    ^jP%                         d dl ZddlmZmZmZ ddlmZmZ  ej                  e
      Z G d ded      Ze G d	 d
e             Zd
gZy)    N   )MultiModalDataProcessingKwargsProcessorMixin)auto_docstringloggingc                   "    e Zd ZddddddidZy)Glm46VProcessorKwargsFT)paddingreturn_token_type_idsreturn_mm_token_type_idsreturn_metadata)text_kwargsvideos_kwargsN)__name__
__module____qualname__	_defaults     w/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/glm46v/processing_glm46v.pyr
   r
      s#     %*(,

 ,T2Ir   r
   F)totalc                        e Zd ZeZd fd	ZdededefdZ	dededefdZ
dd	Z	 dd
Ze fd       Zdedeee      fdZddefdZ xZS )Glm46VProcessorc                    t        |d      sdn|j                  | _        t        |d      sdn|j                  | _        t        |dd       r|j                  n|j                  | j                        | _        t        |dd       r|j                  n|j                  | j                        | _        t        | !  ||||       |j                  d      | _	        |j                  d	      | _
        y )
Nimage_tokenz	<|image|>video_tokenz	<|video|>image_token_idvideo_token_id)chat_templatez<|begin_of_video|>z<|end_of_video|>)hasattrr   r   getattrr   convert_tokens_to_idsr   super__init__video_start_idvideo_end_id)selfimage_processor	tokenizervideo_processorr    kwargs	__class__s         r   r%   zGlm46VProcessor.__init__.   s    .5i.O;U^UjUj.5i.O;U^UjUj y"2D9 $$001A1AB 	 y"2D9 $$001A1AB 	
 	)_Tab'==>RS%;;<NOr   image_inputs	image_idxreturnc                     | j                   j                  dz  }|d   |   j                         |z  }| j                  |z  S )N   image_grid_thw)r)   
merge_sizeprodr   )r(   r.   r/   merge_lengthnum_image_tokenss        r   replace_image_tokenz#Glm46VProcessor.replace_image_token?   sI    ++669'(89)DIIK|["222r   video_inputs	video_idxc                 l   | j                   j                  dz  }|d   |   d   }|d   |   j                         |z  |z  }|d   |   }d}|j                  t        j                  d       |j                  dn|j                  |_        |j                  d d d   }g }	t        dt        |            D ]  }
|	j                  ||
           |	d | }t        |      |k  r'|j                  |r|d   nd       t        |      |k  r't        |      D ]  }||   }| j                  ||	      }||z  }! |S )
Nr2   video_grid_thwr   video_metadata a  GLM46V requires frame timestamps to construct prompts, but the `fps` of the input video could not be inferred. Probably `video_metadata` was missing from inputs and you passed pre-sampled frames. Defaulting to `fps=24`. Please provide `video_metadata` for more accurate results.   )r7   )r+   r4   r5   fpsloggerwarning_once
timestampsrangelenappendreplace_frame_token_id)r(   r9   r:   r6   
num_framesr7   metadatavideo_structurerD   unique_timestampsidxselected_timestamps	frame_idxtimestamp_secframe_structures                  r   replace_video_tokenz#Glm46VProcessor.replace_video_tokenD   sr   ++669!"23I>qA
'(89)DIIK|[_ii 01)<<<e
 &\\1rx||((1-
C
O, 	6C$$Z_5	6 0<%&3&&BU':2'>[\] %&3 z* 	/I/	:M"99-Zj9kO.O	/
 r   c                    i }|t         j                  j                  di       }|j                  |       |j                  dd      xs | j                  j
                  }|D cg c]   } | j                  j                  g || " }}|D 	cg c]
  }	|	|dz  z   }
}	|j                  |
|d       |vt         j                  j                  di       }|j                  |       |D cg c]   } | j                  j                  g || " }}|D 	cg c]
  }	|	dz  z   }}	||d<   t        di |S c c}w c c}	w c c}w c c}	w )	aK  
        Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.
        Args:
            image_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (height, width) per each image.
            video_sizes (`list[list[int]]`, *optional*):
                The input sizes formatted as (num_frames, height, width) per each video.
        Returns:
            `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided
            input modalities, along with other useful data.
        Nimages_kwargsr4   r2   )r7   num_image_patchesr   num_video_tokensr   )
r
   r   getupdater)   r4   get_number_of_image_patchesr+   get_number_of_video_patchesr   )r(   image_sizesvideo_sizesr,   vision_datarT   r4   
image_sizerU   num_patchesr7   r   
video_sizenum_video_patchesrV   s                  r   _get_num_multimodal_tokensz*Glm46VProcessor._get_num_multimodal_tokensc   s    "1;;??QSTM  (&**<>a$BVBVBaBaJ #.! A$$@@\*\m\! ! Sdd;
A!=dd4D[lmn"1;;??QSTM  ( #.! A$$@@\*\m\! ! Sdd;
A!=dd.>K*+,,,#!  e!  es   $%D7D<+%EEc                 B     | j                   j                  |f||d|S )a  
        Post-process the output of the model to decode the text.

        Args:
            generated_outputs (`torch.Tensor` or `np.ndarray`):
                The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
                or `(sequence_length,)`.
            skip_special_tokens (`bool`, *optional*, defaults to `True`):
                Whether or not to remove special tokens in the output. Argument passed to the tokenizer's `batch_decode` method.
            clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
                Whether or not to clean up the tokenization spaces. Argument passed to the tokenizer's `batch_decode` method.
            **kwargs:
                Additional arguments to be passed to the tokenizer's `batch_decode method`.

        Returns:
            `list[str]`: The decoded text.
        )skip_special_tokensclean_up_tokenization_spaces)r*   batch_decode)r(   generated_outputsrd   re   r,   s        r   post_process_image_text_to_textz/Glm46VProcessor.post_process_image_text_to_text   s5    ( +t~~**
 3)E
 	
 	
r   c                      t         |   dgz   S )Nmm_token_type_ids)r$   model_input_names)r(   r-   s    r   rk   z!Glm46VProcessor.model_input_names   s    w(,?+@@@r   	input_idsc                    g }|D ]  }t        j                  |      }t        j                  |      }t        j                  || j                  k(  d      }t        j                  || j
                  k(  d      }||kD  }d||| j                  k(  |z  <   d||| j                  k(  | z  <   |j                  |j                                 |S )Nr   )axisr2      )	nparray
zeros_likecumsumr&   r'   r   rG   tolist)	r(   rl   rj   input	array_idsmm_token_typesstartsendsis_video_modalitys	            r   create_mm_token_type_idsz(Glm46VProcessor.create_mm_token_type_ids   s      	>EI]]51N
 YYyD,?,??aHF99Y$*;*;;!DD &UVNI)<)<<@QQRXYNI)<)<<BSASTU$$^%:%:%<=	> ! r   r7   c                 0    d| j                   |z   d|ddS )Nz<|begin_of_image|>z<|end_of_image|>z.1fz seconds)r   )r(   rP   r7   s      r   rH   z&Glm46VProcessor.replace_frame_token_id   s+    #D$4$47G$G#HHXYfgjXkksttr   )NNNN)NN)TF)ro   )r   r   r   r
   valid_processor_kwargsr%   dictintstrr8   rR   rb   rh   propertyrk   listr{   rH   __classcell__)r-   s   @r   r   r   *   s    2P"3 3 3 3
   >$-N Y^
6 A A!$ !4S	? !*uc ur   r   )numpyrp   processing_utilsr   r   r   utilsr   r   
get_loggerr   rB   r
   r   __all__r   r   r   <module>r      sb   ,  P P , 
		H	%,E  Sun Su Sul 
r   