
    ^j&K                     T   d dl Z d dlZd dlZd dlmZ ddlmZ ddl	m
Z
mZ ddlmZmZmZmZ ddlmZmZ ddlmZ dd	lmZmZmZ d
dlmZ d
dlmZmZ d
dlm Z  d
dl!m"Z"  G d de      Z# G d ded      Z$ G d de      Z% G d de       Z& G d ded      Z' G d de"      Z(g dZ)y)    N)
functional   )BatchFeature)group_images_by_shapereorder_images)ChannelDimensionPILImageResamplingSizeDictget_image_size)ImagesKwargsVideosKwargs)
TensorType)VideoMetadatagroup_videos_by_shapereorder_videos   )Glm46VConfig)Glm46VImageProcessorsmart_resize)Glm46VImageProcessorPil)Glm46VVideoProcessorc                       e Zd ZdZdZy)GlmgaConfiga  
    image_start_token_id (`int`, *optional*, defaults to 151339):
        The image start token index to encode the start of image.
    image_end_token_id (`int`, *optional*, defaults to 151340):
        The image end token index to encode the end of image.
    video_start_token_id (`int`, *optional*, defaults to 151361):
        The video start token index to encode the start of video.
    video_end_token_id (`int`, *optional*, defaults to 151362):
        The video end token index to encode the end of video.

    ```python
    >>> from transformers import AutoModelForImageTextToText, GlmgaConfig

    >>> # Initializing a Glmga style configuration
    >>> configuration = GlmgaConfig()

    >>> # Initializing a model (reusing the GLM-4.6V implementation) from that configuration
    >>> model = AutoModelForImageTextToText.from_config(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```glmgaN)__name__
__module____qualname____doc__
model_type     r/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/glmga/modular_glmga.pyr   r   )   s    . Jr!   r   c                   :    e Zd ZU dZeed<   eed<   eed<   eed<   y)GlmgaImageProcessorKwargsa  
    patch_size (`int`, *optional*, defaults to 14):
        The spatial patch size of the vision encoder.
    temporal_patch_size (`int`, *optional*, defaults to 2):
        The temporal patch size of the vision encoder.
    merge_size (`int`, *optional*, defaults to 2):
        The merge size of the vision encoder to llm encoder.
    patch_expand_factor (`int`, *optional*, defaults to 1):
        The patch_expand_factor of the vision encoder to llm encoder.
    
patch_sizetemporal_patch_size
merge_sizepatch_expand_factorN)r   r   r   r   int__annotations__r    r!   r"   r$   r$   D   s    	 OOr!   r$   F)totalc                        e Zd ZdZded   dededdded	ed
edeee   z  dz  deee   z  dz  dedededededz  de	e
z  dz  def dZy)GlmgaImageProcessor   imagesztorch.Tensor	do_resizesizeresample7PILImageResampling | tvF.InterpolationMode | int | None
do_rescalerescale_factordo_normalize
image_meanN	image_stdr%   r&   r'   r(   disable_groupingreturn_tensorsreturnc                    t        ||      \  }}i }|j                         D ]l  \  }}|j                  dd \  }}|rNt        |||||
|z  |z  |j                  |j
                        \  }}| j                  |t        ||      |      }|||<   n t        ||      }t        ||      \  }}i }i }|j                         D ]<  \  }}|j                  dd \  }}| j                  ||||||	      }|j                  dk(  r|j                  d      }|j                  d   |z  d	k7  rJ|ddd
df   j                  d||j                  d   |z  z
  ddd      }t        j                  ||gd      }|j                  dd \  }} }!| |z  }"||
z  ||
z  }$}#|j                  ||"||!|#|z  ||
|$|z  ||

      }|j!                  d	ddddddddd
      }|j#                  ||"|#z  |$z  |!|z  |
z  |
z        }%|%||<   |"|#|$gg|z  ||<   ? t        ||      }&t        ||      }t        j                  |&d	      }'t        j$                  |      }(t'        |'|(d|      S )z9
        Preprocess an image or batch of images.
        )r9   N
num_framesheightwidthtemporal_factorfactor
min_pixels
max_pixelsr@   rA   r1   r2      r.   r   dimr            r      	   pixel_valuesimage_grid_thwdatatensor_type)r   itemsshaper   shortest_edgelongest_edgeresizer
   r   rescale_and_normalizendim	unsqueezerepeattorchcatviewpermutereshapetensorr   ))selfr/   r0   r1   r2   r4   r5   r6   r7   r8   r%   r&   r'   r(   r9   r:   kwargsgrouped_imagesgrouped_images_indexresized_images_groupedrX   stacked_imagesr@   rA   resized_heightresized_widthresized_imagesprocessed_images_groupedprocessed_gridspatchesrepeats
batch_sizet_lenchannelgrid_tgrid_hgrid_wflatten_patchesprocessed_imagesrR   rS   s)                                            r"   _preprocesszGlmgaImageProcessor._preprocessY   s   . 0EV^n/o,,!#%3%9%9%; 	;!E>*005MFE0<2!$7%
25HH#11#001- "&"!}M% "- "
 -;"5)#	;& ((>@TU/D^fv/w,,#% %3%9%9%; )	M!E>,:,@,@,E)NM00
NL*V_G ||q !++A.}}Q"55:!!RS&/00*gmmA.>AT.TUWXZ[]^  ))Wg$6A>)0r):&Jw11F+z9=J;VFFll#*$*$G ooaAq!Q1aCG%oo&(--
:ZGO />$U+'-vv&>%?*%LOE"S)	MV **BDXY(:NOyy!1q9o6"..Q_m
 	
r!   )r   r   r   r(   listboolr
   floatr)   strr   r   r{   r    r!   r"   r-   r-   V   s    e
^$e
 e
 	e

 Le
 e
 e
 e
 DK'$.e
 4;&-e
 e
 !e
 e
 !e
 +e
  j(4/!e
$ 
%e
r!   r-   c                       e Zd ZdZdeej                     dededddede	d	ed
e	ee	   z  dz  de	ee	   z  dz  de
de
de
de
deez  dz  defdZy)GlmgaImageProcessorPilr.   r/   r0   r1   r2   zPILImageResampling | int | Noner4   r5   r6   r7   Nr8   r(   r%   r&   r'   r:   r;   c                    g }g }|D ]!  }|j                   dd \  }}|rNt        ||||||z  |
z  |j                  |j                        \  }}| j	                  |t        ||      |      }|r| j                  ||      }|r| j                  |||	      }t        j                  |t        j                        }|j                  dk(  rt        j                  |d	      }|j                  d
k(  rt        j                  |d	      }|j                   dd \  }}|j                   d   |z  dk7  rMt        j                  |ddddf   ||j                   d   |z  z
  d	      }t        j                  ||gd	      }|j                   dd \  }}}||z  }||z  ||z  }}|j                  ||||||z  ||||z  ||
      }t        j                   |d      }|j                  |||z  |z  ||z  |z  |z        } |j#                  | j%                  d             |j#                  |||g       $ t        j                  |d	      }!t        j&                  |      }"t)        |!|"d|      S )z?
        Preprocess images one by one for PIL backend.
        r=   Nr>   rF   rG   dtyper   r   )axisrH   r.   rI   )
r   r.   rH   rL   rM   rN   r   r   rO   rP   rQ   rT   )rX   r   rY   rZ   r[   r
   rescale	normalizenpasarrayfloat32r]   expand_dimsr_   concatenaterd   	transposeappendsqueezearrayr   )#rf   r/   r0   r1   r2   r4   r5   r6   r7   r8   r(   r%   r&   r'   r:   rg   rz   rp   imager@   rA   rl   rm   image_arrayrr   rs   rt   ru   rv   rw   rx   rq   ry   rR   rS   s#                                      r"   r{   z"GlmgaImageProcessorPil._preprocess   s   *  D	=E!KK,MFE0<2!$7%
25HH#11#001- !}M% $  UN;uj)D **U"**=K1$ nn[qA1$ nn[qA,7,=,=bc,B)NM  #&99Q>))23'';+<+<Q+?BU+UV
 !nnk7-C!L)4):):2A)>&Jw11F+z9=J;VFF!))#*$*$G ll7,JKG%oo&(--
:ZGO ##O$;$;A$>?""FFF#;<ID	=N ~~&6Q?/2"..Q_m
 	
r!   )r   r   r   r(   r|   r   ndarrayr}   r
   r~   r)   r   r   r   r{   r    r!   r"   r   r      s    d
RZZ d
 d
 	d

 4d
 d
 d
 d
 DK'$.d
 4;&-d
 !d
 d
 !d
 d
 j(4/d
" 
#d
r!   r   c                   T    e Zd ZU eeef   ed<   eed<   eed<   eed<   eed<   eed<   y)GlmgaVideoProcessorInitKwargsmax_image_sizer%   r&   r'   r(   
max_framesN)r   r   r   dictr   r)   r*   r    r!   r"   r   r   +  s+    cN"OOOr!   r   c                    (   e Zd ZdddZddiZdZdZdZ	 dd	ed
e	e
z  dz  fdZdddej                  ddddddddddfdeej                      dedededz  dddede
dede
ee
   z  dz  de
ee
   z  dz  de	dz  de	dz  de	dz  de	dz  deez  dz  fdZy) GlmgaVideoProcessori 1  i6)rY   rZ   rZ   r   r.   i  Nmetadatafpsc                    |t        |dd      t        d      |j                  }|dz
  }|j                  xs t	        ||j
                  z        dz   }||n| j
                  }t        ||z        }t        || j                        }d|j
                  z  }	t        |      D 
cg c]  }
|
|	z  	 }}
||k  r2t        |      D cg c]  }t        j                  ||z  |z         }}nBg }d}d|z  }t        |      D ]+  }||   |k\  s||z  }|j                  |       |||z
  k\  s+ n t        |      |k  rVt        |      dk(  rdt        |dz
  d      }}n
|d   |d   }}t        j                   |||t              j#                         }n<t        |      |kD  r.t        j                   d|dz
  |t              j#                         }t%               g }}|D ])  }||vs|j'                  |       |j                  |       + t        |      dz  r|j                  |d          t        j(                  |      S c c}
w c c}w )a  
        Args:
            metadata (`VideoMetadata`):
                Metadata of the video containing information about total duration, fps and total number of frames.
            fps (`int` or `float`, *optional*):
                Target frames to sample per second. Defaults to `self.fps`.
        Returns:
            np.ndarray:
                Indices to sample video frames.
        Nr   zAsked to sample frames per second but no video metadata was provided which is required when sampling in Glmga. Please pass in `VideoMetadata` object or set `do_sample_frames=False`r.   r   rI   r   )getattr
ValueErrortotal_num_framesdurationroundr   r)   minr   rangemathfloorr   lenmaxr   linspacetolistsetaddr   )rf   r   r   rg   total_framesmax_frame_idxr   
target_fps	extract_tduration_per_framei
timestamps_iframe_indicescurrent_secondinv_fpsframe_indexstartendseenuniqidxs                         r"   sample_framesz!GlmgaVideoProcessor.sample_frames;  sP     wx=EX 
  00$q($$Omhll.J(Ka(OOS
:-.		4??3	-6;L6IJa,,J
J)#QVW`Qab2TZZ\(9I(EFbMbMN*nG$\2 k*n<"g-N!((5%G);; }	)=!Q&L1$4a 8s*1-}R/@sKKsISIPPRM)+KK<!+;YcRYY[MUBd  	!C$C 	!
 t9q=KKR!xx~C K cs   %H="ITgp?videosdo_convert_rgbr0   r1   r2   r3   r4   r5   r6   r7   r8   r(   r%   r&   r'   r:   c                    t        |      \  }}i }|j                         D ]  \  }}|r| j                  |      }|j                  \  }}}}}|||}}}|rzt	        ||||||z  |z  |j
                  |j                        \  }}|j                  ||z  |||      }| j                  |t        ||      |      }|j                  |||||      }|||<    t        ||      } t        |       \  }}i }!i }"|j                         D ]  \  }}t        |d   t        j                        \  }}| j                  |||||	|
      }|}#|#j                  d   |z  dk7  r:|#d d dd f   j                  d|dz
  ddd      }$t!        j"                  |#|$gd      }#|#j                  d d	 \  }%}&}'|&|z  }&||z  ||z  })}(|#j                  |%|&||'|(|z  |||)|z  ||
      }#|#j%                  ddd
dddd	ddd
      }#|#j'                  |%|&|(z  |)z  |'|z  |z  |z        }*|*|!|<   |&|(|)gg|%z  |"|<     t        |!|      }+t        |"|      }"t!        j"                  |+d      },t!        j(                  |"      }-|,|-d}.t+        |.|      S )Nr>   rF   rG   r   )channel_dimr.   rI   rJ   r   rH   rL   rM   rN   r   rO   rP   )pixel_values_videosvideo_grid_thwrT   )r   rW   convert_to_rgbrX   r   rY   rZ   rb   r[   r
   r   r   r   FIRSTr\   r_   r`   ra   rc   rd   re   r   )/rf   r   r   r0   r1   r2   r4   r5   r6   r7   r8   r(   r%   r&   r'   r:   rg   grouped_videosgrouped_videos_indexresized_videos_groupedrX   stacked_videosBTCHWr?   r@   rA   rl   rm   resized_videosprocessed_videos_groupedrp   rq   rr   rs   rv   ru   rw   rx   ry   processed_videosr   r   rU   s/                                                  r"   r{   zGlmgaVideoProcessor._preprocess~  s7   & 0EV/L,,!#%3%9%9%; 	;!E>!%!4!4^!D*00MAq!Q()1aJ0<)!$7%
25HH#11#001- "0!4!4QUAq!!D!%"!}M% "- "
 "0!4!4Q1nm!\,:"5)-	;. ((>@TU 0E^/T,,#% %3%9%9%; %	M!E>,:>!;LZjZpZp,q)NM "77
NL*V_N %G }}Q"55:!!RS&/004G!4KQPQSTU))Wg$6A>*1--*;'J22F+z9=J;VFFll#*$*$G ooaAq!Q1aCG%oo&(--
:ZGO />$U+'-vv&>%?*%LOE"K%	MN **BDXY(:NO#ii(8a@o6#6,

 >BBr!   )N)r   r   r   r1   r   r   r(   r   r   r)   r~   r   r	   BICUBICr|   r`   Tensorr}   r
   r   r   r{   r    r!   r"   r   r   4  s   &8KLD$&9:N
CJ
 #'AA 5[4AL  $ $N`NhNh )!1504*.!%*.!%26!dCU\\"dC dC 	dC
 odC LdC dC dC dC DK'$.dC 4;&-dC !4ZdC $JdC !4ZdC $JdC  j(4/!dCr!   r   )r   r-   r   r   )*r   numpyr   r`   torchvision.transforms.v2r   tvFimage_processing_utilsr   image_transformsr   r   image_utilsr   r	   r
   r   processing_utilsr   r   utilsr   video_utilsr   r   r   glm46v.configuration_glm46vr   glm46v.image_processing_glm46vr   r   "glm46v.image_processing_pil_glm46vr   glm46v.video_processing_glm46vr   r   r$   r-   r   r   r   __all__r    r!   r"   <module>r      s       7 2 E  ;  O O 6 O H A, 6E $h
. h
Vg
4 g
TL nC. nCbr!   