Ë
    ÜÓ^jˆE  ã                   óü  — d Z ddlmZ ddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 dd	lmZmZmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlmZ ddlmZ  ej>                  e «      Z! ed¬«      e G d„ de«      «       «       Z" ed¬«      e G d„ de«      «       «       Z# G d„ dejH                  «      Z%e G d„ de«      «       Z& ed¬«       G d„ de&«      «       Z' ed ¬«       G d!„ d"e&e
«      «       Z(g d#¢Z)y)$zPyTorch Llava model.é    )Ú	dataclassN)Únné   )ÚACT2FN)ÚCache)ÚGenerationMixin)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚModelOutput)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingÚtorch_compilable_check)Úcan_return_tupleÚmerge_with_config_defaultsé   )Ú	AutoModelé   )ÚLlavaConfigzJ
    Base class for Llava outputs, with hidden states and attentions.
    ©Úcustom_introc                   ó:   — e Zd ZU dZdZej                  dz  ed<   y)ÚLlavaModelOutputWithPastaÏ  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    NÚimage_hidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__© ó    ús/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/llava/modeling_llava.pyr   r   $   s   … ñ	ð 59Ð˜×*Ñ*¨TÑ1Ô8r%   r   zQ
    Base class for Llava causal language model (or autoregressive) outputs.
    c                   óú   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   dZej                  dz  ed<   y)	ÚLlavaCausalLMOutputWithPasta4  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    NÚlossÚlogitsÚpast_key_valuesÚhidden_statesÚ
attentionsr   )r   r   r   r    r)   r!   r"   r#   r*   r+   r   r,   Útupler-   r   r$   r%   r&   r(   r(   9   s‹   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø$(€O�U˜T‘\Ó(Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ó6Ø48Ð˜×*Ñ*¨TÑ1Ô8r%   r(   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLlavaMultiModalProjectorÚconfigc                 óö  •— t         ‰| �  «        t        |j                  t        «      rdnt        |j                  «      }t        j                  |j                  j                  |z  |j                  j                  |j                  ¬«      | _        t        |j                     | _        t        j                  |j                  j                  |j                  j                  |j                  ¬«      | _        y )Nr   ©Úbias)ÚsuperÚ__init__Ú
isinstanceÚvision_feature_layerÚintÚlenr   ÚLinearÚvision_configÚhidden_sizeÚtext_configÚmultimodal_projector_biasÚlinear_1r   Úprojector_hidden_actÚactÚlinear_2)Úselfr1   Únum_feature_layersÚ	__class__s      €r&   r6   z!LlavaMultiModalProjector.__init__X   s¹   ø€ Ü‰ÑÔä",¨V×-HÑ-HÌ#Ô"N™QÔTWÐX^×XsÑXsÓTtÐÜŸ	™	Ø× Ñ ×,Ñ,Ð/AÑAØ×Ñ×*Ñ*Ø×1Ñ1ô
ˆŒô
 ˜&×5Ñ5Ñ6ˆŒÜŸ	™	Ø×Ñ×*Ñ*¨F×,>Ñ,>×,JÑ,JÐQW×QqÑQqô
ˆ�r%   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S ©N)r@   rB   rC   )rD   Úimage_featuresr,   s      r&   Úforwardz LlavaMultiModalProjector.forwardf   s2   € ØŸ™ nÓ5ˆØŸ™ Ó/ˆØŸ™ mÓ4ˆØÐr%   )r   r   r   r   r6   rJ   Ú__classcell__©rF   s   @r&   r0   r0   W   s   ø„ ð
˜{õ 
ör%   r0   c                   ó>   — e Zd ZU eed<   dZdZdZdgZdZ	dZ
dZdZdZy)ÚLlavaPreTrainedModelr1   Úmodel)ÚimageÚtextTr+   N)r   r   r   r   r#   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphÚ_supports_flex_attnÚ_supports_attention_backendr$   r%   r&   rN   rN   m   s@   … àÓØÐØ(ÐØ&*Ð#Ø#4Ð"5ÐàÐØ€Nà!ÐØÐØ"&Ñr%   rN   zu
    The Llava model which consists of a vision backbone and a language model, without a language modeling head.
    c                   ó@  ‡ — e Zd Zdefˆ fd„Zee ed¬«      	 	 	 ddej                  de
ee
   z  ee
   z  dz  dedz  d	edz  d
ee   deez  fd„«       «       «       Zdej&                  dej                  dej                  fd„Zee	 	 	 	 	 	 	 	 	 ddej&                  dz  dej                  dz  dej*                  dz  dej&                  dz  dedz  dej                  dz  de
ee
   z  ee
   z  dz  dedz  dej*                  dz  d
ee   deez  fd„«       «       Zˆ xZS )Ú
LlavaModelr1   c                 óô   •— t         ‰| �  |«       t        j                  |j                  «      | _        t        |«      | _        t        j                  |j                  «      | _	        | j                  «        y rH   )r5   r6   r   Úfrom_configr<   Úvision_towerr0   Úmulti_modal_projectorr>   Úlanguage_modelÚ	post_init©rD   r1   rF   s     €r&   r6   zLlavaModel.__init__ƒ   sY   ø€ Ü‰Ñ˜Ô Ü%×1Ñ1°&×2FÑ2FÓGˆÔä%=¸fÓ%EˆÔ"Ü'×3Ñ3°F×4FÑ4FÓGˆÔØ�‰Õr%   zWObtains image last hidden states from the vision tower and apply multimodal projection.r   NÚpixel_valuesr8   Úvision_feature_select_strategyÚoutput_hidden_statesÚkwargsÚreturnc                 ó  — |j                  «       D ��ci c]  \  }}|€Œ	||“Œ }}} | j                  |fdddœ|¤Ž}t        |t        «      r |j                  |   }	|dk(  r\|	d d …dd …f   }	nP|D �
cg c]  }
|j                  |
   ‘Œ }}
|dk(  r|D �cg c]  }|d d …dd …f   ‘Œ }}t        j                  |d¬«      }	| j                  |	«      }|j                  d«      �t        j                  |d   |j                  ¬«      | j                  j                  z  j                  d¬«      j                  «       }t        j                  |j                  d	«      |«      }nt!        |«      }||_        |S c c}}w c c}
w c c}w )
NT)rf   Úreturn_dictÚdefaultr   éÿÿÿÿ©ÚdimÚimage_sizes)Údevicer   )Úitemsr_   r7   r9   r,   r!   Úcatr`   ÚgetÚ	as_tensorrp   Ú
patch_sizeÚprodÚtolistÚsplitÚsqueezeÚlistÚpooler_output)rD   rd   r8   re   rf   rg   ÚkÚvÚimage_outputsÚselected_image_featureÚ	layer_idxÚhs_poolÚhsrI   Úsplit_sizess                  r&   Úget_image_featureszLlavaModel.get_image_features‹   sŠ  € ð $*§<¡<£>×C™4˜1˜a°Q±]�!�Q‘$ÐCˆÑCà)˜×)Ñ)Øð
à!%Øñ
ð ñ	
ˆô Ð*¬CÔ0Ø%2×%@Ñ%@ÐAUÑ%VÐ"Ø-°Ò:Ø)?ÂÀ1Á2ÀÑ)FÑ&àOcÖdÀ)�}×2Ñ2°9Ó=ÐdˆGÐdà-°Ò:Ø/6Ö7¨˜2ša ¡˜e›9Ð7�Ð7Ü%*§Y¡Y¨w¸BÔ%?Ð"à×3Ñ3Ð4JÓKˆð �:‰:�mÓ$Ð0ä—‘ ¨Ñ!6¸~×?TÑ?TÔUÐY]×YjÑYj×YuÑYuÑuß‘˜"�“ß‘“ð ô
 #Ÿ[™[¨×)?Ñ)?ÀÓ)BÀKÓP‰Nä! .Ó1ˆNØ&4ˆÔ#àÐùóK Dùò  eùò 8s   ”
E2ŸE2Á1E8ÂE=Ú	input_idsÚinputs_embedsrI   c                 ó.  — |€m| | j                  «       t        j                  | j                  j                  t        j
                  |j                  ¬«      «      k(  }|j                  d«      }n|| j                  j                  k(  }|j                  «       }|j                  d   |j                  d   z  }|j                  d«      j                  |j                  «      }t        ||j                  d   z  |j                  «       k(  d|› d|› �«       |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        )Údtyperp   rl   r   r   z6Image features and image tokens do not match, tokens: z, features: )Úget_input_embeddingsr!   Útensorr1   Úimage_token_idÚlongrp   ÚallÚsumÚshapeÚ	unsqueezeÚtor   Únumel)rD   r…   r†   rI   Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r&   Úget_placeholder_maskzLlavaModel.get_placeholder_mask¿   s  € ð ÐØ!.Ð2M°$×2KÑ2KÓ2MÜ—‘˜TŸ[™[×7Ñ7¼u¿z¹zÐR_×RfÑRfÔgó3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¨d¯k©k×.HÑ.HÑ!HÐà+×/Ñ/Ó1ˆØ)×/Ñ/°Ñ2°^×5IÑ5IÈ!Ñ5LÑLÐØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐÜØ˜]×0Ñ0°Ñ4Ñ4¸×8LÑ8LÓ8NÑNØDÀ^ÐDTÐT`ÐaqÐ`rÐsô	
ð "Ð!r%   Úattention_maskÚposition_idsr+   ro   c
                 ó  — |d u |d uz  rt        d«      ‚|€ | j                  «       |«      }|��| j                  ||||	d¬«      j                  }t	        j
                  |d¬«      j                  |j                  |j                  «      }| j                  |||¬«      }|j                  ||«      } | j                  d	||||dœ|
¤Ž}t        |j                  |j                  |j                  |j                   |�¬«      S d ¬«      S )
Nz:You must specify exactly one of input_ids or inputs_embedsT)rd   r8   re   ro   rj   r   rm   )r†   rI   )r—   r˜   r+   r†   )Úlast_hidden_stater+   r,   r-   r   r$   )Ú
ValueErrorr‰   r„   r{   r!   rr   r‘   rp   rˆ   r–   Úmasked_scatterra   r   rš   r+   r,   r-   )rD   r…   rd   r—   r˜   r+   r†   r8   re   ro   rg   rI   r“   Úoutputss                 r&   rJ   zLlavaModel.forward×   sH  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø7˜D×5Ñ5Ó7¸	ÓBˆMàÐ#Ø!×4Ñ4Ø)Ø%9Ø/MØ'Ø ð 5ó ÷ ‰mð ô #ŸY™Y ~¸1Ô=×@Ñ@À×AUÑAUÐWd×WjÑWjÓkˆNØ!%×!:Ñ!:Ø¨À~ð ";ó "Ðð *×8Ñ8Ð9KÈ^Ó\ˆMà%�$×%Ñ%ð 
Ø)Ø%Ø+Ø'ñ	
ð
 ñ
ˆô (Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø2>Ð2J ô
ð 	
ð
 QUô
ð 	
r%   )NNN)	NNNNNNNNN)r   r   r   r   r6   r   r   r   r!   r"   r9   rz   ÚstrÚboolr   r   r.   r
   r„   Ú
LongTensorr–   ÚTensorr   r   rJ   rK   rL   s   @r&   r\   r\   }   sï  ø„ ð˜{õ ð  ØÙØnôð DHØ59Ø,0ñ-à×'Ñ'ð-ð " D¨¡I™o°°S±	Ñ9¸DÑ@ð-ð ),¨d©
ð	-ð
 # T™kð-ð Ð+Ñ,ð-ð 
Ð+Ñ	+ò-óó ó  ð
-ð^"Ø×)Ñ)ð"Ø:?×:KÑ:Kð"Ø]b×]nÑ]nó"ð0 Øð .2Ø15Ø.2Ø04Ø(,Ø26ØCGØ59Ø+/ñ/
à×#Ñ# dÑ*ð/
ð ×'Ñ'¨$Ñ.ð/
ð Ÿ™ tÑ+ð	/
ð
 ×&Ñ&¨Ñ-ð/
ð  ™ð/
ð ×(Ñ(¨4Ñ/ð/
ð " D¨¡I™o°°S±	Ñ9¸DÑ@ð/
ð ),¨d©
ð/
ð —\‘\ DÑ(ð/
ð Ð+Ñ,ð/
ð 
Ð)Ñ	)ò/
ó ó ô/
r%   r\   zS
    The LLAVA model which consists of a vision backbone and a language model.
    c                   óB  ‡ — e Zd ZddiZdefˆ fd„Zdej                  fd„Ze		 	 dde
j                  d	eee   z  ee   z  dz  d
edz  dee   deez  f
d„«       Zee		 	 	 	 	 	 	 	 	 	 	 dde
j*                  dz  de
j                  dz  de
j,                  dz  de
j*                  dz  dedz  de
j                  dz  d	eee   z  ee   z  dz  d
edz  de
j*                  dz  dee
j,                  z  de
j,                  dz  dee   deez  fd„«       «       Z	 	 	 	 	 	 dˆ fd„	Zˆ xZS )ÚLlavaForConditionalGenerationzlm_head.weightz(model.language_model.embed_tokens.weightr1   c                 óî   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  j                  |j                  j                  d¬«      | _	        | j                  «        y )NFr3   )r5   r6   r\   rO   r   r;   r>   r=   Ú
vocab_sizeÚlm_headrb   rc   s     €r&   r6   z&LlavaForConditionalGeneration.__init__  sS   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ü—y‘y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÔkˆŒØ�‰Õr%   rh   c                 ó   — | j                   S rH   )r¦   )rD   s    r&   Úget_output_embeddingsz3LlavaForConditionalGeneration.get_output_embeddings  s   € Ø�|‰|Ðr%   Nrd   r8   re   rg   c                 óB   —  | j                   j                  d|||dœ|¤ŽS )N)rd   r8   re   r$   )rO   r„   )rD   rd   r8   re   rg   s        r&   r„   z0LlavaForConditionalGeneration.get_image_features  s5   € ð -ˆt�z‰z×,Ñ,ð 
Ø%Ø!5Ø+Iñ
ð ñ	
ð 	
r%   r…   r—   r˜   r+   r†   ÚlabelsÚlogits_to_keepro   c                 óš  —  | j                   d|||||||||dœ	|¤Ž}|d   }t        |
t        «      rt        |
 d«      n|
}| j	                  |dd…|dd…f   «      }d}|	�4 | j
                  d||	| j                  j                  j                  dœ|¤Ž}t        |||j                  |j                  |j                  |j                  ¬«      S )añ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, LlavaForConditionalGeneration

        >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf")
        >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")

        >>> prompt = "USER: <image>\nWhat's the content of the image? ASSISTANT:"
        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs, max_new_tokens=15)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "USER:  \nWhat's the content of the image? ASSISTANT: The image features a busy city street with a stop sign prominently displayed"
        ```)	r…   rd   r—   r˜   r+   r†   r8   re   ro   r   N)r*   rª   r¥   )r)   r*   r+   r,   r-   r   r$   )rO   r7   r9   Úslicer¦   Úloss_functionr1   r>   r¥   r(   r+   r,   r-   r   )rD   r…   rd   r—   r˜   r+   r†   r8   re   rª   r«   ro   rg   r�   r,   Úslice_indicesr*   r)   s                     r&   rJ   z%LlavaForConditionalGeneration.forward+  sù   € ð\ �$—*‘*ð 
ØØ%Ø)Ø%Ø+Ø'Ø!5Ø+IØ#ñ
ð ñ
ˆð   ™
ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%ð Ø f¸¿¹×9PÑ9P×9[Ñ9[ñØ_eñˆDô +ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ô
ð 	
r%   c           	      óf   •— t        ‰
| �  |f|||||dœ|¤Ž}	|s|j                  dd«      s||	d<   |	S )N)r+   r†   r—   r«   Úis_first_iterationÚ	use_cacheTrd   )r5   Úprepare_inputs_for_generationrs   )rD   r…   r+   r†   rd   r—   r«   r±   rg   Úmodel_inputsrF   s             €r&   r³   z;LlavaForConditionalGeneration.prepare_inputs_for_generationz  sY   ø€ ô ‘wÑ<Øð
à+Ø'Ø)Ø)Ø1ñ
ð ñ
ˆñ  V§Z¡Z°¸TÔ%Bð
 ,8ˆL˜Ñ(àÐr%   )NN)NNNNNNNNNr   N)NNNNNF)r   r   r   Ú_tied_weights_keysr   r6   r   ÚModuler¨   r   r!   r"   r9   rz   rž   r   r   r.   r
   r„   r   r    r¡   r   r(   rJ   r³   rK   rL   s   @r&   r£   r£     s  ø„ ð +Ð,VÐWÐð˜{õ ð r§y¡yó ð ð DHØ59ñ	
à×'Ñ'ð
ð " D¨¡I™o°°S±	Ñ9¸DÑ@ð
ð ),¨d©
ð	
ð
 Ð+Ñ,ð
ð 
Ð+Ñ	+ò
ó ð
ð Øð .2Ø15Ø.2Ø04Ø(,Ø26ØCGØ59Ø*.Ø-.Ø+/ñK
à×#Ñ# dÑ*ðK
ð ×'Ñ'¨$Ñ.ðK
ð Ÿ™ tÑ+ð	K
ð
 ×&Ñ&¨Ñ-ðK
ð  ™ðK
ð ×(Ñ(¨4Ñ/ðK
ð " D¨¡I™o°°S±	Ñ9¸DÑ@ðK
ð ),¨d©
ðK
ð × Ñ  4Ñ'ðK
ð ˜eŸl™lÑ*ðK
ð —\‘\ DÑ(ðK
ð Ð+Ñ,ðK
ð 
Ð,Ñ	,òK
ó ó ðK
ð` ØØØØØ ÷ñ r%   r£   )r£   rN   r\   )*r    Údataclassesr   r!   r   Úactivationsr   Úcache_utilsr   Ú
generationr   Úmodeling_outputsr	   r
   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úautor   Úconfiguration_llavar   Ú
get_loggerr   Úloggerr   r(   r¶   r0   rN   r\   r£   Ú__all__r$   r%   r&   ú<module>rÅ      s3  ðñ å !ã Ý å !Ý  Ý )ß `Ñ `Ý -Ý &ß XÓ Xß IÝ Ý ,ð 
ˆ×	Ñ	˜HÓ	%€ñ ðôð
 ô9Ð6ó 9ó óð9ñ ðôð
 ô9 +ó 9ó óð9ô0˜rŸy™yô ð, ô'˜?ó 'ó ð'ñ ðôô
F
Ð%ó F
óð
F
ñR ðôô
HÐ$8¸/ó Hóð
HòV R�r%   