Ë
    ÜÓ^jw  ã                   óP  — d dl Z d dlmZ d dlmZ d dlZd dlmZ ddlmZ	 ddl
mZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZ ddlmZmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z*m+Z+ e e G d„ de«      «       «       Z, e d¬«      e G d„ de«      «       «       Z- e d¬«      e G d„ de«      «       «       Z. ed«       G d „ d!ej^                  «      «       Z0 G d"„ d#ej^                  «      Z1 G d$„ d%ej^                  «      Z2	 dGd&ej^                  d'ejf                  d(ejf                  d)ejf                  d*ejf                  dz  d+e4d,e4fd-„Z5 G d.„ d/ej^                  «      Z6 G d0„ d1ej^                  «      Z7 G d2„ d3e«      Z8 G d4„ d5ej^                  «      Z9 G d6„ d7ej^                  «      Z: G d8„ d9ejv                  «      Z< G d:„ d;e«      Z=d<ejf                  d=e>fd>„Z? G d?„ d@e=«      Z@ e dA¬«       G dB„ dCe=«      «       ZAe  G dD„ dEe=e«      «       ZBg dF¢ZCy)Hé    N)ÚCallable)Ú	dataclass)Únné   )Úinitialization)ÚACT2FN)ÚCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Úmerge_with_config_defaults)Úcapture_outputsé   )Ú	AutoModelé   )ÚOvis2ConfigÚOvis2VisionConfigc                   ó:   — e Zd ZU dZdZej                  dz  ed<   y)Ú*BaseModelOutputWithVisualIndicatorFeatureszâ
    visual_indicator_features (`torch.FloatTensor` of shape `(batch_size, visual_indicator_size)`):
        Visual indicator features extracted from the model, which can be used for auxiliary tasks or further processing.
    NÚvisual_indicator_features)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r!   ÚtorchÚFloatTensorÚ__annotations__© ó    ús/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/ovis2/modeling_ovis2.pyr    r    ,   s   … ñð
 ;?Ð˜u×0Ñ0°4Ñ7Ô>r*   r    zJ
    Base class for Llava outputs, with hidden states and attentions.
    ©Úcustom_introc                   ó:   — e Zd ZU dZdZej                  dz  ed<   y)ÚOvis2ModelOutputWithPastaÏ  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    NÚimage_hidden_states)r"   r#   r$   r%   r0   r&   r'   r(   r)   r*   r+   r/   r/   7   s   … ñ	ð 59Ð˜×*Ñ*¨TÑ1Ô8r*   r/   zQ
    Base class for Ovis2 causal language model (or autoregressive) outputs.
    c                   óú   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   dZej                  dz  ed<   y)	ÚOvis2CausalLMOutputWithPastaA  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size (batch_size * num_patches, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    NÚlossÚlogitsÚpast_key_valuesÚhidden_statesÚ
attentionsr0   )r"   r#   r$   r%   r3   r&   r'   r(   r4   r5   r	   r6   Útupler7   r0   r)   r*   r+   r2   r2   L   s‹   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø$(€O�U˜T‘\Ó(Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ó6Ø48Ð˜×*Ñ*¨TÑ1Ô8r*   r2   ÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚOvis2RMSNormÚepsÚreturnNc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z;
        Ovis2RMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	Parameterr&   ÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer<   Ú	__class__s      €r+   r@   zOvis2RMSNorm.__init__l   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr*   r6   c                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Nr   éÿÿÿÿT©Úkeepdim)	ÚdtypeÚtor&   Úfloat32ÚpowÚmeanÚrsqrtrD   rC   )rE   r6   Úinput_dtypeÚvariances       r+   ÚforwardzOvis2RMSNorm.forwardt   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r*   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)r8   rC   ÚshaperD   ©rE   s    r+   Ú
extra_reprzOvis2RMSNorm.extra_repr{   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr*   )g�íµ ÷Æ°>)
r"   r#   r$   Úfloatr@   r&   ÚTensorrT   rX   Ú__classcell__©rG   s   @r+   r;   r;   j   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJr*   r;   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚOvis2VisionMLPc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  | j                  | j                  |j                  ¬«      | _	        t        j                  | j                  | j                  |j                  ¬«      | _
        t        |j                     | _        y ©N©Úbias©r?   r@   ÚconfigrF   Úintermediate_sizer   ÚLinearÚmlp_biasÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©rE   rd   rG   s     €r+   r@   zOvis2VisionMLP.__init__€   ó¸   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Ñ/ˆ�r*   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S ©N©rj   rl   rh   ri   ©rE   Úxrj   s      r+   rT   zOvis2VisionMLP.forwardŠ   ó6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr*   ©r"   r#   r$   r@   rT   r[   r\   s   @r+   r^   r^      ó   ø„ ô0ör*   r^   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚOvis2VisionEmbeddingsrd   c                 ó°  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        t        j                  |j                  | j                  | j                  | j                  d¬«      | _
        | j
                  | j                  z  dz  | _        | j                  | _        t        j                  | j                  | j                  «      | _        | j                  dt!        j"                  | j                  «      j%                  d«      d¬«       t'        |j                  |j(                  «      | _        y )NÚvalid)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr   Úposition_ids©r   rI   F)Ú
persistent)r?   r@   rd   rF   Ú	embed_dimÚ
image_sizeÚ
patch_sizer   ÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsÚ	EmbeddingÚposition_embeddingÚregister_bufferr&   ÚarangeÚexpandr;   Úrms_norm_epsÚrms_normrm   s     €r+   r@   zOvis2VisionEmbeddings.__init__�   s   ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿy™yØ×+Ñ+ØŸ™ØŸ™Ø—?‘?Øô 
ˆÔð !ŸO™O¨t¯©Ñ>À1ÑDˆÔØ!×-Ñ-ˆÔÜ"$§,¡,¨t×/AÑ/AÀ4Ç>Á>Ó"RˆÔØ×Ñ˜^¬U¯\©\¸$×:LÑ:LÓ-M×-TÑ-TÐU\Ó-]ÐjoÐÔpÜ$ V×%7Ñ%7¸×9LÑ9LÓMˆ�r*   Úpixel_valuesr=   c                 ó(  — | j                   j                  j                  }| j                  |j                  |¬«      «      }|j	                  d«      j                  dd«      }| j                  |«      }|| j                  | j                  «      z   }|S )N©rL   r   r   )	rˆ   rC   rL   rM   ÚflattenÚ	transposer‘   rŒ   r€   )rE   r’   Útarget_dtypeÚpatch_embedsÚ
embeddingss        r+   rT   zOvis2VisionEmbeddings.forward¥   s€   € Ø×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆØ!×)Ñ)¨!Ó,×6Ñ6°q¸!Ó<ˆ
Ø—]‘] :Ó.ˆ
à $×"9Ñ"9¸$×:KÑ:KÓ"LÑLˆ
àÐr*   )
r"   r#   r$   r   r@   r&   r'   rZ   rT   r[   r\   s   @r+   rx   rx   �   s/   ø„ ðNÐ0õ Nð* E×$5Ñ$5ð ¸%¿,¹,÷ r*   rx   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 óÀ  — t        j                  ||j                  dd«      «      |z  }|�||z   }t        j                  j                  |dt         j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )NrI   éþÿÿÿ)ÚdimrL   )ÚpÚtrainingr   r   )r&   Úmatmulr–   r   Ú
functionalÚsoftmaxrN   rM   rL   r    r¥   Ú
contiguous)
rš   r›   rœ   r�   rž   rŸ   r    ÚkwargsÚattn_weightsÚattn_outputs
             r+   Úeager_attention_forwardr­   °   sº   € ô —<‘<  s§}¡}°R¸Ó'<Ó=ÀÑG€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r*   c            
       ó    ‡ — e Zd ZdZˆ fd„Z	 ddej                  dej                  dz  deej                  ej                  dz  f   fd„Zˆ xZ	S )	ÚOvis2VisionAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                 óx  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚| j                  dz  | _	        |j                  | _        d| _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  | j                  | j                  |j                  ¬«      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      à¿Fra   )r?   r@   rd   rF   rƒ   Únum_attention_headsÚ	num_headsÚhead_dimÚ
ValueErrorÚscaleÚattention_dropoutr    Ú	is_causalr   rf   Úqkv_biasÚk_projÚv_projÚq_projÚout_projrm   s     €r+   r@   zOvis2VisionAttention.__init__Ê   s2  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒØˆŒÜ—i‘i §¡°·±ÀVÇ_Á_ÔUˆŒÜ—i‘i §¡°·±ÀVÇ_Á_ÔUˆŒÜ—i‘i §¡°·±ÀVÇ_Á_ÔUˆŒÜŸ	™	 $§.¡.°$·.±.ÀvÇÁÔWˆ�r*   Nr6   rž   r=   c           
      ó®  — |j                   dd }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }t        j                  | j                  j                  t        «      }	 |	| ||||| j                  | j                  | j                  sdn| j                  ¬«      \  }
} |
j                   g |¢d‘­Ž j#                  «       }
| j%                  |
«      }
|
|fS )z#Input shape: Batch x Time x ChannelNrI   r   r   ç        )r·   rŸ   r    )rV   r³   r»   Úviewr–   r¹   rº   r   Úget_interfacerd   Ú_attn_implementationr­   r·   rµ   r¥   r    Úreshaper©   r¼   )rE   r6   rž   rª   Úinput_shapeÚhidden_shapeÚqueriesÚkeysÚvaluesÚattention_interfacer¬   r«   s               r+   rT   zOvis2VisionAttention.forwardÝ   s<  € ð $×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØ—+‘+˜mÓ,×1Ñ1°,Ó?×IÑIÈ!ÈQÓOˆØ�{‰{˜=Ó)×.Ñ.¨|Ó<×FÑFÀqÈ!ÓLˆØ—‘˜]Ó+×0Ñ0°Ó>×HÑHÈÈAÓNˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØØ—n‘nØ—J‘JØ#Ÿ}š}‘C°$·,±,ô	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m KÓ0ˆà˜LÐ(Ð(r*   rp   )
r"   r#   r$   r%   r@   r&   rZ   r8   rT   r[   r\   s   @r+   r¯   r¯   Ç   sV   ø„ ÙGôXð, /3ñ!)à—|‘|ð!)ð Ÿ™ tÑ+ð!)ð
 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷!)r*   r¯   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚOvis2MLPc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  | j                  | j                  |j                  ¬«      | _	        t        j                  | j                  | j                  |j                  ¬«      | _
        t        |j                     | _        y r`   rc   rm   s     €r+   r@   zOvis2MLP.__init__  rn   r*   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S rp   rq   rr   s      r+   rT   zOvis2MLP.forward  rt   r*   ru   r\   s   @r+   rÊ   rÊ     rv   r*   rÊ   c            	       óˆ   ‡ — e Zd Zdefˆ fd„Z	 d	dej                  dej                  dz  dee   dej                  fd„Z	ˆ xZ
S )
ÚOvis2VisionEncoderLayerrd   c                 óö   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        t        |j                  |j                  «      | _	        t        |j                  |j                  «      | _
        y rp   )r?   r@   r¯   Ú	attentionrÊ   Úffnr;   rF   r�   Ú	rms_norm1Ú	rms_norm2rm   s     €r+   r@   z Ovis2VisionEncoderLayer.__init__  sZ   ø€ Ü‰ÑÔÜ-¨fÓ5ˆŒÜ˜FÓ#ˆŒÜ% f×&8Ñ&8¸&×:MÑ:MÓNˆŒÜ% f×&8Ñ&8¸&×:MÑ:MÓNˆ�r*   Nr6   rž   rª   r=   c                 ó°   — | j                  |«      } | j                  d||dœ|¤Ž\  }}||z   }| j                  |«      }| j                  |«      }||z   }|S )N)r6   rž   r)   )rÒ   rÐ   rÓ   rÑ   )rE   r6   rž   rª   Únorm_hidden_statesr¬   Ú_Ú
mlp_outputs           r+   rT   zOvis2VisionEncoderLayer.forward  sl   € ð "Ÿ^™^¨MÓ:ÐØ'˜Ÿ™ÐrÐ6HÐYgÑrÐkqÑr‰ˆ�Qà%¨Ñ3ˆØ!Ÿ^™^¨MÓ:ÐØ—X‘XÐ0Ó1ˆ
à%¨
Ñ2ˆØÐr*   rp   )r"   r#   r$   r   r@   r&   rZ   r   r   rT   r[   r\   s   @r+   rÎ   rÎ     sY   ø„ ðOÐ0õ Oð /3ñà—|‘|ðð Ÿ™ tÑ+ðð Ð+Ñ,ð	ð
 
�‰÷r*   rÎ   c            	       ót   ‡ — e Zd ZdZdefˆ fd„Zee	 d	dej                  dz  de
e   defd„«       «       Zˆ xZS )
ÚOvis2VisionEncoderz»
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`Ovis2VisionEncoderLayer`].

    Args:
        config: Ovis2VisionConfig
    rd   c                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w ©NF)
r?   r@   rd   r   Ú
ModuleListÚrangeÚnum_hidden_layersrÎ   ÚlayersÚgradient_checkpointing)rE   rd   rÖ   rG   s      €r+   r@   zOvis2VisionEncoder.__init__3  sP   ø€ Ü‰ÑÔØˆŒÜ—m‘mÌeÐTZ×TlÑTlÓNmÖ$nÈÔ%<¸VÕ%DÒ$nÓoˆŒØ&+ˆÕ#ùò %os   ½A#Nrž   rª   r=   c                 óT   — |}| j                   D ]  } |||fi |¤Ž}Œ t        |¬«      S )N©Úlast_hidden_state)rß   r   )rE   Úinputs_embedsrž   rª   r6   Úencoder_layers         r+   rT   zOvis2VisionEncoder.forward:  s<   € ð &ˆØ!Ÿ[™[ò 	SˆMÙ)¨-¸ÑRÈ6ÑR‰Mð	Sô °Ô?Ð?r*   rp   )r"   r#   r$   r%   r   r@   r   r   r&   rZ   r   r   r   rT   r[   r\   s   @r+   rÙ   rÙ   *  sh   ø„ ñð,Ð0õ ,ð Øð /3ñ
@ð Ÿ™ tÑ+ð
@ð Ð+Ñ,ð	
@ð
 
ò
@ó ó ô
@r*   rÙ   c                   óX   ‡ — e Zd Zdefˆ fd„Ze	 ddej                  dz  fd„«       Zˆ xZ	S )ÚOvis2VisionTransformerrd   c                 óÈ   •— t         ‰| �  «        || _        t        |«      | _        t        |«      | _        t        |j                  |j                  «      | _
        d| _        y rÛ   )r?   r@   rd   rx   r™   rÙ   Úencoderr;   rF   r�   r‘   rà   rm   s     €r+   r@   zOvis2VisionTransformer.__init__J  sO   ø€ Ü‰ÑÔØˆŒÜ/°Ó7ˆŒÜ)¨&Ó1ˆŒÜ$ V×%7Ñ%7¸×9LÑ9LÓMˆŒØ&+ˆÕ#r*   Nrž   c                 ó    — | j                  |«      } | j                  d||dœ|¤Ž}|j                  }| j                  |«      }t	        |¬«      S )N)rä   rž   râ   r)   )r™   ré   rã   r‘   r   )rE   r’   rž   rª   r6   Úencoder_outputsrã   s          r+   rT   zOvis2VisionTransformer.forwardR  sa   € ð Ÿ™¨Ó5ˆà+7¨4¯<©<ð ,
Ø'Ø)ñ,
ð ñ,
ˆð ,×=Ñ=ÐØ ŸM™MÐ*;Ó<ÐäÐ1BÔCÐCr*   rp   )
r"   r#   r$   r   r@   r   r&   rZ   rT   r[   r\   s   @r+   rç   rç   I  s?   ø„ ð,Ð0õ ,ð ð /3ñDð Ÿ™ tÑ+òDó ôDr*   rç   c                   óP   ‡ — e Zd Zdej                  dej                  fˆ fd„Zˆ xZS )ÚOvis2VisualEmbeddingTableÚvisual_tokensr=   c                 ó  •— |j                   t        j                  t        j                  t        j                  t        j
                  t        j                  fv rt        ‰| �!  |«      S t        j                  || j                  «      S rp   )rL   r&   Úint8Úint16Úint32Úint64Úlongr?   rT   r¦   rC   )rE   rî   rG   s     €r+   rT   z!Ovis2VisualEmbeddingTable.forwardh  sW   ø€ Ø×Ñ¤5§:¡:¬u¯{©{¼E¿K¹KÌÏÉÔV[×V`ÑV`Ð"aÑaÜ‘7‘? =Ó1Ð1Ü�|‰|˜M¨4¯;©;Ó7Ð7r*   )r"   r#   r$   r&   rZ   rT   r[   r\   s   @r+   rí   rí   g  s#   ø„ ð8 U§\¡\ð 8°e·l±l÷ 8ñ 8r*   rí   c                   óZ   ‡ — e Zd ZU eed<   dZdZdZdgZdgZ	dZ
dZdZdZdZdZˆ fd„Zˆ xZS )ÚOvis2PreTrainedModelrd   Úmodel)ÚimageÚtextTr¯   r5   c                 óø   •— t         ‰| �  |«       t        |t        «      rZt	        j
                  |j                  t        j                  |j                  j                  d   «      j                  d«      «       y y )NrI   r�   )r?   Ú_init_weightsÚ
isinstancerx   ÚinitÚcopy_r€   r&   rŽ   rV   r�   )rE   rš   rG   s     €r+   rû   z"Ovis2PreTrainedModel._init_weights}  s[   ø€ Ü‰Ñ˜fÔ%Ü�fÔ3Ô4Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhð 5r*   )r"   r#   r$   r   r(   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_cache_classÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpaÚ_can_compile_fullgraphÚ_supports_attention_backendrû   r[   r\   s   @r+   rö   rö   n  s\   ø… ØÓØÐØ(ÐØ&*Ð#Ø/Ð0ÐØ#4Ð"5ÐØ ÐØÐØÐØ€Nà!ÐØ"&Ð÷ið ir*   rö   r4   r£   c                 óì   — | j                  |«      }|j                  |d¬«      d   }t        j                  | t        j                  ¬«      j                  ||d«      }||j                  «       z
  |z   }|S )NTrJ   r   )Úmemory_formatg      ð?)r¨   Úmaxr&   Ú
zeros_likeÚlegacy_contiguous_formatÚscatter_Údetach)r4   r£   Úy_softÚindexÚy_hardÚrets         r+   Úhard_softmaxr  ƒ  sk   € Ø�^‰^˜CÓ €Fà�J‰J�s DˆJÓ)¨!Ñ,€EÜ×Ñ˜f´E×4RÑ4RÔS×\Ñ\Ð]`ÐbgÐilÓm€FØ
�6—=‘=“?Ñ
" VÑ
+€Cà€Jr*   c            	       ó‚   ‡ — e Zd ZU eed<   eedœZdefˆ fd„Ze	e
dej                  dee   deez  fd„«       «       Zˆ xZS )ÚOvis2VisionModelrd   )r6   r7   c                 óÈ  •— t         ‰| �  |«       || _        t        |«      | _        |j
                  | _        |j                  | _        t        j                  |j                  |j                  z  |j                  z  | j                  | j
                  z
  d¬«      | _        t        j                  | j                  | j
                  z
  «      | _        | j                  «        y ©NFra   )r?   r@   rd   rç   ÚtransformerÚnum_visual_indicator_tokensÚ
vocab_sizer   rf   rF   Úhidden_strideÚhead_linearÚ	LayerNormÚ	head_normÚ	post_initrm   s     €r+   r@   zOvis2VisionModel.__init__”  s¯   ø€ Ü‰Ñ˜Ô ØˆŒÜ1°&Ó9ˆÔØ+1×+MÑ+MˆÔ(Ø ×+Ñ+ˆŒÜŸ9™9Ø×Ñ ×!5Ñ!5Ñ5¸×8LÑ8LÑLØ�O‰O˜d×>Ñ>Ñ>Øô
ˆÔô
 Ÿ™ d§o¡o¸×8XÑ8XÑ&XÓYˆŒà�‰Õr*   r’   rª   r=   c           	      ó°  —  | j                   |fi |¤Ž}|d   }| j                  j                  dkD  rÚ|j                  \  }}}| j                  j                  }t	        t        j                  |«      «      }	|	|	z  |k7  rt        d«      ‚||	|z  z
  |z  }
t        j                  j                  |ddd|
d|
fdd«      }|	|
z  }	|j                  ||	|z  ||	|z  ||«      }|j                  dddddd«      }|j                  |d	||z  |z  «      }| j                  |«      }| j                  |«      }| j                  j                  d
k(  r#t        j                  j!                  |d	d¬«      }na| j                  j                  dk(  rt#        |d	¬«      }n:| j                  j                  dk(  r!t        j                  j%                  |d	¬«      }t'        |¬«      S )Nr   r   z.Token sequence length must be a perfect squareÚconstantr   r   é   é   rI   Úgumbel_argmaxT)r£   ÚhardÚ	st_argmax©r£   r¨   )rã   Úpooler_output)r  rd   r  rV   ÚintÚmathÚsqrtr´   r   r§   ÚpadrÂ   Úpermuter  r   Útokenize_functionÚgumbel_softmaxr  r¨   r    )rE   r’   rª   Úoutputsrã   Ú
num_imagesÚseq_lenÚ
hidden_dimr  Úsqrt_lÚpad_sizer4   Ú
prob_tokens                r+   rT   zOvis2VisionModel.forward£  sá  € ð
 #�$×"Ñ" <Ñ:°6Ñ:ˆØ# A™JÐØ�;‰;×$Ñ$ qÒ(Ø.?×.EÑ.EÑ+ˆJ˜ Ø ŸK™K×5Ñ5ˆMäœŸ™ 7Ó+Ó,ˆFØ˜‰ 'Ò)Ü Ð!QÓRÐRà%¨°-Ñ)?Ñ@ÀMÑQˆHÜ "§¡× 1Ñ 1Ð2CÀaÈÈAÈxÐYZÐ\dÐEeÐgqÐstÓ uÐØ�hÑˆFà 1× 9Ñ 9Ø˜F mÑ3°]ÀFÈmÑD[Ð]jÐlvó!Ðð !2× 9Ñ 9¸!¸QÀÀ1ÀaÈÓ KÐØ 1× 9Ñ 9Ø˜B °Ñ =À
Ñ Jó!Ðð ×!Ñ!Ð"3Ó4ˆØ—‘ Ó'ˆà�;‰;×(Ñ(¨OÒ;ÜŸ™×5Ñ5°fÀ"È4Ð5ÓP‰JØ�[‰[×*Ñ*¨kÒ9Ü% f°"Ô5‰JØ�[‰[×*Ñ*¨iÒ7ÜŸ™×.Ñ.¨v¸2Ð.Ó>ˆJä9Ø/Ø$ô
ð 	
r*   )r"   r#   r$   r   r(   rÎ   r¯   Ú_can_record_outputsr@   r   r   r&   r'   r   r   r8   r    rT   r[   r\   s   @r+   r  r  �  sj   ø… ØÓà0Ø*ñÐð
Ð0õ ð  Øð&
Ø!×-Ñ-ð&
Ø9?Ð@RÑ9Sð&
à	Ð;Ñ	;ò&
ó ó  ô&
r*   r  zu
    The Ovis2 model which consists of a vision backbone and a language model, without a language modeling head.
    c                   óê  ‡ — e Zd Zdefˆ fd„Ze ed¬«      dej                  de	e
   deez  fd„«       «       Zd	ej                  d
ej                  dej                  fd„Zee	 	 	 	 	 	 	 	 	 dd	ej                  dz  dej                  dz  dej                   dz  dej                  dz  dedz  d
ej                  dz  dej                  dz  dedz  deej                   z  deez  fd„«       «       Zˆ xZS )Ú
Ovis2Modelrd   c                 ó˜  •— t         ‰| �  |«       t        |j                  «      | _        t        j                  |j                  «      | _        t        |j                  j                  |j                  «      | _        |j                  j                  | _        |j                  | _
        |j                  | _        | j                  «        y rp   )r?   r@   r  Úvision_configÚvision_towerr   Úfrom_configÚtext_configÚlanguage_modelrí   r  rF   Úvisual_embeddings_tableÚvisual_vocab_sizeÚvisual_indicator_token_idsr!  rm   s     €r+   r@   zOvis2Model.__init__Ô  s—   ø€ Ü‰Ñ˜Ô Ü,¨V×-AÑ-AÓBˆÔÜ'×3Ñ3°F×4FÑ4FÓGˆÔÜ'@À×AUÑAU×A`ÑA`Ðbh×btÑbtÓ'uˆÔ$à!'×!5Ñ!5×!@Ñ!@ˆÔØ ×+Ñ+ˆŒØ*0×*KÑ*KˆÔ'Ø�‰Õr*   zWObtains image last hidden states from the vision tower and apply multimodal projection.r,   r’   rª   r=   c                 óh  —  | j                   |fddi|¤Ž}|j                  }|j                  \  }}}t        j                  ||| j                   j
                  f|j                  |j                  d|j                  ¬«      }t        j                  ||gd¬«      }| j                  |«      }t        j                  | j                  | j                   j
                  z
  | j                  t        j                  ¬«      j                  |j                  «      }	||_        | j                  |	«      |_        |S )NÚreturn_dictTF)rL   ÚdeviceÚrequires_gradÚlayoutr   r)  r”   )r>  r*  rV   r&   Úzerosr  rL   rG  rI  ÚcatrB  rŽ   rC  rô   rM   r!   )
rE   r’   rª   Úimage_outputsÚimage_featuresÚ
batch_sizeÚimg_seq_lenrÖ   Úpadding_tensorÚvisual_indicators
             r+   Úget_image_featureszOvis2Model.get_image_featuresß  s  € ð *˜×)Ñ)¨,ÑSÀDÐSÈFÑSˆØ&×4Ñ4ˆØ%3×%9Ñ%9Ñ"ˆ
�K ÜŸ™Ø˜ d×&7Ñ&7×&SÑ&SÐTØ ×&Ñ&Ø!×(Ñ(ØØ!×(Ñ(ô
ˆô Ÿ™ N°NÐ#CÈÔKˆØ×5Ñ5°nÓEˆä Ÿ<™<Ø×"Ñ" T×%6Ñ%6×%RÑ%RÑRØ×"Ñ"Ü—*‘*ô
÷ ‰"ˆ^×"Ñ"Ó
#ð	 	ð
 '5ˆÔ#Ø26×2NÑ2NÐO_Ó2`ˆÔ/àÐr*   Ú	input_idsrä   rM  c                 ó.  — |€m| | j                  «       t        j                  | j                  j                  t        j
                  |j                  ¬«      «      k(  }|j                  d«      }n|| j                  j                  k(  }|j                  «       }|j                  d   |j                  d   z  }|j                  d«      j                  |j                  «      }t        ||j                  d   z  |j                  «       k(  d|› d|› �«       |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        ©rL   rG  rI   r   r   z6Image features and image tokens do not match, tokens: z, features: )Úget_input_embeddingsr&   Útensorrd   Úimage_token_idrô   rG  ÚallÚsumrV   Ú	unsqueezerM   r   Únumel)rE   rS  rä   rM  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r+   Úget_placeholder_maskzOvis2Model.get_placeholder_maskÿ  s  € ð ÐØ!.Ð2M°$×2KÑ2KÓ2MÜ—‘˜TŸ[™[×7Ñ7¼u¿z¹zÐR_×RfÑRfÔgó3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¨d¯k©k×.HÑ.HÑ!HÐà+×/Ñ/Ó1ˆØ)×/Ñ/°Ñ2°^×5IÑ5IÈ!Ñ5LÑLÐØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐÜØ˜]×0Ñ0°Ñ4Ñ4¸×8LÑ8LÓ8NÑNØDÀ^ÐDTÐT`ÐaqÐ`rÐsô	
ð "Ð!r*   Nrž   r€   r5   ÚlabelsÚ	use_cacheÚlogits_to_keepc
           
      ó¨  — |d u |d uz  rt        d«      ‚|€ | j                  «       |«      }|��J| j                  |d¬«      }|j                  }|j                  }| j                  |||¬«      }|j                  |j                  |j                  «      }|j                  ||«      }t        | j                  «      D ]»  \  }}|€Y| | j                  «       t        j                  |t        j                  |j                  ¬«      «      k(  }|j                  d«      }n||k(  j                  |j                  «      }|j!                  «       sŒ�||   j                  |j                  |j                  «      ||<   Œ½  | j"                  d	||||||	dœ|
¤Ž}t%        |j&                  |j(                  |j*                  |j,                  |�¬«      S d ¬«      S )
Nz:You must specify exactly one of input_ids or inputs_embedsT)r’   rF  )rä   rM  rU  rI   )rž   r€   r5   rä   rb  rc  )rã   r5   r6   r7   r0   r)   )r´   rV  rR  r*  r!   r`  rM   rG  rL   Úmasked_scatterÚ	enumeraterD  r&   rW  rô   rY  ÚanyrA  r/   rã   r5   r6   r7   )rE   rS  r’   rž   r€   r5   rä   ra  rb  rc  rª   rL  rM  r!   r]  ÚiÚvisual_indicator_idÚmaskr2  s                      r+   rT   zOvis2Model.forward  s  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø7˜D×5Ñ5Ó7¸	ÓBˆMàÑ#Ø ×3Ñ3ÀÐ[_Ð3Ó`ˆMØ*×8Ñ8ˆNØ(5×(OÑ(OÐ%à!%×!:Ñ!:ØØ+Ø-ð ";ó "Ðð
 ,×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ)×8Ñ8Ð9KÈ^Ó\ˆMä*3°D×4SÑ4SÓ*Tò 
uÑ&�Ð&ØÐ$Ø(Ð,G¨D×,EÑ,EÓ,GÜŸ™Ð%8ÄÇ
Á
ÐS`×SgÑSgÔhó-ñ �Dð  Ÿ8™8 B›<‘Dà%Ð)<Ñ<×@Ñ@À×AUÑAUÓV�Dà—8‘8•:Ø*CÀAÑ*F×*IÑ*IÈ-×J^ÑJ^Ð`m×`sÑ`sÓ*t�M $Ò'ð
uð &�$×%Ñ%ð 
Ø)Ø%Ø+Ø'ØØ)ñ
ð ñ
ˆô (Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø2>Ð2J ô
ð 	
ð
 QUô
ð 	
r*   ©	NNNNNNNNr   )r"   r#   r$   r   r@   r   r   r&   r'   r   r   r8   r    rR  Ú
LongTensorr`  rZ   r	   Úboolr+  r/   rT   r[   r\   s   @r+   r;  r;  Î  s‚  ø„ ð	˜{õ 	ð ÙØnôðà×'Ñ'ðð Ð+Ñ,ðð 
Ð;Ñ	;ò	óó ðð8"Ø×)Ñ)ð"Ø:?×:KÑ:Kð"Ø]b×]nÑ]nó"ð0 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ<
à×#Ñ# dÑ*ð<
ð ×'Ñ'¨$Ñ.ð<
ð Ÿ™ tÑ+ð	<
ð
 ×&Ñ&¨Ñ-ð<
ð  ™ð<
ð ×(Ñ(¨4Ñ/ð<
ð × Ñ  4Ñ'ð<
ð ˜$‘;ð<
ð ˜eŸl™lÑ*ð<
ð 
Ð)Ñ	)ò<
ó ó ô<
r*   r;  c                   óÂ  ‡ — e Zd ZddiZdefˆ fd„Zdej                  fd„Ze	de
j                  dee   deez  fd	„«       Zee		 	 	 	 	 	 	 	 	 dde
j$                  d
z  de
j                  d
z  de
j&                  d
z  de
j$                  d
z  ded
z  de
j                  d
z  de
j$                  d
z  ded
z  dee
j&                  z  deez  fd„«       «       Z	 	 	 	 	 	 dˆ fd„	Zˆ xZS )ÚOvis2ForConditionalGenerationzlm_head.weightz(model.language_model.embed_tokens.weightrd   c                 óÆ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        | j                  «        y r  )
r?   r@   r;  r÷   r   rf   rF   r  Úlm_headr!  rm   s     €r+   r@   z&Ovis2ForConditionalGeneration.__init__\  sF   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ�‰Õr*   r=   c                 ó   — | j                   S rp   )rq  rW   s    r+   Úget_output_embeddingsz3Ovis2ForConditionalGeneration.get_output_embeddingsb  s   € Ø�|‰|Ðr*   r’   rª   c                 ó>   —  | j                   j                  dd|i|¤ŽS )Nr’   r)   )r÷   rR  )rE   r’   rª   s      r+   rR  z0Ovis2ForConditionalGeneration.get_image_featurese  s#   € ð -ˆt�z‰z×,Ñ,ÑQ¸,ÐQÈ&ÑQÐQr*   NrS  rž   r€   r5   rä   ra  rb  rc  c
                 ó–  —  | j                   d|||||||dœ|
¤Ž}|d   }t        |	t        «      rt        |	 d«      n|	}| j	                  |dd…|dd…f   «      }d}|�4 | j
                  d||| j                  j                  j                  dœ|
¤Ž}t        |||j                  |j                  |j                  |j                  ¬«      S )aâ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Ovis2ForConditionalGeneration

        >>> model = Ovis2ForConditionalGeneration.from_pretrained("thisisiron/Ovis2-2B-hf")
        >>> processor = AutoProcessor.from_pretrained("thisisiron/Ovis2-2B-hf")

        >>> prompt = "<|im_start|>user\n<image>\nDescribe the image.<|im_end|>\n<|im_start|>assistant\n"
        >>> url = "http://images.cocodataset.org/val2014/COCO_val2014_000000537955.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs, max_new_tokens=15)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True)[0]
        "user\n\nDescribe the image.\nassistant\nThe image features a brown dog standing on a wooden floor, looking up with"
        ```)rS  r’   rž   r€   r5   rä   rb  r   N)r4   ra  r  )r3   r4   r5   r6   r7   r0   r)   )r÷   rü   r+  Úslicerq  Úloss_functionrd   r@  r  r2   r5   r6   r7   r0   )rE   rS  r’   rž   r€   r5   rä   ra  rb  rc  rª   r2  r6   Úslice_indicesr4   r3   s                   r+   rT   z%Ovis2ForConditionalGeneration.forwardk  só   € ðX �$—*‘*ð 	
ØØ%Ø)Ø%Ø+Ø'Øñ	
ð ñ	
ˆð   ™
ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%ð Ø f¸¿¹×9PÑ9P×9[Ñ9[ñØ_eñˆDô +ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ô
ð 	
r*   c           	      óf   •— t        ‰
| �  |f|||||dœ|¤Ž}	|s|j                  dd«      s||	d<   |	S )N)r5   rä   rž   rc  Úis_first_iterationrb  Tr’   )r?   Úprepare_inputs_for_generationÚget)rE   rS  r5   rä   r’   rž   rc  rz  rª   Úmodel_inputsrG   s             €r+   r{  z;Ovis2ForConditionalGeneration.prepare_inputs_for_generation¶  sY   ø€ ô ‘wÑ<Øð
à+Ø'Ø)Ø)Ø1ñ
ð ñ
ˆñ  V§Z¡Z°¸TÔ%Bð
 ,8ˆL˜Ñ(àÐr*   rk  )NNNNNF)r"   r#   r$   Ú_tied_weights_keysr   r@   r   ÚModulers  r   r&   r'   r   r   r8   r    rR  r   rl  rZ   r	   rm  r+  r2   rT   r{  r[   r\   s   @r+   ro  ro  X  s‹  ø„ à*Ð,VÐWÐð˜{õ ð r§y¡yó ð ðRØ!×-Ñ-ðRØ9?Ð@RÑ9SðRà	Ð;Ñ	;òRó ðRð
 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñG
à×#Ñ# dÑ*ðG
ð ×'Ñ'¨$Ñ.ðG
ð Ÿ™ tÑ+ð	G
ð
 ×&Ñ&¨Ñ-ðG
ð  ™ðG
ð ×(Ñ(¨4Ñ/ðG
ð × Ñ  4Ñ'ðG
ð ˜$‘;ðG
ð ˜eŸl™lÑ*ðG
ð 
Ð,Ñ	,òG
ó ó ðG
ðX ØØØØØ ÷ñ r*   ro  )rö   r;  ro  )r¾   )Dr,  Úcollections.abcr   Údataclassesr   r&   r   Ú r   rý   Úactivationsr   Úcache_utilsr	   Ú
generationr
   Úintegrationsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úautor   Úconfiguration_ovis2r   r   r    r/   r2   r  r;   r^   rx   rZ   rY   r­   r¯   rÊ   rÎ   rÙ   rç   r‹   rí   rö   r+  r  r  r;  ro  Ú__all__r)   r*   r+   ú<module>r‘     s_  ðó* Ý $Ý !ã Ý å &Ý !Ý  Ý )Ý 7Ý 9ß dÑ dß FÝ &ß nÕ nÝ 7Ý 5Ý ß ?ð Ø
ô?Ð1Kó ?ó ó ð?ñ ðôð
 ô9Ð6ó 9ó óð9ñ ðôð
 ô9 +ó 9ó óð9ñ0 ˜YÓ'ôJ�2—9‘9ó Jó (ðJô(�R—Y‘Yô ô ˜BŸI™Iô ðP ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ó%ô.7)˜2Ÿ9™9ô 7)ôtˆr�y‰yô ô Ð8ô ô2@˜Ÿ™ô @ô>D˜RŸY™Yô Dô<8 §¡ô 8ôi˜?ô ið*˜Ÿ™ð ¨Có ô>
Ð+ô >
ñB ðôô
B
Ð%ó B
óð
B
ðJ ô{Ð$8¸/ó {ó ð{ò| R�r*   