Ë
    ÛÓ^j÷ ã                   óš  — d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	Z	ddl	m
Z
 ddlmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1m2Z2m3Z3  e)jh                  e5«      Z6e' G d„ de!«      «       Z7dWde	jp                  de	jr                  de:dz  fd„Z;	 dXde	jx                  de	jr                  de	jz                  de:fd„Z>e'e G d „ d!e«      «       «       Z? e'd"¬#«      e G d$„ d%e%«      «       «       Z@ e'd&¬#«      e G d'„ d(e%«      «       «       ZA G d)„ d*e
j„                  «      ZC	 dYd+e
j„                  d,e	jp                  d-e	jp                  d.e	jp                  d/e	jp                  dz  d0eDd1eDfd2„ZE G d3„ d4e
j„                  «      ZF G d5„ d6e
j„                  «      ZG G d7„ d8e«      ZH G d9„ d:e
j„                  «      ZI G d;„ d<e7«      ZJ G d=„ d>e
j„                  «      ZK G d?„ d@e
j„                  «      ZL G dA„ dBe
j„                  «      ZM G dC„ dDe«      ZN G dE„ dFe7«      ZO G dG„ dHe7«      ZP G dI„ dJe7«      ZQ e'dK¬#«       G dL„ dMe7e«      «       ZR G dN„ dOe
j„                  «      ZS e'dP¬#«       G dQ„ dRe7«      «       ZT e'dS¬#«       G dT„ dUe7e«      «       ZUg dV¢ZVy)ZzPyTorch KOSMOS-2 model.é    N)ÚCallable)Ú	dataclass)ÚAny)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚ!CausalLMOutputWithCrossAttentions)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚ	torch_int)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚKosmos2ConfigÚKosmos2TextConfigÚKosmos2VisionConfigc                   óˆ   ‡ — e Zd ZU eed<   dZdZddgZdZdZ	dZ
 ej                  «       dej                  fˆ fd„«       Zˆ xZS )	ÚKosmos2PreTrainedModelÚconfig)ÚimageÚtextTÚKosmos2VisionEncoderLayerÚKosmos2TextBlockFÚmodulec                 óè  •— t         ‰| �  |«       t        | j                  d«      r| j                  j                  }n6t        | j                  d«      r | j                  j
                  j                  }t        | j                  d«      r| j                  j                  }n6t        | j                  d«      r | j                  j                  j                  }t        |t        «      �rt        j                  |j                  d|j                  dz  z  ¬«       t        j                  |j                  j                  |j                  j                   |z  ¬«       t        j                  |j"                  j                  |j                  j                   |z  ¬«       t        j$                  |j&                  t)        j*                  |j&                  j,                  d	   «      j/                  d
«      «       yt        |t0        «      rî|j                  dz  d|j                  j2                  z  dz  z  z  }|j                  dz  |z  }t        j                  |j4                  j                  |¬«       t        j                  |j6                  j                  |¬«       t        j                  |j8                  j                  |¬«       t        j                  |j:                  j                  |¬«       yt        |t<        «      r¯|j                  j>                  dz  d|j                  j2                  z  dz  z  z  }d|j                  j>                  z  dz  |z  }t        j                  |j@                  j                  |¬«       t        j                  |jB                  j                  |¬«       yt        |tD        «      r­t        j                  |j4                  j                  ¬«       t        j                  |j6                  j                  |¬«       t        j                  |j8                  j                  |¬«       t        j                  |j:                  j                  |¬«       yt        |tF        «      rWt        j                  |j@                  j                  ¬«       t        j                  |jB                  j                  |¬«       yt        |tH        «      r,t        j                  |jJ                  j                  ¬«       yt        |tL        «      rKt        j                  |jN                  j                  ¬«       t        j                  |jP                  «       yt        |tR        «      r„t        j                  |jT                  j                  d¬«       |jT                  jV                  �At        jX                  |jT                  j                  |jT                  jV                     «       yyt        |tZ        «      r_|j]                  |j^                  |j`                  z   |jb                  |jV                  «      }t        j$                  |jd                  |«       yy)zInitialize the weightsÚinitializer_factorÚvision_configÚinit_stdÚtext_configç        ç      à¿)ÚmeanÚstd)r4   éÿÿÿÿ©r    r5   é   N)3ÚsuperÚ_init_weightsÚhasattrr&   r-   r.   r/   r0   Ú
isinstanceÚKosmos2VisionEmbeddingsÚinitÚnormal_Úclass_embeddingÚ	embed_dimÚpatch_embeddingÚweightÚinitializer_rangeÚposition_embeddingÚcopy_Úposition_idsÚtorchÚarangeÚshapeÚexpandÚKosmos2VisionAttentionÚnum_hidden_layersÚq_projÚk_projÚv_projÚout_projÚKosmos2VisionMLPÚhidden_sizeÚfc1Úfc2ÚKosmosTextAttentionÚKosmos2TextFFNÚKosmos2TextForCausalLMÚlm_headÚKosmos2ImageToTextProjectionÚdenseÚlatent_queryÚKosmos2TextTransformerÚembed_tokensÚpadding_idxÚzeros_Ú(Kosmos2TextSinusoidalPositionalEmbeddingÚget_embeddingÚnum_positionsÚoffsetÚembedding_dimÚweights)	Úselfr+   Úfactorr4   Úin_proj_stdÚout_proj_stdÚfc_stdÚemb_weightsÚ	__class__s	           €úw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/kosmos2/modeling_kosmos2.pyr9   z$Kosmos2PreTrainedModel._init_weights:   sM  ø€ ô 	‰Ñ˜fÔ%Ü�4—;‘;Ð 4Ô5Ø—[‘[×3Ñ3‰FÜ�T—[‘[ /Ô2Ø—[‘[×.Ñ.×AÑAˆFä�4—;‘; 
Ô+Ø—+‘+×&Ñ&‰CÜ�T—[‘[ -Ô0Ø—+‘+×)Ñ)×2Ñ2ˆCä�fÔ5Õ6Ü�L‰L˜×/Ñ/°c¸v×?OÑ?OÐQUÑ?UÐX^Ñ?^Õ_Ü�L‰L˜×/Ñ/×6Ñ6¸F¿M¹M×<[Ñ<[Ð^dÑ<dÕeÜ�L‰L˜×2Ñ2×9Ñ9¸v¿}¹}×?^Ñ?^ÐagÑ?gÕhÜ�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhÜ˜Ô 6Ô7Ø!×+Ñ+¨TÑ1°q¸6¿=¹=×;ZÑ;ZÑ7ZÐ_cÑ6cÑdÐgmÑmˆKØ"×,Ñ,¨dÑ2°fÑ<ˆLÜ�L‰L˜Ÿ™×-Ñ-°;Õ?Ü�L‰L˜Ÿ™×-Ñ-°;Õ?Ü�L‰L˜Ÿ™×-Ñ-°;Õ?Ü�L‰L˜Ÿ™×/Ñ/°\ÖBÜ˜Ô 0Ô1Ø!Ÿ=™=×4Ñ4°dÑ:ÀÀFÇMÁM×DcÑDcÑ@cÐhlÑ?lÑmÐpvÑvˆKØ˜&Ÿ-™-×3Ñ3Ñ3¸Ñ<¸vÑEˆFÜ�L‰L˜Ÿ™×*Ñ*°Õ7Ü�L‰L˜Ÿ™×*Ñ*°Ö<Ü˜Ô 3Ô4Ü�L‰L˜Ÿ™×-Ñ-°3Õ7Ü�L‰L˜Ÿ™×-Ñ-°3Õ7Ü�L‰L˜Ÿ™×-Ñ-°3Õ7Ü�L‰L˜Ÿ™×/Ñ/°SÖ9Ü˜¤Ô/Ü�L‰L˜Ÿ™×*Ñ*°Õ4Ü�L‰L˜Ÿ™×*Ñ*°Ö4Ü˜Ô 6Ô7Ü�L‰L˜Ÿ™×.Ñ.°CÖ8Ü˜Ô <Ô=Ü�L‰L˜Ÿ™×,Ñ,°#Õ6Ü�L‰L˜×,Ñ,Õ-Ü˜Ô 6Ô7Ü�L‰L˜×,Ñ,×3Ñ3¸#À3ÕGØ×"Ñ"×.Ñ.Ð:Ü—‘˜F×/Ñ/×6Ñ6°v×7JÑ7J×7VÑ7VÑWÕXð ;ä˜Ô HÔIØ ×.Ñ.Ø×$Ñ$ v§}¡}Ñ4°f×6JÑ6JÈF×L^ÑL^óˆKô �J‰J�v—~‘~ {Õ3ð	 Jó    )Ú__name__Ú
__module__Ú__qualname__r!   Ú__annotations__Úinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdparG   Úno_gradr   ÚModuler9   Ú__classcell__©rl   s   @rm   r%   r%   0   sW   ø… àÓØ(ÐØ&*Ð#Ø4Ð6HÐIÐØ"&ÐØ ÐØ€Nà€U‡]�]ƒ_ð34 B§I¡Iô 34ó ô34rn   r%   ÚmaskÚdtypeÚtgt_lenc                 ó2  — | j                  «       \  }}|�|n|}| dd…dddd…f   j                  |d||«      j                  |«      }d|z
  }|j                  |j                  t        j
                  «      t	        j                  |«      j                  «      S )z_
    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
    Nr    ç      ð?)ÚsizerJ   ÚtoÚmasked_fillrG   ÚboolÚfinfoÚmin)r}   r~   r   ÚbszÚsrc_lenÚexpanded_maskÚinverted_masks          rm   Ú_expand_maskrŒ   q   s‰   € ð —9‘9“;�L€CˆØ Ð,‰g°'€Gàš˜D $ªÐ)Ñ*×1Ñ1°#°q¸'À7ÓK×NÑNÈuÓU€Mà˜-Ñ'€Mà×$Ñ$ ]×%5Ñ%5´e·j±jÓ%AÄ5Ç;Á;ÈuÓCU×CYÑCYÓZÐZrn   Úinput_ids_shapeÚdeviceÚpast_key_values_lengthc                 ó  — | \  }}t        j                  ||ft        j                  |«      j                  |¬«      }t        j                  |j                  d«      |¬«      }|j                  ||dz   j                  |j                  d«      d«      k  d«       |j                  |«      }|dkD  r0t        j                  t        j                  ||||¬«      |gd¬«      }|dddd…dd…f   j                  |d|||z   «      S )zB
    Make causal mask used for bi-directional self-attention.
    )rŽ   r5   r    r   ©r~   rŽ   ©ÚdimN)rG   Úfullr†   r‡   rH   r‚   Úmasked_fill_Úviewrƒ   ÚcatÚzerosrJ   )r�   r~   rŽ   r�   rˆ   r   r}   Ú	mask_conds           rm   Ú_make_causal_maskrš      sá   € ð #�L€CˆÜ�:‰:�w Ð(¬%¯+©+°eÓ*<×*@Ñ*@ÈÔP€DÜ—‘˜TŸY™Y r›]°6Ô:€IØ×Ñ�i 9¨q¡=×"6Ñ"6°t·y±yÀ³}ÀaÓ"HÑHÈ!ÔLØ�7‰7�5‹>€Dà Ò!Ü�y‰yœ%Ÿ+™+ gÐ/EÈUÐ[aÔbÐdhÐiÐoqÔrˆØ��dšAšqÐ Ñ!×(Ñ(¨¨a°¸'ÐDZÑ:ZÓ[Ð[rn   c                   ó@   — e Zd ZU dZdZeej                     dz  ed<   y)Ú'BaseModelOutputWithProjectionAttentionsaq  
    projection_attentions (`tuple(torch.FloatTensor)`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights given by `Kosmos2ImageToTextProjection`, after the attention softmax, used to compute
        the weighted average in the self-attention heads.
    NÚprojection_attentions)	ro   rp   rq   Ú__doc__r�   ÚtuplerG   ÚFloatTensorrr   © rn   rm   rœ   rœ   �   s%   … ñð >BÐ˜5 ×!2Ñ!2Ñ3°dÑ:ÔArn   rœ   ze
    Base class for text model's outputs that also contains a pooling of the last hidden states.
    )Úcustom_introc                   ó   — e Zd ZU dZdZej                  dz  ed<   dZe	dz  ed<   dZ
eej                     dz  ed<   dZeej                     dz  ed<   dZej                  dz  ed<   dZeej                     dz  ed<   dZeed	<   d
ee   fd„Zy)ÚKosmos2ModelOutputa±  
    image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*):
        Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`.
    projection_attentions (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights given by `Kosmos2ImageToTextProjection`, after the attention softmax, used to compute
        the weighted average in the self-attention heads.
    vision_model_output (`BaseModelOutputWithPooling`, *optional*):
        The output of the [`Kosmos2VisionModel`].
    NÚlast_hidden_stateÚpast_key_valuesÚhidden_statesÚ
attentionsÚimage_embedsr�   Úvision_model_outputÚreturnc                 óH   ‡ — t        ˆ fd„‰ j                  «       D «       «      S )Nc              3   ód   •K  — | ]'  }|d vr‰|   nt        ‰|«      j                  «       –— Œ) y­w©)Útext_model_outputrª   N©ÚgetattrÚto_tuple©Ú.0Úkrf   s     €rm   ú	<genexpr>z.Kosmos2ModelOutput.to_tuple.<locals>.<genexpr>¼   ó=   øè ø€ ò 
àð Ð LÑLˆD�ŠGÔRYÐZ^Ð`aÓRb×RkÑRkÓRmÓmñ
ùó   ƒ-0©rŸ   Úkeys©rf   s   `rm   r²   zKosmos2ModelOutput.to_tuple»   ó#   ø€ Üó 
à—Y‘Y“[ô
ó 
ð 	
rn   )ro   rp   rq   rž   r¥   rG   r    rr   r¦   r
   r§   rŸ   r¨   r©   r�   rª   r   r   r²   r¡   rn   rm   r¤   r¤   Ÿ   s­   … ñð 37Ð�u×(Ñ(¨4Ñ/Ó6Ø$(€O�U˜T‘\Ó(Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ó6Ø-1€L�%×#Ñ# dÑ*Ó1Ø=AÐ˜5 ×!2Ñ!2Ñ3°dÑ:ÓAØ6:ÐÐ3Ó:ð
˜% ™*ô 
rn   r¤   zC
    Model output class for `Kosmos2ForConditionalGeneration`.
    c                   óH  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   dZej                  dz  ed<   dZeej                     dz  ed	<   dZeed
<   dee   fd„Zy)Ú*Kosmos2ForConditionalGenerationModelOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*):
        Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`.
    projection_attentions (`tuple(torch.FloatTensor)`, *optional*):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights given by `Kosmos2ImageToTextProjection`, after the attention softmax, used to compute
        the weighted average in the self-attention heads.
    vision_model_output (`BaseModelOutputWithPooling`, *optional*):
        The output of the [`Kosmos2VisionModel`].
    NÚlossÚlogitsr¦   r§   r¨   r©   r�   rª   r«   c                 óH   ‡ — t        ˆ fd„‰ j                  «       D «       «      S )Nc              3   ód   •K  — | ]'  }|d vr‰|   nt        ‰|«      j                  «       –— Œ) y­wr®   r°   r³   s     €rm   r¶   zFKosmos2ForConditionalGenerationModelOutput.to_tuple.<locals>.<genexpr>ä   r·   r¸   r¹   r»   s   `rm   r²   z3Kosmos2ForConditionalGenerationModelOutput.to_tupleã   r¼   rn   )ro   rp   rq   rž   r¿   rG   r    rr   rÀ   r¦   r
   r§   rŸ   r¨   r©   r�   rª   r   r   r²   r¡   rn   rm   r¾   r¾   Â   sÁ   … ñð" &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø$(€O�U˜T‘\Ó(Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ó6Ø-1€L�%×#Ñ# dÑ*Ó1Ø=AÐ˜5 ×!2Ñ!2Ñ3°dÑ:ÓAØ6:ÐÐ3Ó:ð
˜% ™*ô 
rn   r¾   c                   óž   ‡ — e Zd Zdefˆ fd„Zdej                  dededej                  fd„Zd
dej                  dej                  fd	„Z
ˆ xZS )r<   r&   c                 óÚ  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        t        j                  t        j                  | j                  «      «      | _        t        j                  |j                  | j                  | j                  | j                  d¬«      | _        | j
                  | j                  z  dz  | _        | j                  dz   | _        t        j"                  | j                   | j                  «      | _        | j'                  dt        j(                  | j                   «      j+                  d«      d¬«       y )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasr7   r    rF   r6   ©Ú
persistent)r8   Ú__init__r&   rR   r@   Ú
image_sizeÚ
patch_sizer   Ú	ParameterrG   Úrandnr?   ÚConv2dÚnum_channelsrA   Únum_patchesrb   Ú	EmbeddingrD   Úregister_bufferrH   rJ   ©rf   r&   rl   s     €rm   rÌ   z Kosmos2VisionEmbeddings.__init__ì   s	  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿ|™|¬E¯K©K¸¿¹Ó,GÓHˆÔä!Ÿy™yØ×+Ñ+ØŸ™ØŸ™Ø—?‘?Øô 
ˆÔð !ŸO™O¨t¯©Ñ>À1ÑDˆÔØ!×-Ñ-°Ñ1ˆÔÜ"$§,¡,¨t×/AÑ/AÀ4Ç>Á>Ó"RˆÔØ×Ñ˜^¬U¯\©\¸$×:LÑ:LÓ-M×-TÑ-TÐU\Ó-]ÐjoÐÕprn   Ú
embeddingsÚheightÚwidthr«   c                 óÒ  — |j                   d   dz
  }| j                  j                  j                  d«      }|j                   d   dz
  }t        j
                  j                  «       s%||k(  r ||k(  r| j                  | j                  «      S |dd…dd…f   }|dd…dd…f   }|j                   d   }	|| j                  z  }
|| j                  z  }t        |dz  «      }|j                  d|||	«      }|j                  dddd«      }t        j                  j                  ||
|fdd	¬
«      }|j                  dddd«      j                  dd|	«      }t	        j                   ||fd¬«      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r    r   Nr5   g      à?r   r7   ÚbicubicF)r‚   ÚmodeÚalign_cornersr’   )rI   rD   rB   Ú	unsqueezerG   ÚjitÚ
is_tracingrF   rÎ   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolater–   r—   )rf   r×   rØ   rÙ   rÓ   rD   rb   Úclass_pos_embedÚpatch_pos_embedr“   Ú
new_heightÚ	new_widthÚsqrt_num_positionss                rm   Úinterpolate_pos_encodingz0Kosmos2VisionEmbeddings.interpolate_pos_encoding  sv  € ð !×&Ñ& qÑ)¨AÑ-ˆØ!×4Ñ4×;Ñ;×EÑEÀaÓHÐØ*×0Ñ0°Ñ3°aÑ7ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×*Ñ*¨4×+<Ñ+<Ó=Ð=à,ªQ°°°¨UÑ3ˆØ,ªQ°±¨UÑ3ˆà×Ñ˜rÑ"ˆà˜tŸ™Ñ.ˆ
Ø˜TŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐI[Ð]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØ˜iÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�y‰y˜/¨?Ð;ÀÔCÐCrn   Úpixel_valuesc                 ó`  — |j                   \  }}}}|sJ|| j                  k7  s|| j                  k7  r,t        d|› d|› d| j                  › d| j                  › d�	«      ‚| j                  j                  j
                  }| j                  |j                  |¬«      «      }|j                  d«      j                  dd«      }| j                  j                  |dd«      }	t        j                  |	|gd¬	«      }
|r|
| j                  |
||«      z   }
|
S |
| j                  | j                  «      z   }
|
S )
NzInput image size (Ú*z) doesn't match model (ú).©r~   r7   r    r5   r’   )rI   rÍ   Ú
ValueErrorrA   rB   r~   rƒ   ÚflattenÚ	transposer?   rJ   rG   r—   rê   rD   rF   )rf   rë   rê   Ú
batch_sizeÚ_rØ   rÙ   Útarget_dtypeÚpatch_embedsÚclass_embedsr×   s              rm   ÚforwardzKosmos2VisionEmbeddings.forward+  s6  € Ø'3×'9Ñ'9Ñ$ˆ
�A�v˜uÙ'¨V°t·±Ò-FÈ%ÐSW×SbÑSbÒJbÜØ$ V H¨A¨e¨WÐ4KÈDÏOÉOÐK\Ð\]Ð^b×^mÑ^mÐ]nÐnpÐqóð ð ×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆØ#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆà×+Ñ+×2Ñ2°:¸qÀ"ÓEˆÜ—Y‘Y ¨lÐ;ÀÔCˆ
Ù#Ø# d×&CÑ&CÀJÐPVÐX]Ó&^Ñ^ˆJð Ðð $ d×&=Ñ&=¸d×>OÑ>OÓ&PÑPˆJØÐrn   ©F)ro   rp   rq   r#   rÌ   rG   ÚTensorÚintrê   r    rø   r{   r|   s   @rm   r<   r<   ë   se   ø„ ðqÐ2õ qð,'D°5·<±<ð 'DÈð 'DÐUXð 'DÐ]b×]iÑ]ió 'DñR E×$5Ñ$5ð ÐZ_×ZfÑZf÷ rn   r<   r+   ÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 óp  — t        j                  ||j                  dd«      «      |z  }|�||z   }t        j                  j                  |d¬«      }t        j                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Nr5   éþÿÿÿr’   ©ÚpÚtrainingr    r7   )	rG   Úmatmulrò   r   rã   Úsoftmaxr  r  Ú
contiguous)
r+   rü   rý   rþ   rÿ   r   r  ÚkwargsÚattn_weightsÚattn_outputs
             rm   Úeager_attention_forwardr  ?  s¢   € ô —<‘<  s§}¡}°R¸Ó'<Ó=ÀÑG€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØ˜Ð$Ð$rn   c                   óª   ‡ — e Zd ZdZˆ fd„Z	 d	dej                  dej                  dz  dee   de	ej                  ej                  dz  f   fd„Z
ˆ xZS )
rK   ú=Multi-headed attention from 'Attention Is All You Need' paperc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚| j                  dz  | _	        |j                  | _        d| _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        y )Nú;embed_dim must be divisible by num_heads (got `embed_dim`: ú and `num_heads`: rî   r2   F)r8   rÌ   r&   rR   r@   Únum_attention_headsÚ	num_headsÚhead_dimrð   ÚscaleÚattention_dropoutr  Ú	is_causalr   ÚLinearrN   rO   rM   rP   rÖ   s     €rm   rÌ   zKosmos2VisionAttention.__init__X  s  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒØˆŒä—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜŸ	™	 $§.¡.°$·.±.ÓAˆ�rn   Nr§   rÿ   r
  r«   c                 ó¼  — |j                   dd }g |¢d‘| j                  ‘­}| j                  |«      }| j                  |«      }| j	                  |«      }|j                  |«      j                  dd«      }|j                  |«      j                  dd«      }|j                  |«      j                  dd«      }t        j                  | j                  j                  t        «      }	 |	| ||||f| j                  | j                  | j                  sdn| j                  dœ|¤Ž\  }
} |
j                   g |¢d‘­Ž j#                  «       }
| j%                  |
«      }
|
|fS )ú#Input shape: Batch x Time x ChannelNr5   r    r7   r1   )r  r   r  )rI   r  rM   rN   rO   r–   rò   r   Úget_interfacer&   Ú_attn_implementationr  r  r  r  r  rá   r	  rP   )rf   r§   rÿ   r
  Úinput_shapeÚhidden_shapeÚqueriesrº   ÚvaluesÚattention_interfacer  r  s               rm   rø   zKosmos2VisionAttention.forwardl  sV  € ð $×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØ—+‘+˜mÓ,ˆØ�{‰{˜=Ó)ˆØ—‘˜]Ó+ˆà—,‘,˜|Ó,×6Ñ6°q¸!Ó<ˆØ�y‰y˜Ó&×0Ñ0°°AÓ6ˆØ—‘˜\Ó*×4Ñ4°Q¸Ó:ˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð —n‘nØ—J‘JØ#Ÿ}š}‘C°$·,±,ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m KÓ0ˆØ˜LÐ(Ð(rn   ©N)ro   rp   rq   rž   rÌ   rG   rú   r   r   rŸ   rø   r{   r|   s   @rm   rK   rK   U  sf   ø„ ÙGôBð. /3ñ%)à—|‘|ð%)ð Ÿ™ tÑ+ð%)ð Ð+Ñ,ð	%)ð
 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷%)rn   rK   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )rQ   c                 ó  •— t         ‰| �  «        || _        t        |j                     | _        t        j                  |j                  |j                  «      | _
        t        j                  |j                  |j                  «      | _        y r#  )r8   rÌ   r&   r	   Ú
hidden_actÚactivation_fnr   r  rR   Úintermediate_sizerS   rT   rÖ   s     €rm   rÌ   zKosmos2VisionMLP.__init__–  sd   ø€ Ü‰ÑÔØˆŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�rn   r§   r«   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r#  )rS   r'  rT   ©rf   r§   s     rm   rø   zKosmos2VisionMLP.forward�  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐrn   )ro   rp   rq   rÌ   rG   rú   rø   r{   r|   s   @rm   rQ   rQ   •  s$   ø„ ôKð U§\¡\ð °e·l±l÷ rn   rQ   c                   ó~   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  dee   dej                  fd„Z
ˆ xZS )r)   r&   c                 óD  •— t         ‰| �  «        |j                  | _        t	        |«      | _        t        j                  | j                  |j                  ¬«      | _	        t        |«      | _        t        j                  | j                  |j                  ¬«      | _        y ©N©Úeps)r8   rÌ   rR   r@   rK   Ú	self_attnr   Ú	LayerNormÚlayer_norm_epsÚlayer_norm1rQ   ÚmlpÚlayer_norm2rÖ   s     €rm   rÌ   z"Kosmos2VisionEncoderLayer.__init__¦  sm   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ/°Ó7ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÔÜ# FÓ+ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÕrn   r§   rÿ   r
  r«   c                 ó¸   — |}| j                  |«      } | j                  d||dœ|¤Ž\  }}||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N)r§   rÿ   r¡   )r3  r0  r5  r4  )rf   r§   rÿ   r
  Úresidualrô   s         rm   rø   z!Kosmos2VisionEncoderLayer.forward®  s‚   € ð !ˆà×(Ñ(¨Ó7ˆØ)˜4Ÿ>™>ð 
Ø'Ø)ñ
ð ñ
Ñˆ�qð
 ! =Ñ0ˆà ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆØ  =Ñ0ˆàÐrn   )ro   rp   rq   r#   rÌ   rG   rú   r   r   r    rø   r{   r|   s   @rm   r)   r)   ¥  sQ   ø„ ðSÐ2õ Sðà—|‘|ðð Ÿ™ðð Ð+Ñ,ð	ð
 
×	Ñ	÷rn   r)   c                   óf   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej                  dz  dee	   de
ez  fd„Zˆ xZS )
ÚKosmos2VisionEncoderz¿
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`Kosmos2VisionEncoderLayer`].

    Args:
        config: Kosmos2VisionConfig
    r&   c                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w ©NF)
r8   rÌ   r&   r   Ú
ModuleListÚrangerL   r)   ÚlayersÚgradient_checkpointing)rf   r&   rô   rl   s      €rm   rÌ   zKosmos2VisionEncoder.__init__Ï  sQ   ø€ Ü‰ÑÔØˆŒÜ—m‘mÔPUÐV\×VnÑVnÓPoÖ$pÈ1Ô%>¸vÕ%FÒ$pÓqˆŒØ&+ˆÕ#ùò %qs   ½A#Nrÿ   r
  r«   c                 óT   — |}| j                   D ]  } |||fi |¤Ž}Œ t        |¬«      S )a7  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
        )r¥   )r>  rœ   )rf   Úinputs_embedsrÿ   r
  r§   Úencoder_layers         rm   rø   zKosmos2VisionEncoder.forwardÕ  sH   € ð( &ˆØ!Ÿ[™[ò 	ˆMÙ)ØØñð ñ‰Mð	ô 7Ø+ô
ð 	
rn   r#  )ro   rp   rq   rž   r#   rÌ   rG   rú   r   r   rŸ   r   rø   r{   r|   s   @rm   r9  r9  Æ  sQ   ø„ ñð,Ð2õ ,ð /3ñ
ð Ÿ™ tÑ+ð
ð Ð+Ñ,ð	
ð
 
�Ñ	 ÷
rn   r9  c                   ó˜   ‡ — e Zd ZeedœZdefˆ fd„Ze e	d¬«      e
	 	 ddej                  dz  ded	ee   d
efd„«       «       «       Zˆ xZS )ÚKosmos2VisionTransformer)r§   r¨   r&   c                 ó4  •— t         ‰| �  |«       |j                  }t        |«      | _        t        j                  ||j                  ¬«      | _        t        |«      | _
        t        j                  ||j                  ¬«      | _        | j                  «        y r-  )r8   rÌ   rR   r<   r×   r   r1  r2  Úpre_layrnormr9  ÚencoderÚpost_layernormÚ	post_init)rf   r&   r@   rl   s      €rm   rÌ   z!Kosmos2VisionTransformer.__init__ý  so   ø€ Ü‰Ñ˜Ô Ø×&Ñ&ˆ	ä1°&Ó9ˆŒÜŸL™L¨¸×8MÑ8MÔNˆÔÜ+¨FÓ3ˆŒÜ Ÿl™l¨9¸&×:OÑ:OÔPˆÔà�‰Õrn   F)Útie_last_hidden_statesNrë   rê   r
  r«   c                 óê   — |€t        d«      ‚| j                  ||¬«      }| j                  |«      } | j                  dd|i|¤Ž}|d   }|d d …dd d …f   }| j	                  |«      }t        ||¬«      S )Nz You have to specify pixel_values)rê   rA  r   )r¥   Úpooler_outputr¡   )rð   r×   rF  rG  rH  r   )rf   rë   rê   r
  r§   Úencoder_outputsr¥   Úpooled_outputs           rm   rø   z Kosmos2VisionTransformer.forward  s™   € ð ÐÜÐ?Ó@Ð@àŸ™¨ÐOg˜ÓhˆØ×)Ñ)¨-Ó8ˆà&˜$Ÿ,™,ñ 
Ø'ð
àñ
ˆð
 ,¨AÑ.ÐØ)ª!¨Q²¨'Ñ2ˆØ×+Ñ+¨MÓ:ˆä)Ø/Ø'ô
ð 	
rn   r;  )ro   rp   rq   r)   rK   Ú_can_record_outputsr#   rÌ   r   r   r   rG   r    r…   r   r   r   rø   r{   r|   s   @rm   rD  rD  ÷  s†   ø„ à2Ø,ñÐð
	Ð2õ 	ð  Ù¨EÔ2Øð 26Ø).ñ
à×'Ñ'¨$Ñ.ð
ð #'ð
ð Ð+Ñ,ð	
ð
 
$ò
ó ó 3ó  ô
rn   rD  c                   ó4  ‡ — e Zd ZdZddedededz  fˆ fd„Zddedededz  fd„Zeddedededz  fd	„«       Z e	j                  «       	 	 	 	 dd
e	j                  dz  de	j                  dz  dede	j                  dz  fd„«       Zed„ «       Zedd„«       Zˆ xZS )r`   zDThis module produces sinusoidal positional embeddings of any length.Nrb   rd   r^   c                 óš   •— t         ‰| �  «        d| _        || _        || _        || _        | j                  || j                  z   ||«       y )Nr7   )r8   rÌ   rc   rb   rd   r^   Úmake_weights)rf   rb   rd   r^   rl   s       €rm   rÌ   z1Kosmos2TextSinusoidalPositionalEmbedding.__init__+  sH   ø€ Ü‰ÑÔØˆŒØ*ˆÔØ*ˆÔØ&ˆÔØ×Ñ˜-¨$¯+©+Ñ5°}ÀkÕRrn   Únum_embeddingsc                 óà   — | j                  |||«      }t        | d«      r;|j                  | j                  j                  | j                  j
                  ¬«      }| j                  d|d¬«       y )Nre   r‘   FrÊ   )ra   r:   rƒ   re   r~   rŽ   rÕ   )rf   rS  rd   r^   rk   s        rm   rR  z5Kosmos2TextSinusoidalPositionalEmbedding.make_weights4  s[   € Ø×(Ñ(¨¸ÈÓTˆÜ�4˜Ô#à%Ÿ.™.¨t¯|©|×/AÑ/AÈ$Ï,É,×J]ÑJ]˜.Ó^ˆKà×Ñ˜Y¨ÀÐÕFrn   c                 óâ  — |dz  }t        j                  d«      |dz
  z  }t        j                  t        j                  |t        j
                  ¬«      j                  «       | z  «      }t        j                  | t        j
                  ¬«      j                  «       j                  d«      |j                  d«      z  }t        j                  t        j                  |«      t        j                  |«      gd¬«      j                  | d«      }|dz  dk(  r-t        j                  |t        j                  | d«      gd¬«      }|�	d||dd…f<   |j                  t        j                  «       «      S )	zÊ
        Build sinusoidal embeddings.

        This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
        "Attention Is All You Need".
        r7   i'  r    rï   r   r’   r5   N)ÚmathÚlogrG   ÚexprH   Úint64ÚfloatrÞ   r—   ÚsinÚcosr–   r˜   rƒ   Úget_default_dtype)rS  rd   r^   Úhalf_dimÚembs        rm   ra   z6Kosmos2TextSinusoidalPositionalEmbedding.get_embedding<  s  € ð ! AÑ%ˆÜ�h‰h�u‹o ¨A¡Ñ.ˆÜ�i‰iœŸ™ X´U·[±[ÔA×GÑGÓIÈSÈDÑPÓQˆÜ�l‰l˜>´·±Ô=×CÑCÓE×OÑOÐPQÓRÐUX×UbÑUbÐcdÓUeÑeˆÜ�i‰iœŸ™ 3›¬¯©°3«Ð8¸aÔ@×EÑEÀnÐVXÓYˆØ˜1Ñ Ò!ä—)‘)˜S¤%§+¡+¨n¸aÓ"@ÐAÀqÔIˆCØÐ"Ø"#ˆC�šQ�Ñà�v‰v”e×-Ñ-Ó/Ó0Ð0rn   Ú	input_idsrA  r�   rF   c                 ó˜  — |�L|j                  «       \  }}|€l| j                  || j                  |«      j                  |j                  «      }n5|j                  «       d d \  }}|€| j                  ||| j                  «      }| j                  dz   |z   |z   }|| j                  j                  d«      kD  r4| j                  || j                  z   | j                  | j                  «       | j                  j                  d|j                  d«      «      j                  ||| j                  j                  d   «      j                  «       S )Nr5   r    r   )r‚   Ú"create_position_ids_from_input_idsr^   rƒ   rŽ   Ú&create_position_ids_from_inputs_embedsre   rR  rc   rd   Úindex_selectr–   rI   Údetach)rf   r`  rA  r�   rF   rˆ   Úseq_lenÚmax_poss           rm   rø   z0Kosmos2TextSinusoidalPositionalEmbedding.forwardR  s8  € ð Ð Ø$Ÿ>™>Ó+‰LˆC�ØÐ#à#×FÑFØ˜t×/Ñ/Ð1Gó ç‘"�Y×%Ñ%Ó&ñ ð )×-Ñ-Ó/°°Ð4‰LˆC�ØÐ#Ø#×JÑJØ!Ð#9¸4×;KÑ;Kó �ð
 ×"Ñ" QÑ&¨Ñ0Ð3IÑIˆØ�T—\‘\×&Ñ& qÓ)Ò)Ø×Ñ˜g¨¯©Ñ3°T×5GÑ5GÈ×IYÑIYÔZà�|‰|×(Ñ(¨¨L×,=Ñ,=¸bÓ,AÓB×GÑGÈÈWÐVZ×VbÑVb×VhÑVhÐikÑVlÓm×tÑtÓvÐvrn   c                 ó  — | j                  «       dd }|d   }t        j                  |dz   ||z   dz   t        j                  | j                  ¬«      }|j                  d«      j                  |«      j                  «       |z   S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr5   r    r‘   r   )r‚   rG   rH   ÚlongrŽ   rÞ   rJ   r	  )rA  r�   r^   r  Úsequence_lengthrF   s         rm   rc  zOKosmos2TextSinusoidalPositionalEmbedding.create_position_ids_from_inputs_embedso  s   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a™.ˆä—|‘|Ø˜!‰O˜_¨{Ñ:¸QÑ>ÄeÇjÁjÐYf×YmÑYmô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<×GÑGÓIÐLbÑbÐbrn   c                 ó¾   — | j                  |«      j                  «       }t        j                  |d¬«      j	                  |«      |z   |z  }|j                  «       |z   S )a  
        Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
        are ignored. This is modified from fairseq's `utils.make_positions`.

        Args:
            x: torch.Tensor x:

        Returns: torch.Tensor
        r    r’   )Únerû   rG   ÚcumsumÚtype_asri  )r`  r^   r�   r}   Úincremental_indicess        rm   rb  zKKosmos2TextSinusoidalPositionalEmbedding.create_position_ids_from_input_ids‚  sW   € ð �|‰|˜KÓ(×,Ñ,Ó.ˆÜ$Ÿ|™|¨D°aÔ8×@Ñ@ÀÓFÐI_Ñ_ÐcgÑgÐØ"×'Ñ'Ó)¨KÑ7Ð7rn   r#  )NNr   N©r   )ro   rp   rq   rž   rû   rÌ   rR  Ústaticmethodra   rG   ry   rú   rø   rc  rb  r{   r|   s   @rm   r`   r`   '  s  ø„ ÙNñS cð S¸#ð SÈCÐRVÉJõ SñG¨3ð G¸sð GÐQTÐW[ÑQ[ó Gð ñ1 cð 1¸#ð 1ÈCÐRVÉJò 1ó ð1ð( €U‡]�]ƒ_ð *.Ø-1Ø&'Ø,0ñwà—<‘< $Ñ&ðwð —|‘| dÑ*ðwð !$ð	wð
 —l‘l TÑ)òwó ðwð8 ñcó ðcð" ò8ó ô8rn   r`   c                   ó  ‡ — e Zd ZdZ	 	 	 	 	 ddededededz  dedz  dedz  d	edz  fˆ fd
„Z	 	 	 ddej                  dej                  dz  de
dz  dej                  dz  deej                  ej                  dz  e
dz  f   f
d„Zˆ xZS )rU   r  Nr@   r  r  Ú
is_decoderÚadd_inner_attn_layernormrÉ   Ú	layer_idxc	                 óx  •— t         ‰	| �  «        || _        || _        || _        || _        ||z  | _        d| _        | j                  |z  | j                  k7  rt        d| j                  › d|› d�«      ‚| j                  dz  | _	        || _
        || _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        d | _        |r't        j&                  ||j(                  ¬«      | _        y y )NTr  r  rî   r2   )rÉ   r.  )r8   rÌ   r&   r@   r  r  r  r  rð   r   rs  ru  r   r  rN   rO   rM   rP   Úinner_attn_lnr1  r2  )
rf   r&   r@   r  r  rs  rt  rÉ   ru  rl   s
            €rm   rÌ   zKosmosTextAttention.__init__˜  s  ø€ ô 	‰ÑÔØˆŒØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØˆŒà�M‰M˜IÑ%¨$¯.©.Ò8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÑ*ˆŒØ$ˆŒØ"ˆŒä—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜŸ	™	 )¨Y¸TÔBˆŒð "ˆÔÙ#Ü!#§¡¨i¸V×=RÑ=RÔ!SˆDÕð $rn   r§   Úencoder_hidden_statesr¦   rÿ   r«   c                 ó  — |du}|j                   dd }g |¢d‘| j                  ‘­}| j                  |«      }	|	j                  |«      j	                  dd«      }	d}
|�St        |t        «      rA|j                  j                  | j                  «      }
|r|j                  }n|j                  }n|}|r|n|}|rK|�I|
rGj                  | j                     j                  }|j                  | j                     j                  }nÍg |j                   dd ¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j!                  |«      j                  |«      j	                  dd«      }|�Kj#                  ||| j                  «      \  }}|r)t        |t        «      rd|j                  | j                  <   t%        j&                  | j(                  j*                  t,        «      } || |	|||f| j.                  sdn| j0                  | j2                  dœ|¤Ž\  }} |j4                  g |¢d‘­Ž j7                  «       }| j8                  �| j9                  |«      }| j;                  |«      }||fS )	r  Nr5   r    r7   FTr1   )r  r   )rI   r  rM   r–   rò   r;   r   Ú
is_updatedÚgetru  Úcross_attention_cacheÚself_attention_cacher>  rº   r!  rN   rO   Úupdater   r  r&   r  r  r  r  r   rá   r	  rw  rP   )rf   r§   rx  r¦   rÿ   r
  Úis_cross_attentionr  r  Úquery_statesrz  Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shaper"  r  r  s                      rm   rø   zKosmosTextAttention.forward¾  s{  € ð 3¸$Ð>ÐØ#×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØ—{‘{ =Ó1ˆØ#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆàˆ
ØÐ&Ü˜/Ô+>Ô?Ø,×7Ñ7×;Ñ;¸D¿N¹NÓK�
Ù%à+:×+PÑ+PÑ(à+:×+OÑ+OÑ(à'6Ð$á2DÑ.È-ˆÙ /Ð"=Á*à-×4Ñ4°T·^±^ÑD×IÑIˆJØ/×6Ñ6°t·~±~ÑF×MÑM‰LàF˜×-Ñ-¨c¨rÐ2ÐF°BÐF¸¿¹ÑFˆHØŸ™ ^Ó4×9Ñ9¸(ÓC×MÑMÈaÐQRÓSˆJØŸ;™; ~Ó6×;Ñ;¸HÓE×OÑOÐPQÐSTÓUˆLàÐ*à+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜Lá%¬*°_ÔFYÔ*ZØAE�O×.Ñ.¨t¯~©~Ñ>ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$·,±,Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ×ÑÐ)Ø×,Ñ,¨[Ó9ˆKà—m‘m KÓ0ˆà˜LÐ(Ð(rn   )r1   FFTN)NNN)ro   rp   rq   rž   rû   rZ  r…   rÌ   rG   rú   r
   rŸ   rø   r{   r|   s   @rm   rU   rU   ”  sø   ø„ ÙGð Ø"'Ø05Ø Ø!%ñ$Tð ð$Tð ð	$Tð
 ð$Tð ˜4‘Kð$Tð #'¨¡+ð$Tð �T‰kð$Tð ˜$‘;õ$TðR 6:Ø(,Ø.2ñE)à—|‘|ðE)ð  %Ÿ|™|¨dÑ2ðE)ð  ™ð	E)ð
 Ÿ™ tÑ+ðE)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸$±,Ð>Ñ	?÷E)rn   rU   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )rV   r&   c                 ó²  •— t         ‰| �  «        |j                  | _        t        |j                     | _        |j                  | _        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        y r-  )r8   rÌ   r  r	   Úactivation_functionr'  Úactivation_dropoutr   r  r@   Úffn_dimrS   rT   r1  r2  Úffn_layernormrÖ   s     €rm   rÌ   zKosmos2TextFFN.__init__  s�   ø€ Ü‰ÑÔà—~‘~ˆŒÜ# F×$>Ñ$>Ñ?ˆÔØ"(×";Ñ";ˆÔä—9‘9˜V×-Ñ-¨v¯~©~Ó>ˆŒÜ—9‘9˜VŸ^™^¨V×-=Ñ-=Ó>ˆŒäŸ\™\¨&¯.©.¸f×>SÑ>SÔTˆÕrn   c                 ób  — | j                  | j                  |«      «      }t        j                  j	                  || j
                  | j                  ¬«      }| j                  |«      }| j                  |«      }t        j                  j	                  || j                  | j                  ¬«      }|S )Nr  )	r'  rS   r   rã   r  r‰  r  r‹  rT   r*  s     rm   rø   zKosmos2TextFFN.forward  s�   € Ø×*Ñ*¨4¯8©8°MÓ+BÓCˆÜŸ™×-Ñ-¨m¸t×?VÑ?VÐae×anÑanÐ-ÓoˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆÜŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-ÓdˆàÐrn   )ro   rp   rq   r"   rÌ   rø   r{   r|   s   @rm   rV   rV     s   ø„ ð
UÐ0õ 
Uörn   rV   c                   ó  ‡ — e Zd Zddefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  dedz  d	edz  d
e	ej                  e	ej                  ej                  f   dz  f   fd„Zˆ xZS )r*   Nr&   c           	      ó’  •— t         ‰| �  «        |j                  | _        t        || j                  |j                  |j
                  dd|¬«      | _        |j                  | _        t        j                  | j                  |j                  ¬«      | _        |j                  ret        || j                  |j                  |j
                  dd|¬«      | _        t        j                  | j                  |j                  ¬«      | _        t        |«      | _        t        j                  | j                  |j                  ¬«      | _        y )NT)r@   r  r  rs  rt  ru  r.  F)r8   rÌ   r@   rU   Úattention_headsr  r0  r  r   r1  r2  Úself_attn_layer_normÚadd_cross_attentionÚencoder_attnÚencoder_attn_layer_normrV   ÚffnÚfinal_layer_norm)rf   r&   ru  rl   s      €rm   rÌ   zKosmos2TextBlock.__init__  sù   ø€ Ü‰ÑÔØ×)Ñ)ˆŒä,ØØ—n‘nØ×,Ñ,Ø×,Ñ,ØØ%)Øô
ˆŒð —~‘~ˆŒÜ$&§L¡L°·±ÀV×EZÑEZÔ$[ˆÔ!à×%Ò%Ü 3ØØŸ.™.Ø ×0Ñ0Ø×0Ñ0ØØ).Ø#ô!ˆDÔô ,.¯<©<¸¿¹ÈF×LaÑLaÔ+bˆDÔ(ä! &Ó)ˆŒÜ "§¡¨T¯^©^À×AVÑAVÔ WˆÕrn   r§   rÿ   rx  Úencoder_attention_maskr¦   Úoutput_attentionsr«   c           	      ó4  — |}| j                  |«      } | j                  d||||dœ|¤Ž\  }}	t        j                  j	                  || j                  | j
                  ¬«      }||z   }|�„t        | d«      st        d| › d�«      ‚|}| j                  |«      } | j                  d|||||dœ|¤Ž\  }}	t        j                  j	                  || j                  | j
                  ¬«      }||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N)r§   r¦   rÿ   r—  r  r’  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)r§   rx  rÿ   r¦   r—  r¡   )r�  r0  r   rã   r  r  r:   rð   r“  r’  r•  r”  )
rf   r§   rÿ   rx  r–  r¦   r—  r
  r7  rô   s
             rm   rø   zKosmos2TextBlock.forward=  sc  € ð !ˆØ×1Ñ1°-Ó@ˆà)˜4Ÿ>™>ð 
Ø'Ø+Ø)Ø/ñ	
ð
 ñ
Ñˆ�qô Ÿ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ  =Ñ0ˆð !Ð,Ü˜4 Ô0Ü Ø=¸d¸Vð DDð Dóð ð
 %ˆHØ ×8Ñ8¸ÓGˆMà0˜t×0Ñ0ð  Ø+Ø&;Ø5Ø /Ø"3ñ ð ñ ÑˆM˜1ô ŸM™M×1Ñ1°-À4Ç<Á<ÐZ^×ZgÑZgÐ1ÓhˆMØ$ }Ñ4ˆMð !ˆà×-Ñ-¨mÓ<ˆð Ÿ™ Ó/ˆØ  =Ñ0ˆàÐrn   r#  )NNNNF)ro   rp   rq   r"   rÌ   rG   rú   r
   r…   rŸ   r    rø   r{   r|   s   @rm   r*   r*     sÀ   ø„ ñXÐ0õ XðD /3Ø59Ø6:Ø(,Ø).ñ6à—|‘|ð6ð Ÿ™ tÑ+ð6ð  %Ÿ|™|¨dÑ2ð	6ð
 !&§¡¨tÑ 3ð6ð  ™ð6ð   $™;ð6ð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷6rn   r*   c            #       ór  ‡ — e Zd ZU eed<   dZe eedd¬«       eedd¬«      dœZ	defˆ fd„Z
d	„ Z	 	 	 	 	 ddej                  d
z  dej                  d
z  dej                  d
z  dedej                  d
z  f
d„Zeee	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  d
z  dej                  d
z  dej                  d
z  dej                  d
z  dej                  d
z  dej                  d
z  ded
z  dej                  d
z  dej                  d
z  ded
z  ded
z  ded
z  ded
z  dee   deez  fd„«       «       «       Zˆ xZS ) r\   r&   ©r(   r    r0  )ÚindexÚ
layer_namer’  )r§   r¨   Úcross_attentionsc           	      óÆ  •— t         ‰| �  |«       |j                  | _        |j                  | _        |j                  rt        j                  |j                  «      nd| _        t        j                  |j                  |j                  |j                  ¬«      | _        t        |j                  |j                  |j                  ¬«      | _        t        j"                  t%        |j&                  «      D �cg c]  }t)        ||¬«      ‘Œ c}«      | _        t        j*                  |j                  |j,                  «      | _        d| _        | j3                  «        y c c}w )Nr�   )r^   )rb   rd   r^   )ru  F)r8   rÌ   r  Ú	layerdropÚscale_embeddingrV  Úsqrtr@   Úembed_scaler   rÔ   Ú
vocab_sizeÚpad_token_idr]   r`   Úmax_position_embeddingsÚembed_positionsr<  r=  r>  r*   r1  r2  Ú
layer_normr?  rI  )rf   r&   Úirl   s      €rm   rÌ   zKosmos2TextTransformer.__init__  s  ø€ Ü‰Ñ˜Ô Ø—~‘~ˆŒØ×)Ñ)ˆŒà:@×:PÒ:Pœ4Ÿ9™9 V×%5Ñ%5Ô6ÐVYˆÔÜŸL™L¨×):Ñ):¸F×<LÑ<LÐZ`×ZmÑZmÔnˆÔäGØ ×8Ñ8Ø ×*Ñ*Ø×+Ñ+ô 
ˆÔô —m‘mÔTYÐZ`×ZgÑZgÓThÖ$iÈqÔ%5°fÈÖ%JÒ$iÓjˆŒÜŸ,™, v×'7Ñ'7¸×9NÑ9NÓOˆŒà&+ˆÔ#à�‰Õùò %js   Ã7Ec                 óÞ   — d }|d   dkD  r#t        ||j                  |j                  |¬«      }|�=t        ||j                  |d   ¬«      j	                  |j                  «      }|€|n||z   }|S )Nr5   r    )rŽ   r�   ©r   )rš   r~   rŽ   rŒ   rƒ   )rf   rÿ   r  rA  r�   Úcombined_attention_maskÚexpanded_attn_masks          rm   Ú_prepare_decoder_attention_maskz6Kosmos2TextTransformer._prepare_decoder_attention_mask”  s’   € ð #'ÐØ�r‰?˜QÒÜ&7ØØ×#Ñ#Ø$×+Ñ+Ø'=ô	'Ð#ð Ð%ä!-¨n¸m×>QÑ>QÐ[fÐgiÑ[jÔ!k×!nÑ!nØ×$Ñ$ó"Ðð '>Ð&EÑ"ÐK]Ð`wÑKwð $ð 'Ð&rn   NrA  r©   Úimg_input_maskr�   rF   c                 óÚ  — |€| j                  |«      }|�[|j                  |j                  «      j                  d|j	                  d«      «      ||j                  t
        j                  ¬«      <   || j                  z  }| j                  ||||¬«      }|j                  |j                  «      }||z   }t        j                  j                  || j                  | j                  ¬«      }|S )Nr5   rï   )r`  rA  r�   rF   r  )r]   rƒ   rŽ   r–   r‚   rG   r…   r¢  r¦  r   rã   r  r  )	rf   r`  rA  r©   r®  r�   rF   Ú	positionsr§   s	            rm   Úforward_embeddingz(Kosmos2TextTransformer.forward_embedding«  sé   € ð Ð Ø ×-Ñ-¨iÓ8ˆMàÐ#ØAMÇÁÐQ^×QeÑQeÓAf×AkÑAkØ�L×%Ñ% bÓ)óBˆM˜.×+Ñ+´%·*±*Ð+Ó=Ñ>ð &¨×(8Ñ(8Ñ8ˆð ×(Ñ(ØØ'Ø#9Ø%ð	 )ó 
ˆ	ð —L‘L ×!5Ñ!5Ó6ˆ	à%¨	Ñ1ˆäŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-ÓdˆàÐrn   r`  rÿ   Úimage_embeds_position_maskrx  r–  r¦   Ú	use_cacher—  Úoutput_hidden_statesÚreturn_dictr
  r«   c           	      ó�  — |�|�t        d«      ‚|�"|j                  }|j                  d|d   «      }n!|�|j                  «       dd }nt        d«      ‚|
rd|€b|€| j                  j
                  r4t        t        | j                  ¬«      t        | j                  ¬«      «      nt        | j                  ¬«      }|�|j                  «       nd}|dkD  rd}d}| j                  ||||||	¬«      }| j                  ||||«      }|�|�t        ||j                  |d   ¬«      }t        j                  j                  || j                  | j                   ¬	«      }| j"                  D ]C  }| j                   r%t%        j&                  g «      }|| j(                  k  rŒ4 ||||f||||
d
œ|¤Ž}ŒE | j+                  |«      }t-        ||¬«      S )áN  
        image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`.
        image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0,
            1]`:

            - 1 for places where to put the image features,
            - 0 for places that are not for image features (i.e. for text tokens).
        NzDYou cannot specify both input_ids and inputs_embeds at the same timer5   z5You have to specify either input_ids or inputs_embeds)r&   r   )r`  rA  r©   r®  r�   rF   rª  r  )r–  r¦   r—  r³  )r¥   r¦   )rð   rI   r–   r‚   r&   Úis_encoder_decoderr   r   Úget_seq_lengthr±  r­  rŒ   r~   r   rã   r  r  r>  rG   ÚrandrŸ  r§  r   )rf   r`  rÿ   r©   r²  rx  r–  r¦   rA  rF   r³  r—  r´  rµ  r
  r  r�   r§   Údecoder_layerÚdropout_probabilitys                       rm   rø   zKosmos2TextTransformer.forwardÎ  s  € ð< Ð  ]Ð%>ÜÐcÓdÐdØÐ"Ø#Ÿ/™/ˆKØ!Ÿ™ r¨;°r©?Ó;‰IØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUá˜Ð0ð )Ð4¸¿¹×8VÒ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð FUÐE` ×!?Ñ!?Ô!AÐfgÐð " AÒ%ØˆLØ)-Ð&à×.Ñ.ØØ'Ø%Ø5Ø#9Ø%ð /ó 
ˆð ×=Ñ=Ø˜K¨Ð8Nó
ˆð
 !Ð,Ð1GÐ1Sä%1Ð2HÈ-×J]ÑJ]ÐgrÐsuÑgvÔ%wÐ"äŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-Ódˆà!Ÿ[™[ò 	ˆMà�}Š}Ü&+§j¡j°£nÐ#Ø&¨¯©Ò7Øá)ØØØ%ð	ð (>Ø /Ø"3Ø#ñ	ð ñ	‰Mð	ð& Ÿ™¨Ó6ˆä8Ø+Ø+ô
ð 	
rn   )NNNr   N)NNNNNNNNNNNNN)ro   rp   rq   r"   rr   rs   r*   r   rU   rO  rÌ   r­  rG   rú   rû   r±  r   r   r   r
   r…   r   r   rŸ   r   rø   r{   r|   s   @rm   r\   r\   v  s  ø… ØÓØ Ðà)Ù$Ð%8ÀÈkÔZÙ*Ð+>ÀaÐTbÔcñÐðÐ0õ ò*'ð4 .2Ø,0Ø.2Ø&'Ø,0ñ!ð —|‘| dÑ*ð!ð —l‘l TÑ)ð	!ð
 Ÿ™ tÑ+ð!ð !$ð!ð —l‘l TÑ)ó!ðF  ØØð *.Ø.2Ø,0Ø:>Ø59Ø6:Ø(,Ø-1Ø,0Ø!%Ø)-Ø,0Ø#'ñ_
à—<‘< $Ñ&ð_
ð Ÿ™ tÑ+ð_
ð —l‘l TÑ)ð	_
ð
 %*§L¡L°4Ñ$7ð_
ð  %Ÿ|™|¨dÑ2ð_
ð !&§¡¨tÑ 3ð_
ð  ™ð_
ð —|‘| dÑ*ð_
ð —l‘l TÑ)ð_
ð ˜$‘;ð_
ð   $™;ð_
ð # T™kð_
ð ˜D‘[ð_
ð Ð-Ñ.ð_
ð  
Ð:Ñ	:ò!_
ó ó ó  ô_
rn   r\   c                   ó°   ‡ — e Zd ZU eed<   dZdZdefˆ fd„Zdej                  fd„Z
ee	 	 ddej                  dz  ded	ee   deez  fd
„«       «       Zˆ xZS )ÚKosmos2VisionModelr&   rë   )r'   c                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y r#  )r8   rÌ   rD  ÚmodelrI  rÖ   s     €rm   rÌ   zKosmos2VisionModel.__init__8  s&   ø€ Ü‰Ñ˜Ô Ü-¨fÓ5ˆŒ
à�‰Õrn   r«   c                 óB   — | j                   j                  j                  S r#  )rÀ  r×   rA   r»   s    rm   Úget_input_embeddingsz'Kosmos2VisionModel.get_input_embeddings>  s   € Ø�z‰z×$Ñ$×4Ñ4Ð4rn   Nrê   r
  c                 ó,   —  | j                   d||dœ|¤ŽS )N)rë   rê   r¡   ©rÀ  )rf   rë   rê   r
  s       rm   rø   zKosmos2VisionModel.forwardA  s,   € ð ˆt�z‰zð 
Ø%Ø%=ñ
ð ñ
ð 	
rn   r;  )ro   rp   rq   r#   rr   Úmain_input_namers   rÌ   r   rz   rÂ  r   r   rG   r    r…   r   r   rŸ   rœ   rø   r{   r|   s   @rm   r¾  r¾  3  s’   ø… ØÓØ$€OØ!ÐðÐ2õ ð5 b§i¡ió 5ð Øð 26Ø).ñ

à×'Ñ'¨$Ñ.ð

ð #'ð

ð Ð+Ñ,ð	

ð
 
Ð8Ñ	8ò

ó ó ô

rn   r¾  c                   óž  ‡ — e Zd ZU eed<   dZdefˆ fd„Zdej                  fd„Z	e
e	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  dej                  dz  dedz  dee   deez  fd„«       «       Zˆ xZS )ÚKosmos2TextModelr&   rš  c                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y r#  )r8   rÌ   r\   rÀ  rI  rÖ   s     €rm   rÌ   zKosmos2TextModel.__init__T  s&   ø€ Ü‰Ñ˜Ô Ü+¨FÓ3ˆŒ
à�‰Õrn   r«   c                 ó.   — | j                   j                  S r#  ©rÀ  r]   r»   s    rm   rÂ  z%Kosmos2TextModel.get_input_embeddingsZ  ó   € Ø�z‰z×&Ñ&Ð&rn   Nr`  rÿ   r©   r²  rx  r–  r¦   rA  rF   r³  r
  c                 ó<   —  | j                   d|||||||||	|
dœ
|¤ŽS )r·  ©
r`  rÿ   r©   r²  rx  r–  r¦   rA  rF   r³  r¡   rÄ  )rf   r`  rÿ   r©   r²  rx  r–  r¦   rA  rF   r³  r
  s               rm   rø   zKosmos2TextModel.forward]  sD   € ð4 ˆt�z‰zð 
ØØ)Ø%Ø'AØ"7Ø#9Ø+Ø'Ø%Øñ
ð ñ
ð 	
rn   )
NNNNNNNNNN)ro   rp   rq   r"   rr   rs   rÌ   r   rz   rÂ  r   r   rG   rú   r
   r…   r   r   rŸ   r   rø   r{   r|   s   @rm   rÇ  rÇ  P  s:  ø… ØÓØ ÐðÐ0õ ð' b§i¡ió 'ð Øð *.Ø.2Ø,0Ø:>Ø59Ø6:Ø(,Ø-1Ø,0Ø!%ñ$
à—<‘< $Ñ&ð$
ð Ÿ™ tÑ+ð$
ð —l‘l TÑ)ð	$
ð
 %*§L¡L°4Ñ$7ð$
ð  %Ÿ|™|¨dÑ2ð$
ð !&§¡¨tÑ 3ð$
ð  ™ð$
ð —|‘| dÑ*ð$
ð —l‘l TÑ)ð$
ð ˜$‘;ð$
ð Ð+Ñ,ð$
ð 
Ð:Ñ	:ò$
ó ó ô$
rn   rÇ  z‰
    The text model from KOSMOS-2 with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    c                    ó  ‡ — e Zd ZU eed<   ddiZdefˆ fd„Zdej                  fd„Z	dej                  fd„Z
ee	 	 	 	 	 	 	 	 	 	 	 	 dd	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  dej                  dz  dej                   dz  dedz  deej                  z  dee   deez  fd„«       «       Z	 	 	 	 	 	 	 dˆ fd„	Zˆ xZS )rW   r&   zlm_head.weightzmodel.embed_tokens.weightc                 óÆ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        | j                  «        y )NF)Úin_featuresÚout_featuresrÉ   )
r8   rÌ   r\   rÀ  r   r  r@   r£  rX   rI  rÖ   s     €rm   rÌ   zKosmos2TextForCausalLM.__init__�  sI   ø€ Ü‰Ñ˜Ô ä+¨FÓ3ˆŒ
Ü—y‘y¨V×-=Ñ-=ÈF×L]ÑL]ÐdiÔjˆŒð 	�‰Õrn   r«   c                 ó.   — | j                   j                  S r#  rÊ  r»   s    rm   rÂ  z+Kosmos2TextForCausalLM.get_input_embeddings™  rË  rn   c                 ó   — | j                   S r#  )rX   r»   s    rm   Úget_output_embeddingsz,Kosmos2TextForCausalLM.get_output_embeddingsœ  s   € Ø�|‰|Ðrn   Nr`  rÿ   r©   r²  rx  r–  r¦   rA  rF   Úlabelsr³  Úlogits_to_keepr
  c                 óÌ  — |
�|rt         j                  d«       d} | j                  d|||||||||	|dœ
|¤Ž}|j                  }t	        |t
        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }d}|
�* | j                  d||
| j                  j                  dœ|¤Ž}t        |||j                  |j                  |j                  |j                  ¬«      S )a  
        image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`.
        image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0,
            1]`:

            - 1 for places where to put the image features,
            - 0 for places that are not for image features (i.e. for text tokens).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        NzJThe `use_cache` argument is changed to `False` since `labels` is provided.FrÍ  )rÀ   rÕ  r£  )r¿   rÀ   r¦   r§   r¨   r�  r¡   )ÚloggerÚwarningrÀ  r¥   r;   rû   ÚslicerX   Úloss_functionr&   r£  r   r¦   r§   r¨   r�  )rf   r`  rÿ   r©   r²  rx  r–  r¦   rA  rF   rÕ  r³  rÖ  r
  Úoutputsr§   Úslice_indicesrÀ   r¿   s                      rm   rø   zKosmos2TextForCausalLM.forwardŸ  s
  € ð@ ÐÙÜ—‘ÐkÔlØˆIà=G¸T¿Z¹Zð >
ØØ)Ø%Ø'AØ"7Ø#9Ø+Ø'Ø%Øñ>
ð ñ>
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
rn   c	                 ó€  •— |s|rd }d }n‰|�‡|�|j                  «       d d n|j                  «       \  }
}|j                  «       d   }t        j                  |t        j                  |
||z
  ft        j                  |j
                  ¬«      fd¬«      }t        ‰| �  |f|||||||dœ|	¤Ž}|j                  dd «       |S )Nr5   )r‚   r~   rŽ   r    r’   )r¦   rÿ   r©   r²  rA  r³  Úis_first_iterationrF   )	r‚   rG   r—   r˜   r…   rŽ   r8   Úprepare_inputs_for_generationÚpop)rf   r`  r©   r²  r¦   rÿ   rA  r³  rß  Úmodel_kwargsró   rf  Úmask_lenÚmodel_inputsrl   s                 €rm   rà  z4Kosmos2TextForCausalLM.prepare_inputs_for_generationä  sì   ø€ ñ$ "¡iØˆLØ)-Ñ&ð (Ð3Ø?LÐ?X -×"4Ñ"4Ó"6°s¸Ñ";Ð^l×^qÑ^qÓ^sÑˆJ˜Ø1×6Ñ6Ó8¸Ñ<ˆHÜ).¯©à.Ü—K‘K j°'¸HÑ2DÐ%EÌUÏZÉZÐ`i×`pÑ`pÔqðð ô*Ð&ô ‘wÑ<Øð

à+Ø)Ø%Ø'AØ'ØØ1ñ

ð ñ

ˆð 	×Ñ˜¨Ô.àÐrn   )NNNNNNNNNNNr   )NNNNNNF)ro   rp   rq   r"   rr   Ú_tied_weights_keysrÌ   r   rz   rÂ  rÔ  r   r   rG   rú   r
   Ú
LongTensorr…   rû   r   r   rŸ   r   rø   rà  r{   r|   s   @rm   rW   rW   †  s²  ø… ð ÓØ*Ð,GÐHÐðÐ0õ ð' b§i¡ió 'ð r§y¡yó ð Øð *.Ø.2Ø,0Ø:>Ø59Ø6:Ø(,Ø-1Ø,0Ø*.Ø!%Ø-.ñA
à—<‘< $Ñ&ðA
ð Ÿ™ tÑ+ðA
ð —l‘l TÑ)ð	A
ð
 %*§L¡L°4Ñ$7ðA
ð  %Ÿ|™|¨dÑ2ðA
ð !&§¡¨tÑ 3ðA
ð  ™ðA
ð —|‘| dÑ*ðA
ð —l‘l TÑ)ðA
ð × Ñ  4Ñ'ðA
ð ˜$‘;ðA
ð ˜eŸl™lÑ*ðA
ð Ð+Ñ,ðA
ð 
Ð2Ñ	2òA
ó ó ðA
ðL Ø#'ØØØØØ ÷0ñ 0rn   rW   c                   ó.   ‡ — e Zd ZdZdefˆ fd„Zd„ Zˆ xZS )rY   zmThe layer that transforms the image model's output to part of the text model's input (namely, image features)r&   c                 óø  •— t         ‰| �  «        t        j                  |j                  j
                  |j                  j                  «      | _        t        j                  t        j                  |j                  |j                  j                  «      «      | _        t        |j                  |j                  j                  |j                  j                  |j                  j                   dd¬«      | _        y )NF)r  rs  rt  )r8   rÌ   r   r  r.   rR   r0   r@   rZ   rÏ   rG   rÐ   Úlatent_query_numr[   rU   r�  r  Úx_attnrÖ   s     €rm   rÌ   z%Kosmos2ImageToTextProjection.__init__  s®   ø€ Ü‰ÑÔÜ—Y‘Y˜v×3Ñ3×?Ñ?À×ASÑAS×A]ÑA]Ó^ˆŒ
ÜŸL™L¬¯©°V×5LÑ5LÈf×N`ÑN`×NjÑNjÓ)kÓlˆÔä)Ø×ÑØ×Ñ×(Ñ(Ø×Ñ×.Ñ.Ø×&Ñ&×8Ñ8ØØ%*ô
ˆ�rn   c                 ó  — | j                  |«      }| j                  j                  d«      j                  |j	                  d«      dd«      }t        j                  ||gd¬«      }| j                  ||d d d ¬«      \  }}||fS )Nr   r5   r    r’   )r§   rx  r¦   rÿ   r—  )rZ   r[   rÞ   rJ   r‚   rG   r—   rê  )rf   Úfeaturesr§   r[   Úkey_value_statesr  s         rm   rø   z$Kosmos2ImageToTextProjection.forward(  s�   € ØŸ
™
 8Ó,ˆð ×(Ñ(×2Ñ2°1Ó5×<Ñ<¸]×=OÑ=OÐPQÓ=RÐTVÐXZÓ[ˆÜ Ÿ9™9 m°\Ð%BÈÔJÐà&*§k¡kØ&Ø"2Ø ØØ"ð '2ó '
Ñ#ˆ�|ð ˜lÐ*Ð*rn   )ro   rp   rq   rž   r!   rÌ   rø   r{   r|   s   @rm   rY   rY     s   ø„ Ùwð
˜}õ 
ö+rn   rY   z}
    KOSMOS-2 Model for generating text and image features. The model consists of a vision encoder and a language model.
    c                   óà  ‡ — e Zd ZU eed<   dZdefˆ fd„Zdej                  fd„Z	d„ Z
ee	 ddej                  dedz  d	ee   deez  fd
„«       «       Zee	 	 	 	 	 	 	 	 	 	 ddej*                  dz  dej*                  dz  dej*                  dz  dej*                  dz  dedz  dej*                  dz  dej*                  dz  dej*                  dz  dedz  ded	ee   deez  fd„«       «       Zˆ xZS )ÚKosmos2Modelr&   rë   c                 óÌ   •— t         ‰| �  |«       t        |j                  «      | _        t        |j                  «      | _        t        |«      | _	        | j                  «        y r#  )r8   rÌ   rÇ  r0   Ú
text_modelr¾  r.   Úvision_modelrY   Úimage_to_text_projectionrI  rÖ   s     €rm   rÌ   zKosmos2Model.__init__C  sN   ø€ Ü‰Ñ˜Ô ä*¨6×+=Ñ+=Ó>ˆŒÜ.¨v×/CÑ/CÓDˆÔÜ(DÀVÓ(LˆÔ%ð 	�‰Õrn   r«   c                 óB   — | j                   j                  j                  S r#  ©rñ  rÀ  r]   r»   s    rm   rÂ  z!Kosmos2Model.get_input_embeddingsM  ó   € Ø�‰×$Ñ$×1Ñ1Ð1rn   c                 ó:   — || j                   j                  _        y r#  rõ  ©rf   rþ   s     rm   Úset_input_embeddingsz!Kosmos2Model.set_input_embeddingsP  ó   € Ø-2ˆ�‰×ÑÕ*rn   rê   Nr
  c                 óh  — d|v r,t        j                  dt        «       |j                  dd «        | j                  d||ddœ|¤Ž}| j                  j
                  j                  |d   «      }t        j                  j                  |d¬«      }| j                  |«      \  }}||_        ||_        |S )	NÚreturn_attentionsz¯`return_attentions` is deprecated and will be removed in a future version. Please use `return_dict` and access `projection_attentions` from the returned `ModelOutput` instead.T)rë   rê   rµ  r   r5   r’   r¡   )ÚwarningsÚwarnÚFutureWarningrá  rò  rÀ  rH  r   rã   Ú	normalizeró  rL  r�   )rf   rë   rê   r
  Úvision_outputr©   r�   s          rm   Úget_image_featureszKosmos2Model.get_image_featuresS  sÇ   € ð  &Ñ(Ü�M‰Mð_äôð
 �J‰JÐ*¨DÔ1àARÀ×ARÑARð B
Ø%Ø%=ØñB
ð ñ	B
ˆð ×(Ñ(×.Ñ.×=Ñ=¸mÈAÑ>NÓOˆä—}‘}×.Ñ.¨|ÀÐ.ÓDˆØ.2×.KÑ.KÈLÓ.YÑ+ˆÐ+Ø&2ˆÔ#Ø.CˆÔ+àÐrn   r`  r²  rÿ   r¦   r©   rA  rF   r³  c                 ó0  — d}d}|€;|€t        d«      ‚ | j                  |f|
ddœ|¤Ž}|j                  }|j                  } | j                  d||||||||	ddœ	|¤Ž}t        |j                  |j                  |j                  |j                  |||¬«      S )a–  
        image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0,
            1]`:

            - 1 for places where to put the image features,
            - 0 for places that are not for image features (i.e. for text tokens).
        image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`.

        Examples:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Kosmos2Model

        >>> model = Kosmos2Model.from_pretrained("microsoft/kosmos-2-patch14-224")
        >>> processor = AutoProcessor.from_pretrained("microsoft/kosmos-2-patch14-224")

        >>> url = "https://huggingface.co/microsoft/kosmos-2-patch14-224/resolve/main/snowman.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> text = (
        ...     "<grounding> An image of<phrase> a snowman</phrase><object><patch_index_0044><patch_index_0863>"
        ...     "</object> warming himself by<phrase> a fire</phrase><object><patch_index_0005><patch_index_0911>"
        ...     "</object>"
        ... )

        >>> inputs = processor(text=text, images=image, return_tensors="pt", add_eos_token=True)

        >>> last_hidden_state = model(
        ...     pixel_values=inputs["pixel_values"],
        ...     input_ids=inputs["input_ids"],
        ...     attention_mask=inputs["attention_mask"],
        ...     image_embeds_position_mask=inputs["image_embeds_position_mask"],
        ... ).last_hidden_state
        >>> list(last_hidden_state.shape)
        [1, 91, 2048]
        ```Nú<You have to specify either `pixel_values` or `image_embeds`.T)rê   rµ  )	r`  rÿ   r©   r²  r¦   rA  rF   r³  rµ  )r¥   r¦   r§   r¨   r©   r�   rª   r¡   )
rð   r  rL  r�   rñ  r¤   r¥   r¦   r§   r¨   )rf   rë   r`  r²  rÿ   r¦   r©   rA  rF   r³  rê   r
  rª   r�   Úimage_featuresrÜ  s                   rm   rø   zKosmos2Model.forwards  sà   € ðt #ÐØ $ÐØÐØÐ#Ü Ð!_Ó`Ð`Ø4˜T×4Ñ4ØðØ7OÐ]añØekñˆNð *×7Ñ7ˆLØ$2×$HÑ$HÐ!à!�$—/‘/ð 
ØØ)Ø%Ø'AØ+Ø'Ø%ØØñ
ð ñ
ˆô "Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø%Ø"7Ø 3ô
ð 	
rn   rù   )
NNNNNNNNNF)ro   rp   rq   r!   rr   rÅ  rÌ   r   rz   rÂ  rù  r   r   rG   r    r…   r   r   rŸ   rœ   r  rú   r
   r¤   rø   r{   r|   s   @rm   rï  rï  :  s¡  ø… ð ÓØ$€Oð˜}õ ð2 b§i¡ió 2ò3ð Øð 16ñà×'Ñ'ðð #'¨¡+ðð Ð+Ñ,ð	ð
 
Ð8Ñ	8òó ó ðð< Øð -1Ø)-Ø:>Ø.2Ø(,Ø,0Ø-1Ø,0Ø!%Ø).ñX
à—l‘l TÑ)ðX
ð —<‘< $Ñ&ðX
ð %*§L¡L°4Ñ$7ð	X
ð
 Ÿ™ tÑ+ðX
ð  ™ðX
ð —l‘l TÑ)ðX
ð —|‘| dÑ*ðX
ð —l‘l TÑ)ðX
ð ˜$‘;ðX
ð #'ðX
ð Ð+Ñ,ðX
ð 
Ð#Ñ	#òX
ó ó ôX
rn   rï  z�
    KOSMOS-2 Model for generating text and bounding boxes given an image. The model consists of a vision encoder and a
    language model.
    c                   óä  ‡ — e Zd ZU eed<   dZddiZdefˆ fd„Zdej                  fd„Z
d„ Zdej                  fd	„Zd
„ Zee	 	 	 	 	 	 	 	 	 	 	 ddej"                  dz  dej"                  dz  dej"                  dz  dej"                  dz  dedz  dej"                  dz  dej"                  dz  dej"                  dz  dej&                  dz  dedz  deej"                  z  dee   deez  fd„«       «       Z ej6                  «       	 	 	 	 	 	 ddej"                  dz  dej"                  dz  dej"                  dz  dej"                  dz  dej"                  dz  dej"                  dz  fd„«       Zˆ xZS )ÚKosmos2ForConditionalGenerationr&   rë   ztext_model.lm_head.weightz$text_model.model.embed_tokens.weightc                 óÌ   •— t         ‰| �  |«       t        |j                  «      | _        t        |j                  «      | _        t        |«      | _	        | j                  «        y r#  )r8   rÌ   rW   r0   rñ  r¾  r.   rò  rY   ró  rI  rÖ   s     €rm   rÌ   z(Kosmos2ForConditionalGeneration.__init__Û  sN   ø€ Ü‰Ñ˜Ô ä0°×1CÑ1CÓDˆŒÜ.¨v×/CÑ/CÓDˆÔä(DÀVÓ(LˆÔ%ð 	�‰Õrn   r«   c                 óB   — | j                   j                  j                  S r#  rõ  r»   s    rm   rÂ  z4Kosmos2ForConditionalGeneration.get_input_embeddingsæ  rö  rn   c                 ó:   — || j                   j                  _        y r#  rõ  rø  s     rm   rù  z4Kosmos2ForConditionalGeneration.set_input_embeddingsé  rú  rn   c                 ó6   — | j                   j                  «       S r#  )rñ  rÔ  r»   s    rm   rÔ  z5Kosmos2ForConditionalGeneration.get_output_embeddingsì  s   € Ø�‰×4Ñ4Ó6Ð6rn   c                 ó:   — | j                   j                  |«       y r#  )rñ  Úset_output_embeddings)rf   Únew_embeddingss     rm   r  z5Kosmos2ForConditionalGeneration.set_output_embeddingsï  s   € Ø�‰×-Ñ-¨nÕ=rn   Nr`  r²  rÿ   r¦   r©   rA  rF   rÕ  r³  rÖ  r
  c                 óÊ  — d}d}|€||€t        d«      ‚| j                  |¬«      }| j                  j                  j                  |d   «      }t        j
                  j                  |d¬«      }| j                  |«      \  }} | j                  d	||||||||	|
|dœ
|¤Ž}t        |j                  |j                  |j                  |j                  |j                  |||¬«      S )
a†  
        image_embeds_position_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to indicate the location in a sequence to insert the image features . Mask values selected in `[0,
            1]`:

            - 1 for places where to put the image features,
            - 0 for places that are not for image features (i.e. for text tokens).
        image_embeds (`torch.FloatTensor` of shape `(batch_size, latent_query_num, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of `Kosmos2ImageToTextProjection`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Kosmos2ForConditionalGeneration

        >>> model = Kosmos2ForConditionalGeneration.from_pretrained("microsoft/kosmos-2-patch14-224")
        >>> processor = AutoProcessor.from_pretrained("microsoft/kosmos-2-patch14-224")

        >>> url = "https://huggingface.co/microsoft/kosmos-2-patch14-224/resolve/main/snowman.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> prompt = "<grounding> An image of"

        >>> inputs = processor(text=prompt, images=image, return_tensors="pt")

        >>> generated_ids = model.generate(
        ...     pixel_values=inputs["pixel_values"],
        ...     input_ids=inputs["input_ids"],
        ...     attention_mask=inputs["attention_mask"],
        ...     image_embeds=None,
        ...     image_embeds_position_mask=inputs["image_embeds_position_mask"],
        ...     use_cache=True,
        ...     max_new_tokens=64,
        ... )
        >>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        >>> processed_text = processor.post_process_generation(generated_text, cleanup_and_extract=False)
        >>> processed_text
        '<grounding> An image of<phrase> a snowman</phrase><object><patch_index_0044><patch_index_0863></object> warming himself by<phrase> a fire</phrase><object><patch_index_0005><patch_index_0911></object>.'

        >>> caption, entities = processor.post_process_generation(generated_text)
        >>> caption
        'An image of a snowman warming himself by a fire.'

        >>> entities
        [('a snowman', (12, 21), [(0.390625, 0.046875, 0.984375, 0.828125)]), ('a fire', (41, 47), [(0.171875, 0.015625, 0.484375, 0.890625)])]
        ```Nr  )rë   r   r5   r’   )
r`  rÿ   r©   r²  r¦   rA  rF   rÕ  r³  rÖ  )r¿   rÀ   r¦   r§   r¨   r©   r�   rª   r¡   )rð   rò  rÀ  rH  r   rã   r   ró  rñ  r¾   r¿   rÀ   r¦   r§   r¨   )rf   rë   r`  r²  rÿ   r¦   r©   rA  rF   rÕ  r³  rÖ  r
  rª   r�   Ú
lm_outputss                   rm   rø   z'Kosmos2ForConditionalGeneration.forwardò  s  € ðN #ÐØ $ÐØÐØÐ#Ü Ð!_Ó`Ð`à"&×"3Ñ"3Ø)ð #4ó #Ðð  ×,Ñ,×2Ñ2×AÑAÐBUÐVWÑBXÓYˆLäŸ=™=×2Ñ2°<ÀRÐ2ÓHˆLØ26×2OÑ2OÐP\Ó2]Ñ/ˆLÐ/à8G¸¿¹ð 9
ØØ)Ø%Ø'AØ+Ø'Ø%ØØØ)ñ9
ð ñ9
ˆ
ô :Ø—‘Ø×$Ñ$Ø&×6Ñ6Ø$×2Ñ2Ø!×,Ñ,Ø%Ø"7Ø 3ô	
ð 		
rn   c           	      ó€  — |j                  dd «      }|�|�t        d|› d�«      ‚|€|�|}|€n| j                  |«      }	| j                  j                  j	                  |	d   «      }t
        j                  j                  |d¬«      }| j                  |«      \  }}
 | j                  j                  d|||||dœ|¤Ž}|S )	NÚinputsz
`inputs`: zp were passed alongside `pixel_values` which is not allowed.Make sure to either pass `inputs` or pixel_values=...r   r5   r’   )r`  rÿ   r©   r²  rA  r¡   )rá  rð   rò  rÀ  rH  r   rã   r   ró  rñ  Úgenerate)rf   rë   r²  r`  rÿ   r©   rA  r
  r  rª   r�   Úoutputs               rm   r  z(Kosmos2ForConditionalGeneration.generatea  sö   € ð —‘˜H dÓ+ˆØÐ#¨Ð(:ÜØ˜V˜Hð %Hð Ióð ð Ð FÐ$6Ø!ˆLàÐØ"&×"3Ñ"3°LÓ"AÐà×,Ñ,×2Ñ2×AÑAÐBUÐVWÑBXÓYˆLäŸ=™=×2Ñ2°<ÀRÐ2ÓHˆLØ26×2OÑ2OÐP\Ó2]Ñ/ˆLÐ/à)�—‘×)Ñ)ð 
ØØ)Ø%Ø'AØ'ñ
ð ñ
ˆð ˆrn   )NNNNNNNNNNr   )NNNNNN)ro   rp   rq   r!   rr   rÅ  rå  rÌ   r   rz   rÂ  rù  rÔ  r  r   r   rG   rú   r
   ræ  r…   rû   r   r   rŸ   r¾   rø   ry   r  r{   r|   s   @rm   r  r  Ð  s(  ø… ð ÓØ$€OØ5Ð7]Ð^Ðð	˜}õ 	ð2 b§i¡ió 2ò3ð7 r§y¡yó 7ò>ð Øð -1Ø)-Ø:>Ø.2Ø(,Ø,0Ø-1Ø,0Ø*.Ø!%Ø-.ñk
à—l‘l TÑ)ðk
ð —<‘< $Ñ&ðk
ð %*§L¡L°4Ñ$7ð	k
ð
 Ÿ™ tÑ+ðk
ð  ™ðk
ð —l‘l TÑ)ðk
ð —|‘| dÑ*ðk
ð —l‘l TÑ)ðk
ð × Ñ  4Ñ'ðk
ð ˜$‘;ðk
ð ˜eŸl™lÑ*ðk
ð Ð+Ñ,ðk
ð 
Ð;Ñ	;òk
ó ó ðk
ðZ €U‡]�]ƒ_ð -1Ø:>Ø)-Ø.2Ø,0Ø-1ñ%à—l‘l TÑ)ð%ð %*§L¡L°4Ñ$7ð%ð —<‘< $Ñ&ð	%ð
 Ÿ™ tÑ+ð%ð —l‘l TÑ)ð%ð —|‘| dÑ*ò%ó ô%rn   r  )r  rï  r%   r#  rp  )r1   )Wrž   rV  rý  Úcollections.abcr   Údataclassesr   Útypingr   rG   r   Ú r   r=   Úactivationsr	   Úcache_utilsr
   r   r   Ú
generationr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úconfiguration_kosmos2r!   r"   r#   Ú
get_loggerro   rØ  r%   rú   r~   rû   rŒ   ÚSizerŽ   rš   rœ   r¤   r¾   rz   r<   rZ  r  rK   rQ   r)   r9  rD  r`   rU   rV   r*   r\   r¾  rÇ  rW   rY   rï  r  Ú__all__r¡   rn   rm   ú<module>r(     sH  ðñ ã Û Ý $Ý !Ý ã Ý å &Ý !ß CÑ CÝ )Ý BÝ 9÷ó ÷ GÝ &ß j× jÝ 7ß Eß XÑ Xð 
ˆ×	Ñ	˜HÓ	%€ð ô=4˜_ó =4ó ð=4ñ@[�u—|‘|ð [¨E¯K©Kð [À#ÈÁ*ó [ð jkñ\Ø—Z‘Zð\Ø(-¯©ð\Ø=B¿\¹\ð\Øcfó\ð" Ø
ô
BÐ.Hó 
Bó ó ð
Bñ ðôð
 ô
˜ó 
ó óð
ñ: ðôð
 ô
°ó 
ó óð
ôFP˜bŸi™iô Pðv ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ó%ô,<)˜RŸY™Yô <)ô@�r—y‘yô ô Ð :ô ôB-
˜2Ÿ9™9ô -
ôb,
Ð5ô ,
ô`j8¨r¯y©yô j8ôZo)˜"Ÿ)™)ô o)ôd�R—Y‘Yô ô.VÐ1ô Vôrz
Ð3ô z
ôz
Ð/ô 
ô:3
Ð-ô 3
ñl ðôôHÐ3°_ó HóðHôV + 2§9¡9ô  +ñF ðôô
N
Ð)ó N
óð
N
ñb ðôôqÐ&<¸oó qóðqòh X�rn   