Ë
    ÛÓ^j8·  ã                   ód  — d dl mZ d dlmZ d dlZd dlmc mZ d dl	m
Z
 d dlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZ ddlmZmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z* ddl+m,Z,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z= ddl>m?Z? ddl@mAZAmBZBmCZC  e(«       r	  e)jˆ                  eE«      ZF e&d¬«      e
 G d„ de?«      «       «       ZG e&d¬«      e
 G d„ de2«      «       «       ZH e&d¬«      e
 G d „ d!e«      «       «       ZIe& G d"„ d#e!«      «       ZJ e&d$¬%«      e G d&„ d'e«      «       «       ZK G d(„ d)e:«      ZL G d*„ d+e;«      ZM G d,„ d-eC«      ZN G d.„ d/ejž                  «      ZP G d0„ d1ejž                  «      ZQ G d2„ d3eB«      ZR G d4„ d5eA«      ZS G d6„ d7e0«      ZT G d8„ d9ejž                  «      ZU G d:„ d;e8«      ZV G d<„ d=e7«      ZW G d>„ d?e5«      ZX G d@„ dAe6«      ZY G dB„ dCejž                  «      ZZ G dD„ dEejž                  «      Z[ G dF„ dGejž                  «      Z\ G dH„ dIejž                  «      Z] G dJ„ dKe4«      Z^ G dL„ dMejž                  «      Z_ G dN„ dOejž                  «      Z` e&dP¬%«       G dQ„ dReJ«      «       Za G dS„ dTeJe«      Zbg dU¢Zcy)Vé    )ÚCallable)Ú	dataclassN)Ústrict)Únné   )Úinitialization)ÚACT2FN)ÚCache)ÚPreTrainedConfig)Ú%ClassifierFreeGuidanceLogitsProcessorÚGenerationMixinÚGenerationModeÚLogitsProcessorList)ÚGenerateDecoderOnlyOutput)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_vision_availableÚloggingÚtorch_compilable_checké   )ÚCONFIG_MAPPINGÚ
AutoConfigÚ	AutoModel)ÚBlip2VisionModel)ÚChameleonVQVAEConfig)ÚChameleonVQVAEÚChameleonVQVAEEncoderAttnBlockÚ#ChameleonVQVAEEncoderConvDownsampleÚ ChameleonVQVAEEncoderResnetBlockÚChameleonVQVAEVectorQuantizer)ÚIdeficsBaseModelOutputWithPastÚIdeficsCausalLMOutputWithPast)Úeager_attention_forward)ÚSiglipVisionConfig)ÚSiglipEncoderÚSiglipEncoderLayerÚSiglipVisionEmbeddingszdeepseek-community/Janus-Pro-1B)Ú
checkpointc                   ó  — e Zd ZU dZdZeed<   dZeed<   dZeed<   dZ	ee
e   z  eeef   z  ed	<   d
Zeed<   dZeez  ed<   dZeed<   dZeez  ed<   dZeed<   dZeez  ed<   dZeed<   dZeed<   dZeed<   dZeed<    e«       Zy)ÚJanusVisionConfigzÕ
    projection_dropout (`float`, *optional*, defaults to 0.0):
        Dropout probability for the projection layer.
    num_image_tokens (`int`, *optional*, defaults to 576):
        Number of image tokens.
    i   Úhidden_sizeé   Únum_hidden_layersé   Únum_attention_headsi€  Ú
image_sizeÚgeluÚ
hidden_actg      @Ú	mlp_ratioTÚattention_biasç        Úhidden_dropout_rateé   Úprojection_dimÚprojection_dropoutFÚuse_qk_normç{®Gáz”?Úinitializer_ranger   Údepthi@  Únum_image_tokensN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r2   ÚintÚ__annotations__r4   r6   r7   ÚlistÚtupler9   Ústrr:   Úfloatr;   Úboolr=   r?   r@   rA   rC   rD   rE   ÚAttributeErrorÚintermediate_size© ó    úr/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/janus/modular_janus.pyr1   r1   @   sÄ   … ñð €K�ÓØÐ�sÓØ!Ð˜Ó!Ø47€J��d˜3‘i‘ %¨¨S¨¡/Ñ1Ó7Ø€J�ÓØ €Iˆu�s‰{Ó Ø€N�DÓØ'*Ð˜ ™Ó*Ø€N�CÓØ&)Ð˜ ™Ó)Ø€K�ÓØ#Ð�uÓ#Ø€Eˆ3ƒNØÐ�cÓÙ&Ó(ÑrT   r1   c                   ó8  — e Zd ZU dZdZeed<   dZeed<   dZe	ed<   dZ
eed	<   d
Zeed<   dZeed<   dZeed<   dZeed<   dZee   eedf   z  ed<   dZeed<   dZeez  ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<    e«       Z e«       Z e«       Zy )!ÚJanusVQVAEConfiga   
    base_channels (`int`, *optional*, defaults to 128):
        Base channel count.
    channel_multiplier (`list[int]`, *optional*, defaults to `[1, 1, 2, 2, 4]`):
        Channel multipliers for each resolution.
    num_res_blocks (`int`, *optional*, defaults to 2):
        Number of residual blocks.
    num_patches (`int`, *optional*, defaults to 32):
        Num of patches the input images can be divided into.
    out_channels (`int`, *optional*, defaults to 3):
        Number of out channels.
    image_token_embed_dim (`int`, *optional*, defaults to 2048):
        Dimension of image embeddings. It should be same as the dimensionality of text embeddings.
    é   Ú	embed_dimi @  Únum_embeddingsFÚdouble_latenté   Úlatent_channelsé    Únum_patchesr   Úin_channelsÚout_channelsé€   Úbase_channels)é   rd   r   r   é   .Úchannel_multiplierr   Únum_res_blocksr<   ÚdropoutrB   rC   r>   r?   r4   r8   r9   Úimage_token_embed_dimN)rF   rG   rH   rI   rY   rJ   rK   rZ   r[   rP   r]   r_   r`   ra   rc   rf   rL   rM   rg   rh   rO   rC   r?   r4   r9   rN   ri   rQ   Ú
resolutionÚattn_resolutionsÚ	attn_typerS   rT   rU   rW   rW   [   sÛ   … ñð €IˆsÓØ€N�CÓØ€M�4ÓØ€O�SÓØ€K�ÓØ€K�ÓØ€L�#ÓØ€M�3ÓØ6EÐ˜˜S™	 E¨#¨s¨(¡OÑ3ÓEØ€N�CÓØ€GˆU�S‰[ÓØ#Ð�uÓ#Ø€N�CÓØÐ�sÓØ€J�ÓØ!%Ð˜3Ó%áÓ!€JÙ%Ó'ÐÙÓ �IrT   rW   c                   óž   ‡ — e Zd ZU dZdZeeedœZdZ	e
ez  dz  ed<   dZe
ez  dz  ed<   dZe
ez  dz  ed<   dZeed	<   d
Zeed<   ˆ fd„Zˆ xZS )ÚJanusConfiga  
    Example:

    ```python
    >>> from transformers import JanusForConditionalGeneration, JanusConfig, JanusVisionConfig, JanusVQVAEConfig, LlamaConfig

    >>> # Initializing a Janus vision config
    >>> vision_config = JanusVisionConfig()

    >>> # Initializing a Llama config
    >>> text_config = LlamaConfig()

    >>> # Initializing a VQ config
    >>> vq_config = JanusVQVAEConfig()

    >>> # Initializing a Janus Pro 1B style configuration
    >>> configuration = JanusConfig(vision_config=vision_config, text_config=text_config, vq_config=vq_config)

    >>> # Initializing a model from the Janus Pro 1B style configuration
    >>> model = JanusForConditionalGeneration(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Újanus)Útext_configÚvision_configÚ	vq_configNrp   rq   rr   iåˆ Úimage_token_idTÚtie_word_embeddingsc                 ór  •— t        | j                  t        «      rT| j                  j                  dd«      | j                  d<   t	        | j                  d      di | j                  ¤Ž| _        n3| j                  €'t
        j                  d«       t	        d   «       | _        | j                  €%t
        j                  d«       t        «       | _        n4t        | j                  t        «      rt        di | j                  ¤Ž| _        | j                  €%t
        j                  d«       t        «       | _	        n4t        | j                  t        «      rt        di | j                  ¤Ž| _	        | j                  j                  | j                  j                  z  | j                  _        t        ‰| �<  di |¤Ž y )NÚ
model_typeÚllamaz7`text_config` is None. Initializing with default valueszK`vision_config` is None. Initializing with default JanusVisionConfig valueszF`vq_config` is None. Initializing with default JanusVQVAEConfig valuesrS   )Ú
isinstancerp   ÚdictÚgetr   ÚloggerÚinforq   r1   rr   rW   r7   Ú
patch_sizer_   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €rU   r   zJanusConfig.__post_init__¬   sN  ø€ Ü�d×&Ñ&¬Ô-Ø-1×-=Ñ-=×-AÑ-AÀ,ÐPWÓ-XˆD×Ñ˜\Ñ*Ü-¨d×.>Ñ.>¸|Ñ.LÑMÑaÐPT×P`ÑP`ÑaˆDÕØ×ÑÐ%Ü�K‰KÐQÔRÜ-¨gÑ6Ó8ˆDÔà×ÑÐ%Ü�K‰KÐeÔfÜ!2Ó!4ˆDÕÜ˜×*Ñ*¬DÔ1Ü!2Ñ!H°T×5GÑ5GÑ!HˆDÔà�>‰>Ð!Ü�K‰KÐ`ÔaÜ-Ó/ˆD�NÜ˜Ÿ™¬Ô-Ü-Ñ?°·±Ñ?ˆDŒNð &*×%7Ñ%7×%BÑ%BÀd×FXÑFX×FcÑFcÑ%cˆ�‰Ô"Ü‰ÑÑ' Ó'rT   )rF   rG   rH   rI   rv   r   r1   rW   Úsub_configsrp   ry   r   rK   rq   rr   rs   rJ   rt   rP   r   Ú__classcell__©r‚   s   @rU   rn   rn   ƒ   s   ø… ñð2 €Jà!Ø*Ø%ñ€Kð 37€K�Ð(Ñ(¨4Ñ/Ó6Ø48€M�4Ð*Ñ*¨TÑ1Ó8Ø04€IˆtÐ&Ñ&¨Ñ-Ó4Ø €N�CÓ Ø $Ð˜Ó$÷(ð (rT   rn   c                   óR   ‡ — e Zd ZU eed<   dZdZdZddgZddgZ	dZ
dZdZˆ fd	„Zˆ xZS )
ÚJanusPreTrainedModelÚconfigÚmodel©ÚimageÚtextTÚLlamaDecoderLayerÚJanusVisionEncoderLayerÚpast_key_valuesÚcausal_maskc                 óø   •— t         ‰| �  |«       t        |t        «      rZt	        j
                  |j                  t        j                  |j                  j                  d   «      j                  d«      «       y y )Néÿÿÿÿ)rd   r’   )r~   Ú_init_weightsrx   ÚJanusVisionEmbeddingsÚinitÚcopy_Úposition_idsÚtorchÚarangeÚshapeÚexpand)r€   Úmoduler‚   s     €rU   r“   z"JanusPreTrainedModel._init_weightsÒ   s[   ø€ Ü‰Ñ˜fÔ%Ü�fÔ3Ô4Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhð 5rT   )rF   rG   rH   rn   rK   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphr“   r„   r…   s   @rU   r‡   r‡   Å   sO   ø… àÓØÐØ(ÐØ&*Ð#Ø,Ð.GÐHÐØ#4°mÐ"DÐØÐØ€Nà!Ð÷ið irT   r‡   z9
    Base class for Janus VQ-VAE mode model outputs.
    )Úcustom_introc                   ób   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   y)ÚJanusVQVAEOutputzÿ
    decoded_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
        Reconstructed pixel values after encoding and decoding the input.
    embedding_loss (`torch.FloatTensor`):
        Embedding loss.
    NÚdecoded_pixel_valuesÚembedding_loss)	rF   rG   rH   rI   r¨   r˜   ÚFloatTensorrK   r©   rS   rT   rU   r§   r§   Ø   s4   … ñð 6:Ð˜%×+Ñ+¨dÑ2Ó9Ø/3€N�E×%Ñ%¨Ñ,Ô3rT   r§   c                   ó   — e Zd Zy)ÚJanusBaseModelOutputWithPastN©rF   rG   rH   rS   rT   rU   r¬   r¬   ê   ó   „ ØrT   r¬   c                   ó   — e Zd Zy)ÚJanusCausalLMOutputWithPastNr­   rS   rT   rU   r°   r°   î   r®   rT   r°   c                   óJ   — e Zd Zddej                  dedej                  fd„Zy)r”   Úpixel_valuesÚinterpolate_pos_encodingÚreturnc                 óX  — |j                   \  }}}}| j                  j                  j                  }| j                  |j	                  |¬«      «      }|j                  d«      j                  dd«      }|r| j                  |||«      }	n| j                  | j                  «      }	||	z   }|S )N)Údtyper   rd   )
rš   Úpatch_embeddingÚweightr¶   ÚtoÚflattenÚ	transposer³   Úposition_embeddingr—   )
r€   r²   r³   Ú_ÚheightÚwidthÚtarget_dtypeÚpatch_embedsÚ
embeddingsÚ
pos_embedss
             rU   ÚforwardzJanusVisionEmbeddings.forwardó   s¢   € Ø*×0Ñ0Ñˆˆ1ˆf�eØ×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆØ!×)Ñ)¨!Ó,×6Ñ6°q¸!Ó<ˆ
á#Ø×6Ñ6°zÀ6È5ÓQ‰Jà×0Ñ0°×1BÑ1BÓCˆJà *Ñ,ˆ
àÐrT   N)F)rF   rG   rH   r˜   ÚTensorrP   rÄ   rS   rT   rU   r”   r”   ò   s'   „ ñ E§L¡Lð ÈDð Ð]b×]iÑ]iô rT   r”   c                   ót   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej                  dej                  dz  dee	   fd„Z
ˆ xZS )
ÚJanusVisionAttentionz(Attention Class for Janus Vision Encoderrˆ   c                 óF  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚| j                  dz  | _	        |j                  | _
        |j                  }|j                  }d| _        d| _        t        j                   | j                  | j                  | j                  z  |j"                  ¬«      | _        t        j                   | j                  | j                  | j                  z  |j"                  ¬«      | _        t        j                   | j                  | j                  | j                  z  |j"                  ¬«      | _        t        j                   | j                  | j                  «      | _        |dkD  rt        j,                  |«      nt        j.                  «       | _        |rt        j0                  | j                  «      nt        j.                  «       | _        |r%t        j0                  | j                  «      | _        y t        j.                  «       | _        y )	Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      à¿Frd   ©Úbiasr   )r~   Ú__init__rˆ   r2   rY   r6   Ú	num_headsÚhead_dimÚ
ValueErrorÚscaleÚattention_dropoutr@   rA   Ú	is_causalÚnum_key_value_groupsr   ÚLinearr;   Úq_projÚk_projÚv_projÚprojection_layerÚDropoutÚIdentityÚ	LayerNormÚq_normÚk_norm)r€   rˆ   Úproj_dropoutÚqk_normr‚   s       €rU   rË   zJanusVisionAttention.__init__  sÖ  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
Ø!'×!9Ñ!9ˆÔØ×0Ñ0ˆØ×$Ñ$ˆØˆŒð %&ˆÔ!ä—i‘i §¡°·±ÀÇÁÑ0NÐU[×UjÑUjÔkˆŒÜ—i‘i §¡°·±ÀÇÁÑ0NÐU[×UjÑUjÔkˆŒÜ—i‘i §¡°·±ÀÇÁÑ0NÐU[×UjÑUjÔkˆŒÜ "§	¡	¨$¯.©.¸$¿.¹.Ó IˆÔØ>JÈQÒ>N¤"§*¡*¨\Ô":ÔTV×T_ÑT_ÓTaˆÔá6=”b—l‘l 4§>¡>Ô2Ä2Ç;Á;Ã=ˆŒÙ6=”b—l‘l 4§>¡>Ó2ˆ�Ä2Ç;Á;Ã=ˆ�rT   NÚhidden_statesÚattention_maskr�   c                 ó"  — |j                  «       \  }}}| j                  |«      }| j                  |«      }| j                  |«      }	|j	                  d| j
                  | j                  «      }| j                  |«      }|j	                  d| j
                  | j                  «      }| j                  |«      }|j	                  ||| j
                  | j                  «      j                  dd«      }|j	                  ||| j
                  | j                  «      j                  dd«      }|	j                  ||| j
                  | j                  «      j                  dd«      }	t        j                  | j                  j                  t        «      }
 |
| |||	|f| j                   sdn| j"                  | j$                  | j&                  dœ|¤Ž\  }}|j	                  ||| j(                  «      }| j+                  |«      }| j-                  |«      }||fS )Nr’   rd   r   r<   )rh   ÚscalingrÑ   )ÚsizerÔ   rÕ   rÖ   ÚreshaperÌ   rÍ   rÛ   rÜ   r»   Úviewr   Úget_interfacerˆ   Ú_attn_implementationr*   ÚtrainingrÐ   rÏ   rÑ   rY   r×   r@   )r€   rß   rà   r�   Ú
batch_sizeÚseq_lenr½   Úquery_statesÚ
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightsÚoutputs                 rU   rÄ   zJanusVisionAttention.forward#  s×  € ð "/×!3Ñ!3Ó!5Ñˆ
�G˜Qà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆà#×+Ñ+¨B°·±ÀÇÁÓNˆØ—{‘{ <Ó0ˆà×'Ñ'¨¨D¯N©N¸D¿M¹MÓJˆ
Ø—[‘[ Ó,ˆ
à#×+Ñ+¨J¸ÀÇÁÐQU×Q^ÑQ^Ó_×iÑiÐjkÐmnÓoˆØ×'Ñ'¨
°G¸T¿^¹^ÈTÏ]É]Ó[×eÑeÐfgÐijÓkˆ
Ø#×(Ñ(¨°W¸d¿n¹nÈdÏmÉmÓ\×fÑfÐghÐjkÓlˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—J‘JØ—n‘nñ
%
ð ñ
%
Ñ!ˆ�\ð "×)Ñ)¨*°g¸t¿~¹~ÓNˆà×&Ñ& {Ó3ˆØ×(Ñ(¨Ó0ˆØ�|Ð#Ð#rT   ©N)rF   rG   rH   rI   r1   rË   r˜   rÅ   r   r   rÄ   r„   r…   s   @rU   rÇ   rÇ     sO   ø„ Ù2ðQÐ0õ Qð@ /3ñ)$à—|‘|ð)$ð Ÿ™ tÑ+ð)$ð Ð+Ñ,÷	)$rT   rÇ   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚJanusVisionMLPrˆ   c                 óú  •— t         ‰| �  «        || _        t        |j                  |j
                  z  «      | _        t        |j                     | _	        t        j                  |j                  | j                  «      | _        t        j                  | j                  |j                  «      | _        t        j                  |j                  «      | _        t        j                  |j                  «      | _        y rò   )r~   rË   rˆ   rJ   r2   r:   rR   r	   r9   Úactivation_fnr   rÓ   Úfc1Úfc2rØ   r=   Údropout1Údropout2©r€   rˆ   r‚   s     €rU   rË   zJanusVisionMLP.__init__P  s®   ø€ Ü‰ÑÔØˆŒÜ!$ V×%7Ñ%7¸&×:JÑ:JÑ%JÓ!KˆÔÜ# F×$5Ñ$5Ñ6ˆÔÜ—9‘9˜V×/Ñ/°×1GÑ1GÓHˆŒÜ—9‘9˜T×3Ñ3°V×5GÑ5GÓHˆŒÜŸ
™
 6×#=Ñ#=Ó>ˆŒÜŸ
™
 6×#=Ñ#=Ó>ˆ�rT   rß   r´   c                 ó°   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }|S rò   )r÷   rö   rù   rø   rú   ©r€   rß   s     rU   rÄ   zJanusVisionMLP.forwardZ  sP   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ mÓ4ˆØŸ™ Ó/ˆØŸ™ mÓ4ˆØÐrT   )	rF   rG   rH   r1   rË   r˜   rÅ   rÄ   r„   r…   s   @rU   rô   rô   O  s+   ø„ ð?Ð0õ ?ð U§\¡\ð °e·l±l÷ rT   rô   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )rŽ   rˆ   c                 óT  •— t         ‰| �  |«       || _        |j                  | _        t        |«      | _        t        j                  | j                  |j                  ¬«      | _
        t        j                  | j                  |j                  ¬«      | _        t        |«      | _        y )N)Úeps)r~   rË   rˆ   r2   rY   rÇ   Ú	self_attnr   rÚ   Úlayer_norm_epsÚlayer_norm1Úlayer_norm2rô   Úmlprû   s     €rU   rË   z JanusVisionEncoderLayer.__init__d  sv   ø€ Ü‰Ñ˜Ô ØˆŒØ×+Ñ+ˆŒÜ-¨fÓ5ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÔÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÔÜ! &Ó)ˆ�rT   ©rF   rG   rH   r1   rË   r„   r…   s   @rU   rŽ   rŽ   c  s   ø„ ð*Ð0÷ *ñ *rT   rŽ   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚJanusVisionEncoderrˆ   c                 ó¶   •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        y c c}w rò   )r~   rË   r   Ú
ModuleListÚranger4   rŽ   Úlayers©r€   rˆ   r½   r‚   s      €rU   rË   zJanusVisionEncoder.__init__o  s@   ø€ Ü‰Ñ˜Ô Ü—m‘mÌeÐTZ×TlÑTlÓNmÖ$nÈÔ%<¸VÕ%DÒ$nÓoˆ�ùÒ$ns   ·Ar  r…   s   @rU   r  r  n  s   ø„ ðpÐ0÷ pñ prT   r  c            
       ór   ‡ — e Zd ZeedœZdefˆ fd„Z	 	 d
dej                  dz  de
dee   deez  fd	„Zˆ xZS )ÚJanusVisionModel©rß   Ú
attentionsrˆ   c                 óD   •— t         ‰| �  |«       t        |«      | _        y rò   )r~   rË   r  Úencoderrû   s     €rU   rË   zJanusVisionModel.__init__z  s   ø€ Ü‰Ñ˜Ô Ü)¨&Ó1ˆ�rT   Nr²   r³   r�   r´   c                 óø   — |€t        d«      ‚| j                  ||¬«      } | j                  dd|i|¤Ž}|j                  }| j	                  |«      }|d d …dd d …f   }| j	                  |«      }t        ||¬«      S )Nz You have to specify pixel_values)r³   Úinputs_embedsr   )Úlast_hidden_stateÚpooler_outputrS   )rÎ   rÂ   r  r  Úpost_layernormr   )r€   r²   r³   r�   rß   Úencoder_outputsr  Úpooled_outputs           rU   rÄ   zJanusVisionModel.forward~  sœ   € ð ÐÜÐ?Ó@Ð@àŸ™¨ÐOg˜Óhˆà+7¨4¯<©<ñ ,
Ø'ð,
àñ,
ˆð
 ,×=Ñ=ÐØ ×/Ñ/Ð0AÓBÐà)ª!¨Q²¨'Ñ2ˆØ×+Ñ+¨MÓ:ˆä)Ø/Ø'ô
ð 	
rT   ©NF)rF   rG   rH   rŽ   rÇ   Ú_can_record_outputsr1   rË   r˜   rª   rP   r   r   rM   r   rÄ   r„   r…   s   @rU   r  r  t  sh   ø„ à0Ø*ñÐð
2Ð0õ 2ð 26Ø).ñ
à×'Ñ'¨$Ñ.ð
ð #'ð
ð Ð+Ñ,ð	
ð
 
Ð+Ñ	+÷
rT   r  c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚJanusVisionAlignerMLPrˆ   c           	      ó‚  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  t        d|j                  «      D �cg c],  }t        j                  |j
                  |j
                  «      ‘Œ. c}«      | _
        t        |j                     | _        y c c}w ©Nrd   )r~   rË   r   rÓ   r2   r?   r÷   r
  r  rD   Úhidden_layersr	   r9   rö   r  s      €rU   rË   zJanusVisionAlignerMLP.__init__›  sŒ   ø€ Ü‰ÑÔä—9‘9˜V×/Ñ/°×1FÑ1FÓGˆŒÜŸ]™]ÜNSÐTUÐW]×WcÑWcÓNdÖeÈŒR�Y‰Y�v×,Ñ,¨f×.CÑ.CÕDÒeó
ˆÔô $ F×$5Ñ$5Ñ6ˆÕùò fó   Á&1B<c                 ó|   — | j                  |«      }| j                  D ]  }| j                  |«      } ||«      }Œ |S rò   ©r÷   r!  rö   ©r€   rß   Úlayers      rU   rÄ   zJanusVisionAlignerMLP.forward¤  óG   € ØŸ™ Ó/ˆØ×'Ñ'ò 	1ˆEØ ×.Ñ.¨}Ó=ˆMÙ! -Ó0‰Mð	1ð ÐrT   )rF   rG   rH   r1   rË   rÄ   r„   r…   s   @rU   r  r  š  s   ø„ ð7Ð0õ 7örT   r  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚJanusVQVAEVectorQuantizerrˆ   c                 óN   •— t         ‰| �  |«       |j                  gdz  | _        y )Nr   )r~   rË   r_   Úquant_state_dimsrû   s     €rU   rË   z"JanusVQVAEVectorQuantizer.__init__­  s&   ø€ Ü‰Ñ˜Ô Ø!'×!3Ñ!3Ð 4°qÑ 8ˆÕrT   Úimage_tokensr´   c                 óB  — |j                   d   }| j                  j                  j                   d   }| j                  |«      }t        j                  |dd¬«      }|j                  |g| j                  ¢|‘­«      }|j                  dddd«      j                  «       }|S )Nr   r’   r   )ÚpÚdimr   rd   )	rš   Ú	embeddingr¸   ÚFÚ	normalizerå   r+  ÚpermuteÚ
contiguous)r€   r,  ré   Úemb_dimÚhidden_state_quants        rU   Úget_codebook_entryz,JanusVQVAEVectorQuantizer.get_codebook_entry±  sž   € Ø!×'Ñ'¨Ñ*ˆ
Ø—~‘~×,Ñ,×2Ñ2°2Ñ6ˆð "Ÿ^™^¨LÓ9ÐäŸ[™[Ð);¸qÀbÔIÐð 0×4Ñ4°jÐ5bÀ4×CXÑCXÐ5bÐZaÑ5bÓcÐØ/×7Ñ7¸¸1¸aÀÓC×NÑNÓPÐà!Ð!rT   )
rF   rG   rH   rW   rË   r˜   Ú
LongTensorrª   r7  r„   r…   s   @rU   r)  r)  ¬  s/   ø„ ð9Ð/õ 9ð"¨u×/?Ñ/?ð "ÀE×DUÑDU÷ "rT   r)  c                   ó   — e Zd Zy)ÚJanusVQVAEResnetBlockNr­   rS   rT   rU   r:  r:  Á  r®   rT   r:  c                   ó   — e Zd Zy)ÚJanusVQVAEAttnBlockNr­   rS   rT   rU   r<  r<  Å  r®   rT   r<  c                   ó   — e Zd Zy)ÚJanusVQVAEConvDownsampleNr­   rS   rT   rU   r>  r>  É  r®   rT   r>  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJanusVQVAEConvUpsamplec                 ót   •— t         ‰| �  «        t        j                  j	                  ||ddd¬«      | _        y )Nr   rd   ©Úkernel_sizeÚstrideÚpadding)r~   rË   r˜   r   ÚConv2dÚconv)r€   r`   r‚   s     €rU   rË   zJanusVQVAEConvUpsample.__init__Î  s.   ø€ Ü‰ÑÔÜ—H‘H—O‘O K°È!ÐTUÐ_`�OÓaˆ�	rT   c                 óX   — t        j                  |dd¬«      }| j                  |«      }|S )Ng       @Únearest)Úscale_factorÚmode)r1  ÚinterpolaterG  rý   s     rU   rÄ   zJanusVQVAEConvUpsample.forwardÒ  s(   € ÜŸ™ mÀ#ÈIÔVˆØŸ	™	 -Ó0ˆØÐrT   )rF   rG   rH   rË   rÄ   r„   r…   s   @rU   r@  r@  Í  s   ø„ ôbörT   r@  c                   ó`   ‡ — e Zd Zdedefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚJanusVQVAEMidBlockrˆ   Úchannelsc                 óŽ   •— t         ‰| �  «        t        |||¬«      | _        t	        |«      | _        t        |||¬«      | _        y )N©rˆ   r`   ra   )r~   rË   r:  Úblock_1r<  Úattn_1Úblock_2)r€   rˆ   rO  r‚   s      €rU   rË   zJanusVQVAEMidBlock.__init__Ù  sF   ø€ Ü‰ÑÔÜ,ØØ Ø!ô
ˆŒô
 *¨(Ó3ˆŒÜ,ØØ Ø!ô
ˆ�rT   rß   r´   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rò   )rR  rS  rT  rý   s     rU   rÄ   zJanusVQVAEMidBlock.forwardç  s2   € ØŸ™ ]Ó3ˆØŸ™ MÓ2ˆØŸ™ ]Ó3ˆØÐrT   )
rF   rG   rH   rW   rJ   rË   r˜   rÅ   rÄ   r„   r…   s   @rU   rN  rN  Ø  s2   ø„ ð
Ð/ð 
¸3õ 
ð U§\¡\ð °e·l±l÷ rT   rN  c                   ó>   ‡ — e Zd Zˆ fd„Zdej
                  fd„Zˆ xZS )ÚJanusVQVAEEncoderc           	      ó¢  •— t         ‰| �  «        t        |j                  «      | _        |j
                  | _        |j                  }|j                  }|j                  }|j                  }|j                  }t        j                  j                  ||ddd¬«      | _        dt        |«      z   }|| _        t        j                   «       | _        t%        | j                  «      D �]   }t        j                   «       }	t        j                   «       }
|||   z  }|||   z  }t%        | j
                  «      D ]N  }|	j'                  t)        |||¬«      «       |}|| j                  dz
  k(  sŒ5|
j'                  t+        |«      «       ŒP t        j,                  «       }|	|_        |
|_        || j                  dz
  k7  rt3        |«      |_        | j"                  j'                  |«       �Œ t7        |«      | _        t        j                  j;                  d|dd¬	«      | _        t        j                  j                  ||rd
|z  n|ddd¬«      | _        y )Nr   rd   rB  )rd   rQ  r^   ç�íµ ÷Æ°>T©Ú
num_groupsÚnum_channelsr   Úaffiner   ) r~   rË   Úlenrf   Únum_resolutionsrg   rc   r`   r[   r]   r˜   r   rF  Úconv_inrM   Úin_channel_multiplierr
  Údownr  Úappendr:  r<  ÚModuleÚblockÚattnr>  Ú
downsamplerN  ÚmidÚ	GroupNormÚnorm_outÚconv_out)r€   rˆ   rc   r`   r[   r]   rf   ra  Úi_levelre  rf  Úblock_inÚ	block_outÚi_blockrb  r‚   s                  €rU   rË   zJanusVQVAEEncoder.__init__ï  s  ø€ Ü‰ÑÔä" 6×#<Ñ#<Ó=ˆÔØ$×3Ñ3ˆÔØ×,Ñ,ˆØ×(Ñ(ˆØ×,Ñ,ˆØ ×0Ñ0ˆØ#×6Ñ6Ðä—x‘x—‘ {°MÈqÐYZÐde�ÓfˆŒà $¤uÐ-?Ó'@Ñ @ÐØ%:ˆÔ"Ü—M‘M“OˆŒ	Ü˜T×1Ñ1Ó2ó 	#ˆGÜ—M‘M“OˆEÜ—=‘=“?ˆDØ$Ð'<¸WÑ'EÑEˆHØ%Ð(:¸7Ñ(CÑCˆIÜ  ×!4Ñ!4Ó5ò 
?�Ø—‘Ü)Ø%Ø$,Ø%.ôôð %�Ø˜d×2Ñ2°QÑ6Ó6Ø—K‘KÔ 3°HÓ =Õ>ð
?ô —9‘9“;ˆDØˆDŒJØˆDŒIØ˜$×.Ñ.°Ñ2Ò2Ü":¸8Ó"D�”Ø�I‰I×Ñ˜TÖ"ð-	#ô0 & f¨hÓ7ˆŒäŸ™×*Ñ*°bÀxÐUYÐbfÐ*ÓgˆŒÜŸ™Ÿ™ØÙ#0ˆA�Ò°oØØØð (ó 
ˆ�rT   r²   c                 ó²  — | j                  |«      g}t        | j                  «      D ]Ü  }t        | j                  «      D ]  } | j                  |   j
                  |   |d   «      }t        | j                  |   j                  «      dkD  r" | j                  |   j                  |   |«      }|j                  |«       Œ� || j                  dz
  k7  sŒ­|j                  | j                  |   j                  |d   «      «       ŒÞ |d   }| j                  |«      }| j                  |«      }|t        j                  |«      z  }| j                  |«      }|S )Nr’   r   rd   )r`  r  r_  rg   rb  re  r^  rf  rc  rg  rh  rj  r˜   Úsigmoidrk  )r€   r²   rß   rl  ro  Úhidden_stater  s          rU   rÄ   zJanusVQVAEEncoder.forward"  sT  € àŸ™ lÓ3Ð4ˆÜ˜T×1Ñ1Ó2ò 		WˆGÜ  ×!4Ñ!4Ó5ò 3�Ø@˜tŸy™y¨Ñ1×7Ñ7¸Ñ@Ø! "Ñ%ó �ô �t—y‘y Ñ)×.Ñ.Ó/°!Ò3Ø#C 4§9¡9¨WÑ#5×#:Ñ#:¸7Ñ#CÀLÓ#Q�LØ×$Ñ$ \Õ2ð3ð ˜$×.Ñ.°Ñ2Ó2Ø×$Ñ$ T§Y¡Y¨wÑ%7×%BÑ%BÀ=ÐQSÑCTÓ%UÕVð		Wð *¨"Ñ-ÐØ ŸH™HÐ%6Ó7Ðð !ŸM™MÐ*;Ó<ÐØœUŸ]™]Ð+<Ó=Ñ=ÐØ ŸM™MÐ*;Ó<ÐØ Ð rT   )rF   rG   rH   rË   r˜   r8  rÄ   r„   r…   s   @rU   rW  rW  î  s   ø„ ô1
ðf! E×$4Ñ$4÷ !rT   rW  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚJanusVQVAEDecoderc           	      óv  •— t         ‰| �  «        t        |j                  «      | _        |j
                  | _        |j                  }|j                  }|j                  }||j                  | j                  dz
     z  }t        j                  j                  ||ddd¬«      | _        t        ||«      | _        t        j                  «       | _        t#        t%        | j                  «      «      D ]÷  }t        j                  «       }t        j                  «       }||j                  |   z  }	t%        | j
                  dz   «      D ]N  }
|j'                  t)        |||	¬«      «       |	}|| j                  dz
  k(  sŒ5|j'                  t+        |«      «       ŒP t        j,                  «       }||_        ||_        |dk7  rt3        |«      |_        | j                   j'                  |«       Œù t        j                  j7                  d|dd¬	«      | _        t        j                  j                  ||ddd¬«      | _        y )
Nrd   r   rB  rQ  r   r^   rY  TrZ  )r~   rË   r^  rf   r_  rg   rc   r]   ra   r˜   r   rF  r`  rN  rh  r
  ÚupÚreversedr  rc  r:  r<  rd  re  rf  r@  Úupsampleri  rj  rk  )r€   rˆ   rc   r]   ra   rm  rl  re  rf  rn  ro  rv  r‚   s               €rU   rË   zJanusVQVAEDecoder.__init__<  sç  ø€ Ü‰ÑÔä" 6×#<Ñ#<Ó=ˆÔØ$×3Ñ3ˆÔØ×,Ñ,ˆØ ×0Ñ0ˆØ×*Ñ*ˆð ! 6×#<Ñ#<¸T×=QÑ=QÐTUÑ=UÑ#VÑVˆô —x‘x—‘ ¸ÈaÐXYÐcd�ÓeˆŒô & f¨hÓ7ˆŒô —-‘-“/ˆŒÜ¤ d×&:Ñ&:Ó ;Ó<ò 	ˆGÜ—M‘M“OˆEÜ—=‘=“?ˆDØ%¨×(AÑ(AÀ'Ñ(JÑJˆIÜ  ×!4Ñ!4°qÑ!8Ó9ò 
?�Ø—‘Ü)Ø%Ø$,Ø%.ôôð %�Ø˜d×2Ñ2°QÑ6Ó6Ø—K‘KÔ 3°HÓ =Õ>ð
?ô —‘“ˆBØˆBŒHØˆBŒGØ˜!Š|Ü4°XÓ>�”Ø�G‰G�N‰N˜2Õð)	ô. Ÿ™×*Ñ*°bÀxÐUYÐbfÐ*ÓgˆŒÜŸ™Ÿ™¨°,ÈAÐVWÐab˜Ócˆ�rT   rr  r´   c                 ób  — | j                  |«      }| j                  |«      }t        | j                  «      D ]º  }t        | j                  dz   «      D ]l  } | j
                  |   j                  |   |«      }t        | j
                  |   j                  «      dkD  sŒK | j
                  |   j                  |   |«      }Œn || j                  dz
  k7  sŒ�| j
                  |   j                  |«      }Œ¼ | j                  |«      }|t        j                  |«      z  }| j                  |«      }|S )Nrd   r   )r`  rh  r  r_  rg   rv  re  r^  rf  rx  rj  r˜   rq  rk  )r€   rr  rl  ro  s       rU   rÄ   zJanusVQVAEDecoder.forwardj  s   € Ø—|‘| LÓ1ˆð —x‘x Ó-ˆô ˜T×1Ñ1Ó2ò 	GˆGÜ  ×!4Ñ!4°qÑ!8Ó9ò P�Ø>˜tŸw™w wÑ/×5Ñ5°gÑ>¸|ÓL�Ü�t—w‘w˜wÑ'×,Ñ,Ó-°Ó1Ø#A 4§7¡7¨7Ñ#3×#8Ñ#8¸Ñ#AÀ,Ó#O‘LðPð ˜$×.Ñ.°Ñ2Ó2Ø#Ÿw™w wÑ/×8Ñ8¸ÓF‘ð	Gð —}‘} \Ó2ˆØœŸ™ lÓ3Ñ3ˆØ—}‘} \Ó2ˆØÐrT   )rF   rG   rH   rË   r˜   rª   rÄ   r„   r…   s   @rU   rt  rt  ;  s)   ø„ ô,dð\ E×$5Ñ$5ð ¸%×:KÑ:K÷ rT   rt  c                   óÜ   ‡ — e Zd Zg d¢ZeedœZdZdefˆ fd„Z	de
j                  de
j                  fd„Zeede
j                  dee
j                  e
j                  f   fd	„«       «       Zˆ xZS )
Ú
JanusVQVAE)r<  r:  r)  r  r²   rˆ   c                 ór   •— t         ‰| �  |«       t        |«      | _        d| _        | j                  «        y r  )r~   rË   rt  ÚdecoderÚgradient_checkpointingÚ	post_initrû   s     €rU   rË   zJanusVQVAE.__init__‹  s0   ø€ Ü‰Ñ˜Ô Ü(¨Ó0ˆŒØ&+ˆÔ#ð 	�‰ÕrT   r,  r´   c                 ó   — |j                   d   | j                  j                  d   | j                  j                  d   z  k7  rMt        d| j                  j                  d   | j                  j                  d   z  › d|j                   › d�«      ‚| j                  j	                  |«      }| j                  |«      }| j                  |«      }|S )aG  
        Decodes quantized token IDs into pixel values.
        Args:
            image_tokens (torch.LongTensor): Batch of token IDs.
        Returns:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
                Pixel values decoded from the token IDs.
        rd   r   z4Expected `image_tokens` to have shape `(batch_size, z)`, but got shape `z`.)rš   Úquantizer+  rÎ   r7  Úpost_quant_convr}  )r€   r,  Úcodebook_entryrß   r²   s        rU   ÚdecodezJanusVQVAE.decode“  sã   € ð ×Ñ˜aÑ  D§M¡M×$BÑ$BÀ1Ñ$EÈÏÉ×HfÑHfÐghÑHiÑ$iÒiÜØFÀtÇ}Á}×GeÑGeÐfgÑGhÐko×kxÑkx÷  lJñ  lJð  KLñ  lMñ  HMð  GNð N"Ø".×"4Ñ"4Ð!5°Rð9óð ð Ÿ™×9Ñ9¸,ÓGˆØ×,Ñ,¨^Ó<ˆØ—|‘| MÓ2ˆØÐrT   c                 óÌ   — |j                   d   } | j                  |fddi|¤Ž}| j                  |j                  j	                  |d«      «      }t        ||j                  «      S )Nr   Úreturn_dictTr’   )rš   Úencoder„  r,  rå   r§   r©   )r€   r²   r�   ré   Úencode_outputsr¨   s         rU   rÄ   zJanusVQVAE.forward¦  se   € ð "×'Ñ'¨Ñ*ˆ
Ø$˜Ÿ™ \ÑN¸tÐNÀvÑNˆØ#Ÿ{™{¨>×+FÑ+F×+KÑ+KÈJÐXZÓ+[Ó\ÐäÐ 4°n×6SÑ6SÓTÐTrT   )rF   rG   rH   r    r:  r<  r  Úmain_input_namerW   rË   r˜   r8  rª   r„  r   r   rM   rÄ   r„   r…   s   @rU   r{  r{    sœ   ø„ òÐð /Ø)ñÐð %€OðÐ/õ ð 5×#3Ñ#3ð ¸×8IÑ8Ió ð& Øð	Uà×'Ñ'ð	Uð 
ˆu× Ñ  %×"3Ñ"3Ð3Ñ	4ò		Uó ó ô	UrT   r{  c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚJanusVQVAEAlignerMLPrˆ   c           	      ó‚  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  t        d|j                  «      D �cg c],  }t        j                  |j
                  |j
                  «      ‘Œ. c}«      | _
        t        |j                     | _        y c c}w r   )r~   rË   r   rÓ   rY   r?   r÷   r
  r  r4   r!  r	   r9   rö   r  s      €rU   rË   zJanusVQVAEAlignerMLP.__init__µ  sŒ   ø€ Ü‰ÑÔä—9‘9˜V×-Ñ-¨v×/DÑ/DÓEˆŒÜŸ]™]ÜNSÐTUÐW]×WoÑWoÓNpÖqÈŒR�Y‰Y�v×,Ñ,¨f×.CÑ.CÕDÒqó
ˆÔô $ F×$5Ñ$5Ñ6ˆÕùò rr"  c                 ó|   — | j                  |«      }| j                  D ]  }| j                  |«      } ||«      }Œ |S rò   r$  r%  s      rU   rÄ   zJanusVQVAEAlignerMLP.forward¾  r'  rT   )rF   rG   rH   rW   rË   rÄ   r„   r…   s   @rU   r‹  r‹  ´  s   ø„ ð7Ð/õ 7örT   r‹  c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )ÚJanusVQVAEHeadzOHead used for sampling tokens in image generation, replacing the usual lm head.rˆ   c                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                     | _	        t        j                  |j
                  |j                  «      | _        y rò   )r~   rË   r   rÓ   ri   r?   Úproj_outr	   r9   rö   rZ   Úvision_headrû   s     €rU   rË   zJanusVQVAEHead.__init__É  s^   ø€ Ü‰ÑÔÜŸ	™	 &×">Ñ">À×@UÑ@UÓVˆŒÜ# F×$5Ñ$5Ñ6ˆÔÜŸ9™9 V×%:Ñ%:¸F×<QÑ<QÓRˆÕrT   rß   r´   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rò   )r‘  rö   r’  rý   s     rU   rÄ   zJanusVQVAEHead.forwardÏ  s6   € ØŸ™ mÓ4ˆØ×*Ñ*¨=Ó9ˆØ×(Ñ(¨Ó7ˆØÐrT   )rF   rG   rH   rI   rW   rË   r˜   rÅ   ÚtensorrÄ   r„   r…   s   @rU   r�  r�  Æ  s0   ø„ ÙYðSÐ/õ Sð U§\¡\ð °e·l±l÷ rT   r�  zl
    The Janus model which consists of a siglip vision backbone, a Llama language model and a VQ model.
    c                   ó¶  ‡ — e Zd Zdefˆ fd„Zeedej                  de	e
   deez  fd„«       «       Zdej                  dej                  d	ej                  fd
„Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                   dz  dej                  dz  dedz  dej                  dz  dedz  deej                   z  defd„«       «       Zˆ xZS )Ú
JanusModelrˆ   c                 óÀ  •— t         ‰| �  |«       || _        t        j	                  |j
                  «      | _        t        | j                  j                  «      | _        t        j	                  |j                  «      | _        t        j                  | j                  j                  j                  | j                  j                  j                  «      | _        t#        | j                  j                  «      | _        t'        | j                  j                  «      | _        t+        j,                  |j.                  ¬«      | _        d| _        | j5                  «        y )N)rˆ   F)r~   rË   rˆ   r  Ú_from_configrq   Úvision_modelr  Úalignerr{  rr   Úvqmodelr   Ú	EmbeddingrZ   rY   Úgeneration_embeddingsr‹  Úgeneration_alignerr�  Úgeneration_headr    Úfrom_configrp   Úlanguage_modelr~  r  rû   s     €rU   rË   zJanusModel.__init__Ü  sñ   ø€ Ü‰Ñ˜Ô ØˆŒä,×9Ñ9¸&×:NÑ:NÓOˆÔÜ,¨T×->Ñ->×-EÑ-EÓFˆŒä!×.Ñ.¨v×/?Ñ/?Ó@ˆŒô &(§\¡\°$·,±,×2EÑ2E×2TÑ2TÐVZ×VbÑVb×ViÑVi×VsÑVsÓ%tˆÔ"Ü"6°t·|±|×7JÑ7JÓ"KˆÔÜ-¨d¯l©l×.AÑ.AÓBˆÔä'×3Ñ3¸6×;MÑ;MÔNˆÔà&+ˆÔ#à�‰ÕrT   r²   r�   r´   c                 óp   —  | j                   |fddi|¤Ž}| j                  |j                  «      |_        |S )Nr†  T)r™  rš  r  r  )r€   r²   r�   Úvision_outputss       rU   Úget_image_featureszJanusModel.get_image_featuresñ  s=   € ð
 +˜×*Ñ*¨<ÑTÀTÐTÈVÑTˆØ'+§|¡|°N×4TÑ4TÓ'UˆÔ$àÐrT   Ú	input_idsr  Úimage_featuresc                 ó.  — |€m| | j                  «       t        j                  | j                  j                  t        j
                  |j                  ¬«      «      k(  }|j                  d«      }n|| j                  j                  k(  }|j                  «       }|j                  d   |j                  d   z  }|j                  d«      j                  |j                  «      }t        ||j                  d   z  |j                  «       k(  d|› d|› �«       |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        ©r¶   Údevicer’   r   rd   z6Image features and image tokens do not match, tokens: z, features: )Úget_input_embeddingsr˜   r”  rˆ   rs   Úlongr©  ÚallÚsumrš   Ú	unsqueezer¹   r   Únumel)r€   r¥  r  r¦  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          rU   Úget_placeholder_maskzJanusModel.get_placeholder_maskû  s  € ð ÐØ!.Ð2M°$×2KÑ2KÓ2MÜ—‘˜TŸ[™[×7Ñ7¼u¿z¹zÐR_×RfÑRfÔgó3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¨d¯k©k×.HÑ.HÑ!HÐà+×/Ñ/Ó1ˆØ)×/Ñ/°Ñ2°^×5IÑ5IÈ!Ñ5LÑLÐØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐÜØ˜]×0Ñ0°Ñ4Ñ4¸×8LÑ8LÓ8NÑNØDÀ^ÐDTÐT`ÐaqÐ`rÐsô	
ð "Ð!rT   Nrà   r—   r�   Ú	use_cacheÚlogits_to_keepc	           
      ó"  — |d u |d uz  rt        d«      ‚|€ | j                  «       |«      }|�ˆ| j                  |d¬«      j                  }
|
j	                  d|j
                  d   «      }|j                  |j                  |j                  «      }| j                  |||¬«      }|j                  ||«      } | j                  d||||||dœ|	¤Ž}t        |j                  |j                  |j                  |j                   |�
¬«      S d ¬«      S )	NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either oneT)r†  r’   )r  r¦  )r  rà   r—   r�   r´  rµ  )r  r�   rß   r  Úimage_hidden_statesrS   )rÎ   rª  r¤  r  rä   rš   r¹   r©  r¶   r³  Úmasked_scatterr¡  r¬   r  r�   rß   r  )r€   r¥  r²   rà   r—   r�   r  r´  rµ  r�   Úimage_embedsr¦  Úimage_attention_maskÚ	lm_outputs                 rU   rÄ   zJanusModel.forward  sP  € ð ˜Ð -°tÐ";Ò<ÜØsóð ð Ð Ø7˜D×5Ñ5Ó7¸	ÓBˆMàÐ#Ø×2Ñ2°<ÈTÐ2ÓR×`Ñ`ˆLØ)×1Ñ1°"°m×6IÑ6IÈ"Ñ6MÓNˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ#'×#<Ñ#<Ø¨À~ð $=ó $Ð ð *×8Ñ8Ð9MÈ~Ó^ˆMà'�D×'Ñ'ð 
Ø'Ø)Ø%Ø+ØØ)ñ
ð ñ
ˆ	ô ,Ø'×9Ñ9Ø%×5Ñ5Ø#×1Ñ1Ø ×+Ñ+Ø0<Ð0H ô
ð 	
ð
 OSô
ð 	
rT   )NNNNNNNr   )rF   rG   rH   rn   rË   r   r   r˜   rª   r   r   rM   r   r¤  r8  r³  rÅ   r
   rP   rJ   r¬   rÄ   r„   r…   s   @rU   r–  r–  Ö  sX  ø„ ð˜{õ ð* ØðØ!×-Ñ-ðØ9?Ð@RÑ9Sðà	Ð+Ñ	+òó ó ðð"Ø×)Ñ)ð"Ø:?×:KÑ:Kð"Ø]b×]nÑ]nó"ð0 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø!%Ø-.ñ,
à×#Ñ# dÑ*ð,
ð ×'Ñ'¨$Ñ.ð,
ð Ÿ™ tÑ+ð	,
ð
 ×&Ñ&¨Ñ-ð,
ð  ™ð,
ð ×(Ñ(¨4Ñ/ð,
ð ˜$‘;ð,
ð ˜eŸl™lÑ*ð,
ð 
&ò,
ó ó ô,
rT   r–  c                   óJ  ‡ — e Zd ZddiZdZdZdefˆ fd„Zdej                  dej                  fd	„Z
ee	 	 	 	 	 	 	 	 	 ddej                  d
z  dej                  d
z  dej                  d
z  dej                  d
z  ded
z  dej                  d
z  dej                  d
z  ded
z  deej                  z  dee   defd„«       «       Z	 	 	 	 	 	 dˆ fd„	Zdej                  fd„Z ej0                  «       	 	 	 ddej                  d
z  dej                  d
z  ded
z  fˆ fd„«       Zˆ xZS )ÚJanusForConditionalGenerationzlm_head.weightz(model.language_model.embed_tokens.weightrŠ   Trˆ   c                 óü   •— t         ‰| �  |«       || _        t        |«      | _        t        j                  |j                  j                  |j                  j                  d¬«      | _
        | j                  «        y )NFrÉ   )r~   rË   rˆ   r–  r‰   r   rÓ   rp   r2   Ú
vocab_sizeÚlm_headr  rû   s     €rU   rË   z&JanusForConditionalGeneration.__init__I  s\   ø€ Ü‰Ñ˜Ô ØˆŒÜ Ó'ˆŒ
Ü—y‘y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÔkˆŒð 	�‰ÕrT   Úinputsr´   c                 ór   — | j                   j                  |«      }| j                   j                  |«      }|S rò   )r‰   r�  rž  )r€   rÁ  rr  s      rU   Ú'prepare_embeddings_for_image_generationzEJanusForConditionalGeneration.prepare_embeddings_for_image_generationR  s0   € Ø—z‘z×7Ñ7¸Ó?ˆØ—z‘z×4Ñ4°\ÓBˆØÐrT   Nr¥  r²   rà   r—   r�   r  Úlabelsr´  rµ  r�   c
                 ó¤  —  | j                   d|||||||dœ|
¤Ž}|j                  }t        |	t        «      rt	        |	 d«      n|	}| j                  |dd…|dd…f   «      }d}|�4 | j                  d||| j                  j                  j                  dœ|
¤Ž}t        |||j                  |j                  |j                  |j                  ¬«      S )a­  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        )r¥  r²   rà   r—   r�   r  r´  N)ÚlogitsrÄ  r¿  )ÚlossrÆ  r�   rß   r  r·  rS   )r‰   r  rx   rJ   ÚslicerÀ  Úloss_functionrˆ   rp   r¿  r°   r�   rß   r  r·  )r€   r¥  r²   rà   r—   r�   r  rÄ  r´  rµ  r�   Úoutputsrß   Úslice_indicesrÆ  rÇ  s                   rU   rÄ   z%JanusForConditionalGeneration.forwardW  sô   € ð* �$—*‘*ð 	
ØØ%Ø)Ø%Ø+Ø'Øñ	
ð ñ	
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%ð Ø f¸¿¹×9PÑ9P×9[Ñ9[ñØ_eñˆDô +ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ô
ð 	
rT   c           	      óf   •— t        ‰
| �  |f|||||dœ|¤Ž}	|s|j                  dd«      s||	d<   |	S )N)r�   r  rà   rµ  Úis_first_iterationr´  Tr²   )r~   Úprepare_inputs_for_generationrz   )r€   r¥  r²   r�   rà   r  rµ  rÍ  r�   Úmodel_inputsr‚   s             €rU   rÎ  z;JanusForConditionalGeneration.prepare_inputs_for_generationŠ  sW   ø€ ô ‘wÑ<Øð
à+Ø'Ø)Ø)Ø1ñ
ð ñ
ˆñ  V§Z¡Z°¸TÔ%BØ+7ˆL˜Ñ(àÐrT   r,  c                 óx   — | j                   j                  j                  |«      }|j                  dddd«      }|S )a,  
        Decodes generated image tokens from language model to continuous pixel values
        with VQGAN module via upsampling.
        Args:
            image_tokens (`torch.LongTensor` of shape `(batch_size, num_of_tokens)`):
                The tensors corresponding to the input images.
        r   r   r   rd   )r‰   r›  r„  r3  )r€   r,  Údecoded_images      rU   Údecode_image_tokensz1JanusForConditionalGeneration.decode_image_tokensª  s:   € ð Ÿ
™
×*Ñ*×1Ñ1°,Ó?ˆØ%×-Ñ-¨a°°A°qÓ9ˆØÐrT   Úlogits_processorc           	      ó.  •— |j                  dd«      } | j                  |j                  dd «      fi |¤Ž\  }}|dk(  rt        ‰%| �  d|||d dœ|¤ŽS |j	                  «       t
        j                  t
        j                  fvrt        d«      ‚|j                  «        | j                  |j                  «       «       |�|n	t        «       }d|d<   |j                  €t        j                  d«       d	|_        |j                  |d
<   | j!                  ||j"                  |«      \  }}	}|j$                  |j&                  }}
t)        |j*                  «      dk7  rt        d|j*                  › d�«      ‚|d u}| j-                  |||j&                  ¬«       |j                  r:|j                  dkD  r+|j/                  t1        |j                  «      «       d |_        | j3                  ||j*                  d   |d ||¬«      } | j4                  d|||j6                  dœ|¤Ž\  }}| j8                  j:                  j<                  j>                  }|j*                  \  }}|jA                  dd«      }|j                  dd «      }|jA                  dd«      }||d<   ||d …d d …f   |j"                  k7  ||d …d d …f   |jB                  d   k7  z  }||d …d d …f   jE                  ||jF                  «        | jI                  «       |«      }|jK                  dd «      €@| jM                  |jN                  xs d|dz  tQ        |jR                  ||z   «      |¬«      |d<   tU        jV                  ||f|
|¬«      }|jX                  }|jZ                  }|j\                  }|j^                  }|j`                  }|r|rdnd }|r|rdnd }|r|rdnd }|r|rdnd }tc        |«      D �]\  } | jd                  d||ddœ|¤Ž}d|v r!|d   jg                  |j&                  «      |d<    | j8                  jh                  di |¤||dœ¤Ž}| jk                  ||«      }|jl                  d d …dd d …f   jo                  «       } | j8                  jq                  | «      }! |||!«      }"|jr                  r>tU        jt                  |"d¬«      }#tU        jv                  |#d¬«      jy                  d«      }$ntU        jz                  |"d¬«      }$|$|d d …|f<   tU        j|                  |$|$g«      }$|$j                  d«      }$| j�                  |$«      }�Œ_ |r@|r|!fz  }|r| jƒ                  «       fz  }|r|j„                  z  }|r|j†                  z  }|rt‰        |!|||jŠ                  ¬«      S |S )NÚgeneration_moderŒ   Úgeneration_config)rÁ  rà   rÖ  Úguidance_scalez’Got incompatible mode for Image Generation, should be one of greedy or sampling. Ensure that beam search is de-activated by setting `num_beams=1`.Tr´  zU`guidance_scale` is required for CFG but not provided. Setting to default value of 5.é   r×  r   z;Expected input ids of shape (batch_size, seq_len), but got z3Passing `inputs embeds` is not supported currently.)r©  rd   )rÖ  Úinput_ids_seq_lengthÚencoder_input_idsÚprefix_allowed_tokens_fnrÓ  r©  )r¥  rà   Úexpand_sizerà   Úboi_token_idr�   Ústatic)Úcache_implementationré   Úmax_cache_lenÚmodel_kwargsr¨  rS   )r  r¥  rÍ  )Úoutput_attentionsÚoutput_hidden_statesr’   )r/  )Únum_samples)Ú	sequencesÚscoresrÆ  r  rß   r�   )FÚpopÚ_prepare_generation_configr~   ÚgenerateÚget_generation_moder   ÚSAMPLEÚGREEDY_SEARCHrÎ   ÚvalidateÚ_validate_model_kwargsÚcopyr   r×  r{   ÚwarningÚ_prepare_model_inputsÚbos_token_idr¶   r©  r^  rš   Ú_prepare_special_tokensrc  r   Ú_get_logits_processorÚ_expand_inputs_for_generationÚnum_return_sequencesr‰   r™  rˆ   rE   ÚrepeatÚgeneration_kwargsÚmasked_fill_Úpad_token_idrª  rz   Ú_prepare_static_cacherß  ÚmaxÚ
max_lengthr˜   Úzerosrâ  rã  Úoutput_scoresÚoutput_logitsÚreturn_dict_in_generater  rÎ  r¹   r¡  Ú#_update_model_kwargs_for_generationr  ÚclonerŸ  Ú	do_sampleÚsoftmaxÚmultinomialÚsqueezeÚargmaxÚcatr®  rÃ  rO   r  rß   r   r�   )&r€   rÁ  rà   rÓ  r�   rÕ  rÖ  rá  r¥  Úmodel_input_namer¶   r©  Úkwargs_has_attention_maskrE   ré   rê   Úinput_tokensÚmaskr  Úgenerated_tokensrâ  rã  rÿ  r   r  Ú
raw_scoresÚ
raw_logitsÚdecoder_hidden_statesÚdecoder_attentionsÚirÏ  rÊ  rr  ræ  Únext_token_scoresÚprobsÚ
next_tokenr‚   s&                                        €rU   ré  z&JanusForConditionalGeneration.generate¶  sF  ø€ ð !Ÿ*™*Ð%6¸Ó?ˆØ*I¨$×*IÑ*IØ�J‰JÐ*¨DÓ1ñ+
Ø5;ñ+
Ñ'Ð˜<ð
 ˜fÒ$ä‘7Ñ#ð ØØ-Ø"3Ø#ñ	ð
 ñð ð ×0Ñ0Ó2¼>×;PÑ;PÔR`×RnÑRnÐ:oÑoÜðTóð ð 	×"Ñ"Ô$Ø×#Ñ# L×$5Ñ$5Ó$7Ô8ð 0@Ð/KÑ+ÔQdÓQfÐð %)ˆ�[Ñ!à×+Ñ+Ð3Ü�N‰NÐrÔsØ/0ÐÔ,Ø):×)IÑ)IˆÐ%Ñ&ð 59×4NÑ4NØÐ%×2Ñ2°Ló5
Ñ1ˆ	Ð# \ð "Ÿ™¨×)9Ñ)9ˆvˆäˆy�‰Ó 1Ò$ÜØMÈiÏoÉoÐM^ØEðFóð ð %3¸$Ð$>Ð!Ø×$Ñ$Ð%6Ð8QÐZc×ZjÑZjÐ$Ôkð ×+Ò+Ð0A×0PÑ0PÐSTÒ0TØ×#Ñ#Ô$IÐJ[×JjÑJjÓ$kÔlØ/3ÐÔ,ð  ×5Ñ5Ø/Ø!*§¡°Ñ!3Ø'Ø%)Ø-Øð 6ó 
Ðð #E $×"DÑ"Dð #
ØØ)Ø)×>Ñ>ñ#
ð ñ	#
Ñˆ	�<ð  Ÿ:™:×2Ñ2×9Ñ9×JÑJÐØ'Ÿo™oÑˆ
�Gà ×'Ñ'¨¨1Ó-ˆØ%×)Ñ)Ð*:¸DÓAˆØ'×.Ñ.¨q°!Ó4ˆØ)7ˆÐ%Ñ&ð ˜Z™[ª!˜^Ñ,Ð0A×0NÑ0NÑNØ˜™¢a˜Ñ(Ð,=×,OÑ,OÐP^Ñ,_Ñ_ñ
ˆð 	�Z‘[¢!�^Ñ$×1Ñ1°$Ð8I×8VÑ8VÔWà3˜×1Ñ1Ó3°LÓAˆà×ÑÐ-¨tÓ4Ð<à.2×.HÑ.HØ%6×%KÑ%KÒ%WÈxà%¨™>ä!Ð"3×">Ñ">Ð@PÐSZÑ@ZÓ[Ø)ð /Ió /ˆLÐ*Ñ+ô !Ÿ;™;¨
Ð4DÐ'EÈUÐ[aÔbÐð .×?Ñ?ÐØ0×EÑEÐØ)×7Ñ7ˆØ)×7Ñ7ˆØ"3×"KÑ"KÐá3¹‘RÈDˆ
Ù3¹‘RÈDˆ
Ù'>ÑCW¡Ð^bÐÙ$;Ñ@Q™RÐX\ÐäÐ'Ó(ó (	UˆAð >˜4×=Ñ=ð Ø+°|ÐX\ñØ`lñˆLð   <Ñ/Ø1=Ð>NÑ1O×1RÑ1RÐS`×SgÑSgÓ1h�Ð-Ñ.à/�d—j‘j×/Ñ/ñ Øðà"3Ø%9óˆGð  ×CÑCÀGÈ\ÓZˆLØ"×4Ñ4²Q¸ºA°XÑ>×DÑDÓFˆLð —Z‘Z×/Ñ/°Ó=ˆFÙ 0°¸FÓ CÐð !×*Ò*ÜŸ™Ð&7¸RÔ@�Ü"×.Ñ.¨uÀ!ÔD×LÑLÈRÓP‘
ä"Ÿ\™\Ð*;ÀÔD�
à%/ÐšQ ˜TÑ"ô Ÿ™ J°
Ð#;Ó<ˆJØ#×-Ñ-¨bÓ1ˆJà ×HÑHÈÓTŠMðQ(	UñT #ÙØ˜v˜iÑ'�
ÙØ˜|×1Ñ1Ó3Ð5Ñ5�
Ù Ø" g×&8Ñ&8Ñ8Ð"Ù#Ø%¨×)>Ñ)>Ñ>Ð%á"Ü,Ø*ØØ!Ø-Ø3Ø '× 7Ñ 7ôð ð $Ð#rT   )	NNNNNNNNr   )NNNNNF)NNN)rF   rG   rH   Ú_tied_weights_keysÚoutput_modalitiesr¤   rn   rË   r˜   rÅ   rÃ  r   r   r8  rª   r
   rP   rJ   r   r   r°   rÄ   rÎ  rÒ  Úno_gradr   ré  r„   r…   s   @rU   r½  r½  D  sÊ  ø„ Ø*Ð,VÐWÐØ)ÐØ!Ðð˜{õ ð¸e¿l¹lð ÈuÏ|É|ó ð
 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ/
à×#Ñ# dÑ*ð/
ð ×'Ñ'¨$Ñ.ð/
ð Ÿ™ tÑ+ð	/
ð
 ×&Ñ&¨Ñ-ð/
ð  ™ð/
ð ×(Ñ(¨4Ñ/ð/
ð × Ñ  4Ñ'ð/
ð ˜$‘;ð/
ð ˜eŸl™lÑ*ð/
ð Ð+Ñ,ð/
ð 
%ò/
ó ó ð/
ðh ØØØØØ õð@
°·±ó 
ð €U‡]�]ƒ_ð '+Ø26Ø7;ñ	$à—‘˜tÑ#ð$ð ×(Ñ(¨4Ñ/ð$ð .°Ñ4ô	$ó ô$rT   r½  )r‡   r½  r–  r{  r  rW   r1   rn   )dÚcollections.abcr   Údataclassesr   r˜   Útorch.nn.functionalr   Ú
functionalr1  Úhuggingface_hub.dataclassesr   Ú r   r•   Úactivationsr	   Úcache_utilsr
   Úconfiguration_utilsr   Ú
generationr   r   r   r   Úgeneration.utilsr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úautor   r   r    Úblip_2.modeling_blip_2r!   Ú!chameleon.configuration_chameleonr"   Úchameleon.modeling_chameleonr#   r$   r%   r&   r'   Úidefics.modeling_ideficsr(   r)   Úllama.modeling_llamar*   Úsiglip.configuration_siglipr+   Úsiglip.modeling_siglipr,   r-   r.   Ú
get_loggerrF   r{   r1   rW   rn   r‡   r§   r¬   r°   r”   rd  rÇ   rô   rŽ   r  r  r  r)  r:  r<  r>  r@  rN  rW  rt  r{  r‹  r�  r–  r½  Ú__all__rS   rT   rU   ú<module>r3     sÊ  ðõ %Ý !ã ß Ð Ý .Ý å &Ý !Ý  Ý 3ß uÓ uÝ 9ß XÑ Xß FÝ &÷÷ ÷ 9Ñ 8Ý 5Ý D÷õ ÷ eÝ :Ý <ß ^Ñ ^ñ ÔØà	ˆ×	Ñ	˜HÓ	%€ñ
 Ð<Ô=Øô)Ð*ó )ó ó >ð)ñ2 Ð<Ô=Øô#!Ð+ó #!ó ó >ð#!ñL Ð<Ô=Øô=(Ð"ó =(ó ó >ð=(ð@ ôi˜?ó ió ðiñ$ ðôð
 ô	4�{ó 	4ó óð	4ô	Ð#Aô 	ô	Ð"?ô 	ôÐ2ô ô"I$˜2Ÿ9™9ô I$ôX�R—Y‘Yô ô(*Ð0ô *ôp˜ô pô#
Ð'ô #
ôL˜BŸI™Iô ô$"Ð =ô "ô*	Ð<ô 	ô	Ð8ô 	ô	ÐBô 	ô˜RŸY™Yô ô˜Ÿ™ô ô,J!˜Ÿ	™	ô J!ôZA˜Ÿ	™	ô AôH2U�ô 2Uôj˜2Ÿ9™9ô ô$�R—Y‘Yô ñ  ðôô
f
Ð%ó f
óð
f
ôRr$Ð$8¸/ô r$òj		�rT   