Ë
    ÛÓ^j¶ ã                   óÔ  — d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dl	Z	d dl
mZ d dlmc mZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6 ddl7m8Z8m9Z9m:Z: e/e G d„ de#«      «       «       Z;d„ Z< ed«      ded„«       Z=de	j|                  de?de	j|                  fd„Z@	 dfd ej‚                  d!e	j|                  d"e	j|                  d#e	j|                  d$e	j|                  dz  d%eBd&eBd'e,e.   fd(„ZC ee=«       G d)„ d*ej‚                  «      «       ZD ed+«       G d,„ d-ej‚                  «      «       ZE G d.„ d/ej‚                  «      ZF G d0„ d1e «      ZG G d2„ d3ej‚                  «      ZH G d4„ d5ej‚                  «      ZI G d6„ d7ej‚                  «      ZJ G d8„ d9ej‚                  «      ZK G d:„ d;ej‚                  «      ZL G d<„ d=ej‚                  «      ZM G d>„ d?ej‚                  «      ZN G d@„ dAej‚                  «      ZO G dB„ dCej‚                  «      ZP G dD„ dEej‚                  «      ZQ G dF„ dGej¤                  «      ZS G dH„ dIej‚                  «      ZT G dJ„ dKej‚                  «      ZU G dL„ dMej‚                  «      ZV G dN„ dOej‚                  «      ZW G dP„ dQej‚                  «      ZX e/dR¬S«       G dT„ dUe*«      «       ZY G dV„ dW«      ZZe/ G dX„ dYe*«      «       Z[ G dZ„ d[ej‚                  «      Z\e/ G d\„ d]e[«      «       Z]e/ G d^„ d_e[e«      «       Z^ G d`„ dae[«      Z_ G db„ dce[e«      Z`g dd¢Zay)gé    N)ÚCallable)Ú	dataclass)Úcached_property)ÚOptionalé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú
Emu3ConfigÚEmu3TextConfigÚEmu3VQVAEConfigc                   ó:   — e Zd ZU dZdZej                  dz  ed<   y)ÚEmu3VQVAEModelOutputz™
    image_tokens (`torch.LongTensor` of shape `(batch_size, config.vocab_size`):
        Indices of the image tokens predicted by the VQ-VAE model.
    NÚimage_tokens)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r'   ÚtorchÚ
LongTensorÚ__annotations__© ó    úq/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/emu3/modeling_emu3.pyr&   r&   1   s   … ñð
 -1€L�%×"Ñ" TÑ)Ô0r0   r&   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Néÿÿÿÿé   ©Údim)Úshaper,   Úcat)ÚxÚx1Úx2s      r1   Úrotate_halfr<   <   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r0   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer<   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r1   Úapply_rotary_pos_embrG   C   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr0   Úhidden_statesÚn_repÚreturnc                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r!   N)r7   ÚexpandÚreshape)rH   rI   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r1   Ú	repeat_kvrR   ]   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr0   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )Nr4   r   r3   )r6   Údtype)ÚpÚtrainingr!   )rR   Únum_key_value_groupsr,   ÚmatmulÚ	transposeÚnnÚ
functionalÚsoftmaxÚfloat32Útor\   rY   r^   Ú
contiguous)rS   rT   rU   rV   rW   rX   rY   rZ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r1   Úeager_attention_forwardrl   i   sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r0   c                   óø   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
dz  d
ee   de	ej                  ej                  f   fd„Zˆ xZS )ÚEmu3Attentionú=Multi-headed attention from 'Attention Is All You Need' paperÚconfigÚ	layer_idxc                 ód  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                  ¬«      | _        y )NrQ   ç      à¿T©Úbias)ÚsuperÚ__init__rp   rq   ÚgetattrÚhidden_sizeÚnum_attention_headsrQ   rO   r_   rX   Úattention_dropoutÚ	is_causalrb   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_proj©Úselfrp   rq   Ú	__class__s      €r1   rw   zEmu3Attention.__init__†   sM  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆ�r0   NrH   Úposition_embeddingsrW   Úpast_key_valuesrZ   rJ   c                 ó
  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  sdn| j                   | j"                  dœ|¤Ž\  }} |j$                  g |¢d‘­Ž j'                  «       }| j)                  |«      }||fS )Nr3   r!   r4   ç        )rY   rX   )r7   rQ   r   Úviewra   r€   r�   rG   Úupdaterq   r   Úget_interfacerp   Ú_attn_implementationrl   r^   r{   rX   rM   rg   r‚   )r„   rH   r†   rW   r‡   rZ   Úinput_shapeÚhidden_shapeÚquery_statesrh   ri   rB   rC   Úattention_interfacerk   rj   s                   r1   ÚforwardzEmu3Attention.forward�   s‘  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r0   ©NNN)r(   r)   r*   r+   r"   Úintrw   r,   ÚTensorÚtupler
   r   r   r’   Ú__classcell__©r…   s   @r1   rn   rn   ‚   s¥   ø„ áGð
˜zð 
°cõ 
ð4 IMØ.2Ø(,ñ&)à—|‘|ð&)ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð&)ð Ÿ™ tÑ+ð	&)ð
  ™ð&)ð Ð+Ñ,ð&)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷&)r0   rn   ÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚEmu3RMSNormÚepsrJ   Nc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z:
        Emu3RMSNorm is equivalent to T5LayerNorm
        N)rv   rw   rb   Ú	Parameterr,   ÚonesÚweightÚvariance_epsilon)r„   ry   rœ   r…   s      €r1   rw   zEmu3RMSNorm.__init__È   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr0   rH   c                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Nr4   r3   T)Úkeepdim)	r\   rf   r,   re   ÚpowÚmeanÚrsqrtr¡   r    )r„   rH   Úinput_dtypeÚvariances       r1   r’   zEmu3RMSNorm.forwardÐ   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r0   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)r–   r    r7   r¡   ©r„   s    r1   Ú
extra_reprzEmu3RMSNorm.extra_repr×   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr0   )ç�íµ ÷Æ°>)
r(   r)   r*   Úfloatrw   r,   r•   r’   r«   r—   r˜   s   @r1   r›   r›   Æ   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJr0   r›   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEmu3MLPc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  | j                  | j                  |j                  ¬«      | _	        t        j                  | j                  | j                  |j                  ¬«      | _
        t        |j                     | _        y )Nrt   )rv   rw   rp   ry   Úintermediate_sizerb   r}   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr	   Ú
hidden_actÚact_fn©r„   rp   r…   s     €r1   rw   zEmu3MLP.__init__Ü   s¸   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Ñ/ˆ�r0   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S ©N)rµ   r·   r³   r´   )r„   r9   rµ   s      r1   r’   zEmu3MLP.forwardæ   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr0   ©r(   r)   r*   rw   r’   r—   r˜   s   @r1   r¯   r¯   Û   s   ø„ ô0ör0   r¯   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fd„Zˆ xZS )ÚEmu3DecoderLayerrp   rq   c                 óh  •— t         ‰| �  «        |j                  | _        t        ||¬«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _	        t        |j                  |j                  ¬«      | _
        t        j                  |j                  «      | _        y )N)rp   rq   ©rœ   )rv   rw   ry   rn   Ú	self_attnr¯   Úmlpr›   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrb   ÚDropoutr{   rY   rƒ   s      €r1   rw   zEmu3DecoderLayer.__init__ì   s�   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä&¨fÀ	ÔJˆŒä˜6“?ˆŒÜ*¨6×+=Ñ+=À6×CVÑCVÔWˆÔÜ(3°F×4FÑ4FÈF×L_ÑL_Ô(`ˆÔ%Ü—z‘z &×":Ñ":Ó;ˆ�r0   NrH   rW   Úposition_idsr‡   Ú	use_cacher†   rZ   rJ   c           
      óü   — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	|| j                  |«      z   }|}| j                  |«      }| j	                  |«      }|| j                  |«      z   }|S )N)rH   rW   rÆ   r‡   rÇ   r†   r/   )rÃ   rÀ   rY   rÄ   rÁ   )
r„   rH   rW   rÆ   r‡   rÇ   r†   rZ   ÚresidualÚ_s
             r1   r’   zEmu3DecoderLayer.forward÷   s    € ð !ˆØ×,Ñ,¨]Ó;ˆà)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð ! 4§<¡<°Ó#>Ñ>ˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ  4§<¡<°Ó#>Ñ>ˆØÐr0   )NNNFN)r(   r)   r*   r"   r”   rw   r,   r•   r-   r
   Úboolr–   r   r   r’   r—   r˜   s   @r1   r½   r½   ë   s»   ø„ ð	<˜zð 	<°cõ 	<ð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷r0   r½   c                   óH   ‡ — e Zd ZdZdefˆ fd„Zdej                  fd„Zˆ xZ	S )ÚEmu3VQVAEVectorQuantizeraâ  
    A module for vector quantization using learned embedding vectors.

    This module implements the quantization process similar to te one described in
    the VQ-VAE (Vector Quantized Variational AutoEncoder) paper. It quantizes continuous
    input vectors into discrete codebook vectors, which are learned during training.
    Current implementation improves over previous ones by avoiding costly matrix multiplications
    and allowing for post-hoc remapping of indices.
    rp   c                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        | j                  j                  j                  j                  d|j                  z  d|j                  z  «       y )Ng      ð¿ç      ð?)
rv   rw   rb   Ú	EmbeddingÚcodebook_sizeÚ	embed_dimÚ	embeddingr    ÚdataÚuniform_r¸   s     €r1   rw   z!Emu3VQVAEVectorQuantizer.__init__!  sb   ø€ Ü‰ÑÔÜŸ™ f×&:Ñ&:¸F×<LÑ<LÓMˆŒØ�‰×Ñ×"Ñ"×+Ñ+¨D°6×3GÑ3GÑ,GÈÈv×OcÑOcÑIcÕdr0   Úhidden_statec                 ó  — |j                   \  }}}}}|j                  ddddd«      j                  «       }|j                  d|«      }t	        j
                  |dz  dd¬«      }t	        j
                  | j                  j                  dz  d¬	«      }	dt	        j                  || j                  j                  j                  dd«      «      z  }
||	z   |
z
  }
t	        j                  |
d¬	«      }|j                  ||||«      }|S )
Nr   r!   r   é   r4   r3   T)r6   r£   r5   )r7   Úpermuterg   rŠ   r,   ÚsumrÓ   r    r`   ra   Úargmin)r„   rÖ   Ú
batch_sizeÚtemporalÚchannelsÚheightÚwidthÚhidden_state_flattenedÚhidden_state_sumÚembedding_sumÚ	distancesÚmin_encoding_indicess               r1   r’   z Emu3VQVAEVectorQuantizer.forward&  sõ   € Ø8D×8JÑ8JÑ5ˆ
�H˜h¨°Ø#×+Ñ+¨A¨q°!°Q¸Ó:×EÑEÓGˆØ!-×!2Ñ!2°2°xÓ!@Ðô !Ÿ9™9Ð%;¸QÑ%>ÀAÈtÔTÐÜŸ	™	 $§.¡.×"7Ñ"7¸Ñ":ÀÔBˆð œŸ™Ð%;¸T¿^¹^×=RÑ=R×=\Ñ=\Ð]^Ð`aÓ=bÓcÑcˆ	Ø$ }Ñ4°yÑ@ˆ	ä$Ÿ|™|¨I¸1Ô=ÐØ3×8Ñ8¸ÀXÈvÐW\Ó]ÐØ#Ð#r0   )
r(   r)   r*   r+   r$   rw   r,   r•   r’   r—   r˜   s   @r1   rÍ   rÍ     s&   ø„ ñðe˜õ eð
$ E§L¡L÷ $r0   rÍ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEmu3VQVAEEncoderConvDownsamplec                 ó`   •— t         ‰| �  «        t        j                  ||ddd¬«      | _        y )Nr   r4   r   ©Úkernel_sizeÚstrideÚpadding©rv   rw   rb   ÚConv2dÚconv©r„   Úin_channelsr…   s     €r1   rw   z'Emu3VQVAEEncoderConvDownsample.__init__9  ó'   ø€ Ü‰ÑÔÜ—I‘I˜k¨;ÀAÈaÐYZÔ[ˆ�	r0   c                 óZ   — t        j                  |ddd¬«      }| j                  |«      }|S )N)r   r!   r   r!   Úconstantr   )ÚpadÚmoderV   )ÚFrõ   rï   ©r„   rH   s     r1   r’   z&Emu3VQVAEEncoderConvDownsample.forward=  s+   € äŸ™˜m°ÀJÐVWÔXˆØŸ	™	 -Ó0ˆØÐr0   r»   r˜   s   @r1   rç   rç   8  s   ø„ ô\ör0   rç   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEmu3VQVAEEncoderConvUpsamplec                 ó`   •— t         ‰| �  «        t        j                  ||ddd¬«      | _        y )Nr   r!   ré   rí   rð   s     €r1   rw   z%Emu3VQVAEEncoderConvUpsample.__init__E  rò   r0   c                 óX   — t        j                  |dd¬«      }| j                  |«      }|S )Nç       @Únearest©Úscale_factorrö   )r÷   Úinterpolaterï   rø   s     r1   r’   z$Emu3VQVAEEncoderConvUpsample.forwardI  s(   € ÜŸ™ mÀ#ÈIÔVˆØŸ	™	 -Ó0ˆØÐr0   r»   r˜   s   @r1   rú   rú   D  s   ø„ ô\ör0   rú   c            	       ó\   ‡ — e Zd Zdededee   dee   fˆ fd„Zdej                  fd„Zˆ xZ	S )ÚEmu3VQVAEConv3dÚ
in_channelÚout_channelrê   rë   c                 óP  •— t         ‰	| �  «        t        |dd  |dd  «      D ��cg c]
  \  }}||z
  ‘Œ }}}d| _        |d d d…   D ]%  }| xj                  |dz  |dz  z   |dz  fz  c_        Œ' | xj                  dz  c_        t	        j
                  ||||¬«      | _        y c c}}w )Nr!   r/   r3   r4   )r4   r   )rë   )rv   rw   Úziprì   rb   ÚConv3drï   )
r„   r  r  rê   rë   Ú
one_kernelÚ
one_strideÚpadding_sizesÚpad_sizer…   s
            €r1   rw   zEmu3VQVAEConv3d.__init__P  s½   ø€ ô 	‰ÑÔäORÐS^Ð_`Ð_aÐSbÐdjÐklÐkmÐdnÓOo×pÑ5K°ZÀ˜ jÓ0ÐpˆÑpØˆŒØ%¡d¨ dÑ+ò 	JˆHØ�LŠL˜X¨™]¨X¸©\Ñ9¸8Àq¹=ÐIÑIŽLð	Jà�Š˜Ñ�ä—I‘IØØØØô	
ˆ�	ùó qs   ¥B"rH   c                 óh   — t        j                  || j                  «      }| j                  |«      }|S rº   )r÷   rõ   rì   rï   rø   s     r1   r’   zEmu3VQVAEConv3d.forwardf  s*   € ÜŸ™˜m¨T¯\©\Ó:ˆØŸ	™	 -Ó0ˆØÐr0   )
r(   r)   r*   r”   r–   rw   r,   r•   r’   r—   r˜   s   @r1   r  r  O  sF   ø„ ð
àð
ð ð
ð ˜3‘Zð	
ð
 �c‘
õ
ð, U§\¡\÷ r0   r  c                   ó`   ‡ — e Zd Zdedefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚEmu3VQVAESpatialNormrñ   Úout_channelsc                 óÚ   •— t         ‰| �  «        t        j                  |ddd¬«      | _        t        j
                  ||ddd¬«      | _        t        j
                  ||ddd¬«      | _        y )Né    r¬   T©Únum_channelsÚ
num_groupsrœ   Úaffiner!   r   ré   )rv   rw   rb   Ú	GroupNormÚ
norm_layerrî   Úconv_yÚconv_b©r„   rñ   r  r…   s      €r1   rw   zEmu3VQVAESpatialNorm.__init__m  sn   ø€ ô
 	‰ÑÔÜŸ,™,Ø%ØØØô	
ˆŒô —i‘iØØØØØô
ˆŒô —i‘iØØØØØô
ˆ�r0   rH   Úquant_statesc                 ó¾   — t        j                  ||j                  dd  d¬«      }| j                  |«      }|| j	                  |«      z  | j                  |«      z   }|S )Néþÿÿÿrþ   )Úsizerö   )r÷   r  r7   r  r  r  )r„   rH   r  s      r1   r’   zEmu3VQVAESpatialNorm.forward‰  sX   € Ü—}‘} \¸×8KÑ8KÈBÈCÐ8PÐW`ÔaˆØŸ™¨Ó6ˆØ%¨¯©°LÓ(AÑAÀDÇKÁKÐP\ÓD]Ñ]ˆØÐr0   ©	r(   r)   r*   r”   rw   r,   r•   r’   r—   r˜   s   @r1   r  r  l  s5   ø„ ð
àð
ð õ
ð8 U§\¡\ð ÀÇÁ÷ r0   r  c                   óH   ‡ — e Zd Zdedefˆ fd„Zdej                  fd„Zˆ xZS )ÚEmu3VQVAETemporalUpsampler  r  c                 óJ   •— t         ‰| �  «        t        ||dd¬«      | _        y )N©r   r   r   ©r!   r!   r!   ©rê   rë   ©rv   rw   r  rï   ©r„   r  r  r…   s      €r1   rw   z"Emu3VQVAETemporalUpsample.__init__‘  ó(   ø€ ô
 	‰ÑÔÜ#ØØØ!Øô	
ˆ�	r0   rH   c                 óP  — |j                   \  }}}}}|j                  ddddd«      j                  «       j                  |d|«      }t	        j
                  |dd¬	«      }|j                  ||||d«      j                  ddddd«      j                  «       }| j                  |«      }|S )
Nr   r!   r   rØ   r4   r3   rý   rþ   rÿ   )r7   rÙ   rg   rŠ   r÷   r  rï   )r„   rH   rÜ   rÞ   rÝ   rß   rà   s          r1   r’   z!Emu3VQVAETemporalUpsample.forwardž  sª   € Ø8E×8KÑ8KÑ5ˆ
�H˜h¨°Ø%×-Ñ-¨a°°A°q¸!Ó<×GÑGÓI×NÑNÈzÐ[]Ð_gÓhˆÜŸ™ mÀ#ÈIÔVˆØ%×*Ñ*¨:°xÀÈÐPRÓS×[Ñ[Ð\]Ð_`ÐbcÐefÐhiÓj×uÑuÓwˆØŸ	™	 -Ó0ˆØÐr0   r   r˜   s   @r1   r"  r"  �  s*   ø„ ð
àð
ð õ
ð U§\¡\÷ r0   r"  c                   óH   ‡ — e Zd Zdedefˆ fd„Zdej                  fd„Zˆ xZS )ÚEmu3VQVAETemporalDownsampler  r  c                 óJ   •— t         ‰| �  «        t        ||dd¬«      | _        y )N)rØ   r   r   )r4   r!   r!   r&  r'  r(  s      €r1   rw   z$Emu3VQVAETemporalDownsample.__init__¨  r)  r0   rH   c                 ó(   — | j                  |«      }|S rº   )rï   rø   s     r1   r’   z#Emu3VQVAETemporalDownsample.forwardµ  s   € ØŸ	™	 -Ó0ˆØÐr0   r   r˜   s   @r1   r,  r,  §  s*   ø„ ð
àð
ð õ
ð U§\¡\÷ r0   r,  c                   ó(   ‡ — e Zd Z	 dˆ fd„	Zd„ Zˆ xZS )ÚEmu3VQVAETemporalResnetBlockc                 óp  •— t         ‰| �  «        || _        |€|n|| _        t	        j
                  |«      | _        t        ||dd¬«      | _        t	        j
                  |«      | _	        t        ||dd¬«      | _
        | j                  | j                  k7  r t	        j                  ||ddd¬«      | _        y y )Nr$  r%  r&  r!   r   ré   )rv   rw   rñ   r  rb   ÚBatchNorm3dÚnorm1r  Úconv1Únorm2Úconv2r  Únin_shortcutr  s      €r1   rw   z%Emu3VQVAETemporalResnetBlock.__init__»  s´   ø€ ô
 	‰ÑÔØ&ˆÔØ+7Ð+?™KÀ\ˆÔä—^‘^ KÓ0ˆŒ
Ü$ØØØ!Øô	
ˆŒ
ô —^‘^ LÓ1ˆŒ
Ü$ØØØ!Øô	
ˆŒ
ð ×Ñ˜t×0Ñ0Ò0Ü "§	¡	ØØØØØô!ˆDÕð 1r0   c                 óL  — |}| j                  |«      }|t        j                  |«      z  }| j                  |«      }| j	                  |«      }|t        j                  |«      z  }| j                  |«      }| j                  | j                  k7  r| j                  |«      }||z   S rº   )	r3  r,   Úsigmoidr4  r5  r6  rñ   r  r7  )r„   rH   rÉ   s      r1   r’   z$Emu3VQVAETemporalResnetBlock.forwardÛ  s”   € Ø ˆØŸ
™
 =Ó1ˆØœŸ™ }Ó5Ñ5ˆØŸ
™
 =Ó1ˆàŸ
™
 =Ó1ˆØœŸ™ }Ó5Ñ5ˆØŸ
™
 =Ó1ˆà×Ñ˜t×0Ñ0Ò0Ø×(Ñ(¨Ó2ˆHà˜-Ñ'Ð'r0   rº   r»   r˜   s   @r1   r0  r0  º  s   ø„ ð õö@(r0   r0  c                   ó~   ‡ — e Zd Z	 	 ddededz  dedz  fˆ fd„Zd	dej                  dej                  dz  fd„Zˆ xZS )
ÚEmu3VQVAEResnetBlockNrñ   r  Úquant_channelsc                 ó  •— t         ‰| �  «        || _        |€|n|}|| _        || _        |€=t        j                  |ddd¬«      | _        t        j                  |ddd¬«      | _        n"t        ||«      | _        t        ||«      | _        t        j                  ||ddd¬«      | _        t        j                  ||ddd¬«      | _        | j                  | j                  k7  r t        j                  ||ddd¬«      | _        y y )	Nr  r¬   Tr  r   r!   ré   r   )rv   rw   rñ   r  r<  rb   r  r3  r5  r  rî   r4  r6  r7  )r„   rñ   r  r<  r…   s       €r1   rw   zEmu3VQVAEResnetBlock.__init__ì  s  ø€ ô 	‰ÑÔØ&ˆÔØ&2Ð&:‘{ÀˆØ(ˆÔØ,ˆÔàÐ!ÜŸ™°;È2ÐSWÐ`dÔeˆDŒJÜŸ™°<ÈBÐTXÐaeÔfˆD�Jä-¨n¸kÓJˆDŒJÜ-¨n¸lÓKˆDŒJä—Y‘YØØØØØô
ˆŒ
ô —Y‘YØØØØØô
ˆŒ
ð ×Ñ˜t×0Ñ0Ò0Ü "§	¡	ØØØØØô!ˆDÕð 1r0   rH   c                 óv  — | j                   €dn|f}|} | j                  |g|¢­Ž }|t        j                  |«      z  }| j	                  |«      } | j
                  |g|¢­Ž }|t        j                  |«      z  }| j                  |«      }| j                  | j                  k7  r| j                  |«      }||z   S ©Nr/   )
r<  r3  r,   r9  r4  r5  r6  rñ   r  r7  )r„   rH   r<  Ú	norm_argsrÉ   s        r1   r’   zEmu3VQVAEResnetBlock.forward  s·   € Ø×-Ñ-Ð5‘B¸NÐ;Lˆ	à ˆØ"˜Ÿ
™
 =Ð=°9Ò=ˆØœŸ™ }Ó5Ñ5ˆØŸ
™
 =Ó1ˆà"˜Ÿ
™
 =Ð=°9Ò=ˆØœŸ™ }Ó5Ñ5ˆØŸ
™
 =Ó1ˆà×Ñ˜t×0Ñ0Ò0Ø×(Ñ(¨Ó2ˆHà˜-Ñ'Ð'r0   )NNrº   r   r˜   s   @r1   r;  r;  ë  sV   ø„ ð $(Ø%)ñ	*àð*ð ˜D‘jð*ð ˜d™
õ	*ñX( U§\¡\ð (À5Ç<Á<ÐRVÑCV÷ (r0   r;  c            
       ó¦   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej                  dej                  dz  deej                  ej                  dz  f   fd„Z	ˆ xZ
S )
ÚEmu3VQVAEAttentionBlockro   rp   c                 ó&  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚| j                  dz  | _	        |j                  | _        d| _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        d| _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).rs   Fr!   )rv   rw   rp   ry   rÒ   rz   Ú	num_headsrQ   Ú
ValueErrorÚscaler{   rY   r|   rb   r}   r€   r�   r   Úout_projr_   r¸   s     €r1   rw   z Emu3VQVAEAttentionBlock.__init__-  s$  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒØˆŒä—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜŸ	™	 $§.¡.°$·.±.ÓAˆŒð %&ˆÕ!r0   NrH   rW   rJ   c           
      ó®  — |j                   dd }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }t        j                  | j                  j                  t        «      }	 |	| ||||| j                  | j                  | j                  sdn| j                  ¬«      \  }
} |
j                   g |¢d‘­Ž j#                  «       }
| j%                  |
«      }
|
|fS )z#Input shape: Batch x Time x ChannelNr3   r!   r4   r‰   )r|   rX   rY   )r7   rQ   r   rŠ   ra   r€   r�   r   rŒ   rp   r�   rl   r|   rF  r^   rY   rM   rg   rG  )r„   rH   rW   rZ   rŽ   r�   ÚqueriesÚkeysÚvaluesr‘   rk   rj   s               r1   r’   zEmu3VQVAEAttentionBlock.forwardD  s<  € ð $×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØ—+‘+˜mÓ,×1Ñ1°,Ó?×IÑIÈ!ÈQÓOˆØ�{‰{˜=Ó)×.Ñ.¨|Ó<×FÑFÀqÈ!ÓLˆØ—‘˜]Ó+×0Ñ0°Ó>×HÑHÈÈAÓNˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØØ—n‘nØ—J‘JØ#Ÿ}š}‘C°$·,±,ô	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m KÓ0ˆà˜LÐ(Ð(r0   rº   )r(   r)   r*   r+   r$   rw   r,   r•   r–   r’   r—   r˜   s   @r1   rB  rB  *  s\   ø„ ÙGð&˜õ &ð4 /3ñ!)à—|‘|ð!)ð Ÿ™ tÑ+ð!)ð
 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷!)r0   rB  c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚEmu3VQVAEGroupNormzá
    Same as the torch GroupNorm with the only difference that this ones accepts
    an optional kwarg `quant_states` which is not used. This class makes it easier to
    use SpatialNorm or GroupNorm without conditionals
    c                 ó$   •— t        ‰| �  di |¤Ž y r?  )rv   rw   )r„   rZ   r…   s     €r1   rw   zEmu3VQVAEGroupNorm.__init__o  s   ø€ Ü‰ÑÑ"˜6Ó"r0   c                 ó„   — t        j                  || j                  | j                  | j                  | j
                  «      S rº   )r÷   Ú
group_normr  r    ru   rœ   )r„   Úinputr  s      r1   r’   zEmu3VQVAEGroupNorm.forwardr  s)   € Ü�|‰|˜E 4§?¡?°D·K±KÀÇÁÈDÏHÉHÓUÐUr0   rº   )r(   r)   r*   r+   rw   r’   r—   r˜   s   @r1   rM  rM  h  s   ø„ ñô#÷Vr0   rM  c                   ó`   ‡ — e Zd Zdˆ fd„	Zddej
                  dej
                  dz  fd„Zˆ xZS )ÚEmu3VQVAEMiddleBlockNc                 óÞ   •— t         ‰| �  «        t        |||¬«      | _        t	        |«      | _        |€t        |ddd¬«      | _        nt        ||«      | _        t        |||¬«      | _	        y )N©rñ   r  r<  r  r¬   Tr  )
rv   rw   r;  Úblock_1rB  Úattn_1rM  Ú	attn_normr  Úblock_2)r„   rp   rñ   r<  r…   s       €r1   rw   zEmu3VQVAEMiddleBlock.__init__w  so   ø€ Ü‰ÑÔä+Ø#Ø$Ø)ô
ˆŒô
 .¨fÓ5ˆŒØÐ!Ü/¸[ÐUWÐ]aÐjnÔoˆD�Nä1°.À+ÓNˆDŒNä+Ø#Ø$Ø)ô
ˆ�r0   rH   r  c                 ób  — | j                  ||«      }|}| j                  ||«      }|j                  \  }}}}|j                  ||||z  «      j	                  dd«      }| j                  |«      d   }|j                  ||||«      j                  dddd«      }||z   }| j                  ||«      }|S )Nr!   r4   r   r   )	rV  rX  r7   rŠ   ra   rW  rM   rÙ   rY  )r„   rH   r  rÉ   rÜ   rÞ   rß   rà   s           r1   r’   zEmu3VQVAEMiddleBlock.forward‹  sÀ   € ØŸ™ ]°LÓAˆØ ˆØŸ™ }°lÓCˆØ.;×.AÑ.AÑ+ˆ
�H˜f eØ%×*Ñ*¨:°xÀÈ%ÁÓP×ZÑZÐ[\Ð^_Ó`ˆØŸ™ MÓ2°1Ñ5ˆØ%×-Ñ-¨j¸&À%ÈÓR×ZÑZÐ[\Ð^_ÐabÐdeÓfˆØ  =Ñ0ˆØŸ™ ]°LÓAˆØÐr0   rº   ©r(   r)   r*   rw   r,   ÚFloatTensorr’   r—   r˜   s   @r1   rS  rS  v  s-   ø„ õ
ñ(
 U×%6Ñ%6ð 
Àe×FWÑFWÐZ^ÑF^÷ 
r0   rS  c                   ó>   ‡ — e Zd Zˆ fd„Zdej
                  fd„Zˆ xZS )ÚEmu3VQVAEDownBlockc           
      óÌ  •— t         ‰| �  «        t        |j                  «      | _        |j
                  | _        |j                  }|j                  }dt        |«      z   }|| _        t        j                  «       | _        t        | j                  «      D �]K  }t        j                  «       }t        j                  «       }t        j                  «       }|||   z  }	|||   z  }
t        | j
                  «      D ]~  }|j                  t        |	|
¬«      «       |
}	|j                  €Œ.||j                  v sŒ=|j                  t!        |«      «       |j                  t        j"                  |	ddd¬«      «       Œ€ t        j$                  «       }||_        ||_        ||_        || j                  dz
  k7  rt-        |	«      |_        | j                  j                  |«       �ŒN y )N©r!   ©rñ   r  r  r¬   Tr  r!   )rv   rw   ÚlenÚchannel_multiplierÚnum_resolutionsÚnum_res_blocksÚbase_channelsr–   Úin_channel_multiplierrb   Ú
ModuleListÚdownÚrangeÚappendr;  Úattn_resolutionsrB  r  ÚModuleÚblockÚattnÚ
attn_normsrç   Ú
downsample)r„   rp   rf  rc  rg  Úi_levelrn  ro  rp  Úblock_inÚ	block_outÚi_blockri  r…   s                €r1   rw   zEmu3VQVAEDownBlock.__init__™  s¢  ø€ Ü‰ÑÔä" 6×#<Ñ#<Ó=ˆÔØ$×3Ñ3ˆÔØ×,Ñ,ˆØ#×6Ñ6Ðà $¤uÐ-?Ó'@Ñ @ÐØ%:ˆÔ"Ü—M‘M“OˆŒ	Ü˜T×1Ñ1Ó2ó 	#ˆGÜ—M‘M“OˆEÜ—=‘=“?ˆDÜŸ™›ˆJØ$Ð'<¸WÑ'EÑEˆHØ%Ð(:¸7Ñ(CÑCˆIÜ  ×!4Ñ!4Ó5ò 
q�Ø—‘Ü(Ø$,Ø%.ôôð %�Ø×*Ñ*Ñ6¸7Àf×F]ÑF]Ò;]Ø—K‘KÔ 7¸Ó ?Ô@Ø×%Ñ%¤b§l¡lÀÐUWÐ]aÐjnÔ&oÕpð
qô —9‘9“;ˆDØˆDŒJØˆDŒIØ(ˆDŒOØ˜$×.Ñ.°Ñ2Ò2Ü"@ÀÓ"J�”Ø�I‰I×Ñ˜TÖ"ñ1	#r0   rH   c                 ó>  — t        | j                  «      D �]  \  }}t        | j                  «      D ]Á  } |j                  |   |«      }t        |j                  «      dkD  sŒ1|} |j                  |   |«      }|j                  \  }}}}	|j                  ||||	z  «      j                  dd«      } |j                  |   |«      d   }|j                  |||	|«      j                  dddd«      }||z   }ŒÃ || j                  dz
  k7  sŒó|j                  |«      }�Œ |S )Nr   r!   r4   r   )Ú	enumerateri  rj  re  rn  rb  ro  rp  r7   rŠ   ra   rM   rÙ   rd  rq  )
r„   rH   rr  Úblocksru  rÉ   rÜ   rÞ   rß   rà   s
             r1   r’   zEmu3VQVAEDownBlock.forward¾  s5  € Ü(¨¯©Ó3ó 	A‰OˆG�VÜ  ×!4Ñ!4Ó5ò =�Ø 5 §¡¨WÑ 5°mÓ D�Ü�v—{‘{Ó# aÓ'Ø,�HØ$> F×$5Ñ$5°gÑ$>¸}Ó$M�Mà:G×:MÑ:MÑ7�J ¨&°%Ø$1×$6Ñ$6°zÀ8ÈVÐV[É^Ó$\×$fÑ$fÐghÐjkÓ$l�MØ$8 F§K¡K°Ñ$8¸Ó$GÈÑ$J�Mà$1×$9Ñ$9¸*ÀfÈeÐU]Ó$^×$fÑ$fÐghÐjkÐmnÐpqÓ$r�MØ$,¨}Ñ$<‘Mð=ð ˜$×.Ñ.°Ñ2Ó2Ø &× 1Ñ 1°-Ó @’ð	Að" Ðr0   r[  r˜   s   @r1   r^  r^  ˜  s   ø„ ô##ðJ U×%6Ñ%6÷ r0   r^  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚEmu3VQVAEUpBlockc           	      ó¨  •— t         ‰| �  «        t        |j                  «      | _        |j
                  | _        |j                  }|j                  |j                  d   z  }t        j                  «       | _
        t        t        | j                  «      «      D �]5  }t        j                  «       }t        j                  «       }t        j                  «       }|j                  |j                  |   z  }t        | j
                  dz   «      D ]e  }	|j                  t        |||¬«      «       |}||j                  v sŒ1|j                  t!        |«      «       |j                  t#        ||«      «       Œg t        j$                  «       }
||
_        ||
_        ||
_        |dk7  rt-        |«      |
_        | j                  j1                  d|
«       �Œ8 y )Nr3   r!   rU  r   )rv   rw   rb  rc  rd  re  rÒ   rf  rb   rh  ÚupÚreversedrj  rk  r;  rl  rB  r  rm  rn  ro  rp  rú   ÚupsampleÚinsert)r„   rp   r<  rs  rr  rn  ro  rp  rt  ru  r|  r…   s              €r1   rw   zEmu3VQVAEUpBlock.__init__Ô  s  ø€ Ü‰ÑÔä" 6×#<Ñ#<Ó=ˆÔØ$×3Ñ3ˆÔà×)Ñ)ˆØ×'Ñ'¨&×*CÑ*CÀBÑ*GÑGˆä—-‘-“/ˆŒÜ¤ d×&:Ñ&:Ó ;Ó<ó 	"ˆGÜ—M‘M“OˆEÜ—=‘=“?ˆDÜŸ™›ˆJØ×,Ñ,¨v×/HÑ/HÈÑ/QÑQˆIÜ  ×!4Ñ!4°qÑ!8Ó9ò V�Ø—‘Ü(Ø$,Ø%.Ø'5ôôð %�Ø˜f×5Ñ5Ò5Ø—K‘KÔ 7¸Ó ?Ô@Ø×%Ñ%Ô&:¸>È8Ó&TÕUðVô —‘“ˆBØˆBŒHØˆBŒGØ&ˆBŒMØ˜!Š|Ü:¸8ÓD�”à�G‰G�N‰N˜1˜bÖ!ñ3	"r0   rH   r  c                 óh  — t        | j                  d d d…   «      D �]  \  }}t        | j                  dz   «      D ]Ã  } |j                  |   ||«      }t        |j                  «      dkD  sŒ2|} |j                  |   ||«      }|j                  \  }}}	}
|j                  |||	|
z  «      j                  dd«      } |j                  |   |«      d   }|j                  ||	|
|«      j                  dddd«      }||z   }ŒÅ |t        | j                  «      dz
  k7  s�Œ|j                  |«      }�Œ |S )Nr3   r!   r   r4   r   )rw  r|  rj  re  rn  rb  ro  rp  r7   rŠ   ra   rM   rÙ   r~  )r„   rH   r  rr  rx  ru  rÉ   rÜ   rÞ   rß   rà   s              r1   r’   zEmu3VQVAEUpBlock.forwardù  sD  € Ü(¨¯©±°2°©Ó7ó 	?‰OˆG�VÜ  ×!4Ñ!4°qÑ!8Ó9ò =�Ø 5 §¡¨WÑ 5°mÀ\Ó R�Ü�v—{‘{Ó# aÓ'Ø,�HØ$> F×$5Ñ$5°gÑ$>¸}ÈlÓ$[�Mà:G×:MÑ:MÑ7�J ¨&°%Ø$1×$6Ñ$6°zÀ8ÈVÐV[É^Ó$\×$fÑ$fÐghÐjkÓ$l�MØ$8 F§K¡K°Ñ$8¸Ó$GÈÑ$J�Mà$1×$9Ñ$9¸*ÀfÈeÐU]Ó$^×$fÑ$fÐghÐjkÐmnÐpqÓ$r�MØ$,¨}Ñ$<‘Mð=ð œ#˜dŸg™g›,¨Ñ*Ô*Ø &§¡°Ó >’ð	?ð  Ðr0   r[  r˜   s   @r1   rz  rz  Ó  s(   ø„ ô#"ðJ U×%6Ñ%6ð Àe×FWÑFW÷ r0   rz  c                   ó>   ‡ — e Zd Zˆ fd„Zdej
                  fd„Zˆ xZS )ÚEmu3VQVAEEncoderc                 ó˜  •— t         ‰| �  «        |j                  }|j                  }|j                  }|j
                  }|j                  }|rd|z  n|}||d   z  }t        j                  j                  ||ddd¬«      | _
        t        |«      | _        t        ||«      | _        t        j                  j                  d|dd¬	«      | _        t        j                  j                  ||ddd¬«      | _        t%        t'        j(                  |j*                  «      «      }	t        j,                  «       | _        t        j,                  «       | _        t3        |	«      D ])  }
t5        ||«      }| j.                  j7                  |«       Œ+ t3        |j8                  «      D ]*  }t;        ||¬
«      }| j0                  j7                  |«       Œ, y )Nr4   r3   r   r!   ré   r  r¬   T)r  r  rœ   r  ra  )rv   rw   rf  rñ   Údouble_latentÚlatent_channelsrc  r,   rb   rî   Úconv_inr^  Ú
down_blockrS  Úmiddle_blockr  Únorm_outÚconv_outr”   ÚmathÚlog2Útemporal_downsample_factorrh  Ú	time_convÚtime_res_stackrj  r,  rk  re  r0  )r„   rp   rf  rñ   r„  r…  rc  r  rs  Útemporal_down_blocksÚirï   rÊ   Útime_res_convr…   s                 €r1   rw   zEmu3VQVAEEncoder.__init__  s”  ø€ Ü‰ÑÔà×,Ñ,ˆØ×(Ñ(ˆØ×,Ñ,ˆØ ×0Ñ0ˆØ#×6Ñ6ÐÙ.;�q˜?Ò*ÀˆØ Ð#5°bÑ#9Ñ9ˆä—x‘x—‘ {°MÈqÐYZÐde�ÓfˆŒÜ,¨VÓ4ˆŒÜ0°¸ÓBˆÔäŸ™×*Ñ*°bÀxÐUYÐbfÐ*ÓgˆŒÜŸ™Ÿ™ØØØØØð (ó 
ˆŒô  #¤4§9¡9¨V×-NÑ-NÓ#OÓPÐÜŸ™›ˆŒÜ Ÿm™m›oˆÔäÐ+Ó,ò 	(ˆAÜ.¨|¸\ÓJˆDØ�N‰N×!Ñ! $Õ'ð	(ô �v×,Ñ,Ó-ò 	6ˆAÜ8Ø(Ø)ôˆMð ×Ñ×&Ñ& }Õ5ñ	6r0   Úpixel_valuesc                 óh  — |j                   d   } |j                  dg|j                   dd  ¢­Ž }| j                  |«      }| j                  |«      }| j	                  |«      }| j                  |«      }|t        j                  |«      z  }| j                  |«      } |j                  d|g|j                   dd  ¢­Ž }|j                  ddddd«      }| j                  D ]"  } ||«      }|t        j                  |«      z  }Œ$ | j                  D ]
  } ||«      }Œ |j                  ddddd«      }|S )Nr!   r3   r4   r   r   rØ   )r7   rM   r†  r‡  rˆ  r‰  r,   r9  rŠ  rÙ   rŽ  r�  )r„   r“  Útemporal_dimrH   rï   Úlayers         r1   r’   zEmu3VQVAEEncoder.forward5  sH  € Ø#×)Ñ)¨!Ñ,ˆØ+�|×+Ñ+¨BÐH°×1CÑ1CÀAÀBÐ1GÒHˆð Ÿ™ \Ó2ˆØŸ™¨Ó6ˆØ×)Ñ)¨-Ó8ˆð Ÿ™ mÓ4ˆØœŸ™ }Ó5Ñ5ˆØŸ™ mÓ4ˆà-˜×-Ñ-¨b°,ÐYÀ×ATÑATÐUVÐUWÐAXÒYˆØ%×-Ñ-¨a°°A°q¸!Ó<ˆð —N‘Nò 	:ˆDÙ  Ó/ˆMØœUŸ]™]¨=Ó9Ñ9‰Mð	:ð ×(Ñ(ò 	1ˆEÙ! -Ó0‰Mð	1ð &×-Ñ-¨a°°A°q¸!Ó<ˆàÐr0   )r(   r)   r*   rw   r,   r-   r’   r—   r˜   s   @r1   r‚  r‚    s   ø„ ô%6ðN E×$4Ñ$4÷ r0   r‚  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚEmu3VQVAEDecoderrp   c                 ó¬  •— t         ‰	| �  «        |j                  }|j                  |j                  d   z  }t        j                  «       | _        t        |j                  «      D ]>  }t        |j                  |j                  ¬«      }| j                  j                  |«       Œ@ t        t        j                  |j                   «      «      }t        j                  «       | _        t        |«      D ]=  }t%        |j                  |j                  «      }| j"                  j                  |«       Œ? t        j&                  |j                  |ddd¬«      | _        t+        |||¬«      | _        t/        |«      | _        |j                  |j                  d   z  }t3        ||«      | _        t        j&                  ||j6                  ddd¬«      | _        y )Nr3   ra  r   r!   ré   )r<  r   )rv   rw   rÒ   rf  rc  rb   rh  r�  rj  re  r0  r…  rk  r”   r‹  rŒ  r�  rŽ  r"  rî   r†  rS  rˆ  rz  Úup_blockr  r‰  r  rŠ  )
r„   rp   r<  rs  rÊ   r’  Útemp_upsample_block_numr‘  rï   r…   s
            €r1   rw   zEmu3VQVAEDecoder.__init__T  sŠ  ø€ Ü‰ÑÔà×)Ñ)ˆØ×'Ñ'¨&×*CÑ*CÀBÑ*GÑGˆÜ Ÿm™m›oˆÔÜ�v×,Ñ,Ó-ò 	6ˆAÜ8Ø"×2Ñ2À×AWÑAWôˆMð ×Ñ×&Ñ& }Õ5ð		6ô #&¤d§i¡i°×0QÑ0QÓ&RÓ"SÐÜŸ™›ˆŒÜÐ.Ó/ò 	(ˆAÜ,¨V×-CÑ-CÀV×E[ÑE[Ó\ˆDØ�N‰N×!Ñ! $Õ'ð	(ô —y‘yØ×"Ñ"ØØØØô
ˆŒô 1°¸ÐR`ÔaˆÔÜ(¨Ó0ˆŒà×'Ñ'¨&×*CÑ*CÀAÑ*FÑFˆÜ,¨^¸XÓFˆŒÜŸ	™	ØØ×ÑØØØô
ˆ�r0   rH   r  c                 ó¶  — t        j                  ||fd¬«      }|j                  ddddd«      }| j                  D ]
  } ||«      }Œ | j                  D ]"  } ||«      }|t        j
                  |«      z  }Œ$ |j                  ddddd«      }t        j                  |dd¬«      \  }} |j                  dg|j                  dd  ¢­Ž } |j                  dg|j                  dd  ¢­Ž }| j                  |«      }| j                  ||«      }| j                  ||«      }| j                  ||«      }|t        j
                  |«      z  }| j                  |«      }|S )Nr   r5   r4   r!   r   rØ   r3   )r,   r8   rÙ   r�  rŽ  r9  ÚchunkrM   r7   r†  rˆ  rš  r‰  rŠ  )r„   rH   r  Úhidden_quant_statesr–  s        r1   r’   zEmu3VQVAEDecoder.forward{  sp  € Ü#Ÿi™i¨¸Ð(EÈ1ÔMÐØ1×9Ñ9¸!¸QÀÀ1ÀaÓHÐð ×(Ñ(ò 	=ˆEÙ"'Ð(;Ó"<Ñð	=ð —^‘^ò 	FˆEÙ"'Ð(;Ó"<ÐØ¤5§=¡=Ð1DÓ#EÑEÑð	Fð 2×9Ñ9¸!¸QÀÀ1ÀaÓHÐÜ&+§k¡kÐ2EÀqÈaÔ&PÑ#ˆ�|Ø-˜×-Ñ-¨bÐK°=×3FÑ3FÀqÀrÐ3JÒKˆØ+�|×+Ñ+¨BÐH°×1CÑ1CÀAÀBÐ1GÒHˆàŸ™ ]Ó3ˆð ×)Ñ)¨-¸ÓFˆØŸ™ m°\ÓBˆàŸ™ m°\ÓBˆØœŸ™ }Ó5Ñ5ˆØŸ™ mÓ4ˆàÐr0   )	r(   r)   r*   r$   rw   r,   r•   r’   r—   r˜   s   @r1   r˜  r˜  S  s+   ø„ ð%
˜õ %
ðN U§\¡\ð ÀÇÁ÷ r0   r˜  aR  
    The VQ-VAE model used in Emu3 for encoding/decoding images into discrete tokens.
    This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from
    [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv
    Taigman](https://huggingface.co/papers/2203.13131).
    ©Úcustom_introc            
       ó  ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZg d¢ZeegedœZ ej$                  «       ˆ fd„«       Zdefˆ fd	„Zeedej.                  d
ej.                  dee   defd„«       «       Zdej.                  fd„Zˆ xZS )Ú	Emu3VQVAErp   Ú
emuvideovqr“  )ÚimageT)r0  rB  r;  rÍ   ©rH   Ú
attentionsc                 ót  •— t         ‰| �  |«       t        |t        j                  t        j
                  f«      r t        j                  |j                  dd¬«       |j                  �qt        j                  j                  j                  |j                  «      \  }}dt        j                  |«      z  }t        j                  |j                  | |«       y y t        |t        j                  «      r¹t        j                   |j                  t        j                  d«      ¬«       |j                  �xt        j                  j                  j                  |j                  «      \  }}|dkD  rdt        j                  |«      z  nd}t        j                  |j                  | |«       y y t        |t        j"                  «      rqt        j$                  |j                  «       |j&                  �Et)        |j                  dd	«      s-t        j*                  |j                  |j&                     «       y y y y )
NÚfan_outÚrelu)rö   Únonlinearityr!   é   )Úar   Ú_is_hf_initializedF)rv   Ú_init_weightsÚ
isinstancerb   rî   r  ÚinitÚkaiming_normal_r    ru   r,   Ú_calculate_fan_in_and_fan_outr‹  ÚsqrtrÕ   r}   Úkaiming_uniform_rÐ   Únormal_Úpadding_idxrx   Úzeros_)r„   rS   Úfan_inrÊ   Úboundr…   s        €r1   r®  zEmu3VQVAE._init_weightsµ  s‡  ø€ ä‰Ñ˜fÔ%Ü�fœrŸy™y¬"¯)©)Ð4Ô5Ü× Ñ  §¡°YÈVÕTØ�{‰{Ð&Ü!ŸH™HŸM™M×GÑGÈÏÉÓV‘	�˜ØœDŸI™I fÓ-Ñ-�Ü—‘˜fŸk™k¨E¨6°5Õ9ð 'ô ˜¤§	¡	Ô*Ü×!Ñ! &§-¡-´4·9±9¸Q³<Õ@Ø�{‰{Ð&Ü!ŸH™HŸM™M×GÑGÈÏÉÓV‘	�˜Ø17¸!²˜œDŸI™I fÓ-Ò-À�Ü—‘˜fŸk™k¨E¨6°5Õ9ð 'ô ˜¤§¡Ô-Ü�L‰L˜Ÿ™Ô'à×!Ñ!Ð-´g¸f¿m¹mÐMaÐchÔ6iÜ—‘˜FŸM™M¨&×*<Ñ*<Ñ=Õ>ð 7jÐ-ð .r0   c                 óò  •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        t        |«      | _        dt        |j                  «      dz
  z  | _        t        |j                  |j                  dd¬«      | _        t        |j                  |j                  dd¬«      | _        dt        |j                  «      dz
  z  | _        | j%                  «        | j'                  «        y )Nr4   r!   )r   r!   r!   r%  r&  )rv   rw   rp   r‚  Úencoderr˜  ÚdecoderrÍ   Úquantizerb  rc  Úvision_spatial_factorr  r…  rÒ   Ú
quant_convÚpost_quant_convÚspatial_scale_factorÚevalÚ	post_initr¸   s     €r1   rw   zEmu3VQVAE.__init__Ê  sÑ   ø€ Ü‰Ñ˜Ô àˆŒä'¨Ó/ˆŒÜ'¨Ó/ˆŒÜ0°Ó8ˆŒØ%&¬3¨v×/HÑ/HÓ+IÈAÑ+MÑ%NˆÔ"ä)Ø×"Ñ" F×$4Ñ$4À)ÐT]ô
ˆŒô  /Ø×Ñ˜f×4Ñ4À)ÐT]ô 
ˆÔð %&¬#¨f×.GÑ.GÓ*HÈ1Ñ*LÑ$MˆÔ!Ø�	‰	Œà�‰Õr0   Úimage_sizesrZ   rJ   c                 óœ  — |j                   dk(  }|rL| j                  j                  }|j                  \  }}}}	|j	                  d«      j                  d|ddd«      }n|j                  \  }}}}}	| j                  |«      }
|
j                  ddddd«      }| j                  |«      }|j                  ddddd«      }| j                  |«      }|r|j                  d«      n|}t        ||«      D ��cg c]B  \  }}|d t        |d   | j                  z  «      …d t        |d   | j                  z  «      …f   ‘ŒD }}}t        |
|¬«      S c c}}w )NrØ   r!   r   r4   r   )Úlast_hidden_stater'   )Úndimrp   r�  r7   r?   Úrepeatr»  rÙ   r¿  r½  Úsqueezer  r”   r¾  r&   )r„   r“  rÄ  rZ   Úis_imagerÝ   rÜ   rÞ   rß   rà   rH   Úconv_hidden_statesÚcodesr'   Úsingle_imager  s                   r1   ÚencodezEmu3VQVAE.encodeß  sl  € ð
  ×$Ñ$¨Ñ)ˆÙØ—{‘{×=Ñ=ˆHØ2>×2DÑ2DÑ/ˆJ˜ &¨%Ø'×1Ñ1°!Ó4×;Ñ;¸A¸xÈÈAÈqÓQ‰Là<H×<NÑ<NÑ9ˆJ˜ (¨F°EàŸ™ \Ó2ˆð +×2Ñ2°1°a¸¸A¸qÓAÐØ!Ÿ_™_Ð-?Ó@Ðð 0×7Ñ7¸¸1¸aÀÀAÓFÐØ—‘Ð0Ó1ˆá+3�u—}‘} QÔ'¸ˆô '*¨,¸Ó&D÷
á"�˜dð ÐDœ3˜t A™w¨×)CÑ)CÑCÓDÐDÐFqÌÈDÐQRÉGÐVZ×VpÑVpÑLpÓHqÐFqÐqÓrð
ˆñ 
ô
 $Ø+Ø%ô
ð 	
ùó
s   Ã1AErH   c                 ó¬  — |j                   dk(  }|r|j                  d«      }|j                  \  }}}}| j                  j	                  |j                  «       «      }|j                  d   }|j                  |||||«      j                  ddddd«      j                  «       }| j                  |«      }	|j                  ddddd«      }|	j                  ddddd«      }	| j                  |	|«      }
|
j                  ||| j                  j                  z  | j                  j                  || j                  z  || j                  z  «      }
|r	|
d d …df   S |
S )Nr   r!   r3   r   rØ   r4   )rÇ  r?   r7   r½  rÓ   ÚflattenrŠ   rÙ   rg   rÀ  r¼  rM   rp   r�  r  rÁ  )r„   rH   rÊ  rÜ   rÝ   rß   rà   ÚquantrÞ   Ú
post_quantÚvideos              r1   ÚdecodezEmu3VQVAE.decode  sK  € Ø ×%Ñ%¨Ñ*ˆÙØ)×3Ñ3°AÓ6ˆMà.;×.AÑ.AÑ+ˆ
�H˜f eØ—‘×'Ñ'¨×(=Ñ(=Ó(?Ó@ˆà—;‘;˜r‘?ˆØ—
‘
˜: x°¸ÀÓI×QÑQÐRSÐUVÐXYÐ[\Ð^_Ó`×kÑkÓmˆØ×)Ñ)¨%Ó0ˆ
à—‘˜a  A q¨!Ó,ˆØ×'Ñ'¨¨1¨a°°AÓ6ˆ
à—‘˜Z¨Ó/ˆØ—‘ØØ�t—{‘{×=Ñ=Ñ=Ø�K‰K×$Ñ$Ø�T×.Ñ.Ñ.Ø�D×-Ñ-Ñ-ó
ˆñ 'ˆu’Q˜�T‰{Ð1¨EÐ1r0   )r(   r)   r*   r$   r.   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendÚ_no_split_modulesr;  r0  rB  Ú_can_record_outputsr,   Úno_gradr®  rw   r   r    r•   r   r   r&   rÎ  rÔ  r—   r˜   s   @r1   r¢  r¢  ™  sÔ   ø… ð ÓØ$ÐØ$€OØ!ÐØ€NØÐØÐØ"&ÐòÐð /Ð0LÐMØ-ñÐð
 €U‡]�]ƒ_ó?ó ð?ð(˜õ ð*  Øð
Ø!ŸL™Lð
Ø7<·|±|ð
ØOUÐVhÑOið
à	ò
ó ó  ð
ðB2 E§L¡L÷ 2r0   r¢  c                   óì   — e Zd ZdZd„ Zed„ «       Zed„ «       Zed„ «       Zed„ «       Z	ed„ «       Z
ed„ «       Zd	eej                     d
ej                  fd„Zd	ej                  d
ej                  fd„Zy)ÚEmu3ImageVocabularyMappingzM
    A class for mapping discrete image tokens from VQGAN to BPE tokens.
    c                 ój   — || _         |j                  d«      | _        |j                  d«      | _        y )Nz<|extra_200|>z<image>)Ú	vocab_mapÚgetÚeol_token_idÚimage_token_id)r„   râ  s     r1   rw   z#Emu3ImageVocabularyMapping.__init__!  s+   € Ø"ˆŒØ%ŸM™M¨/Ó:ˆÔØ'Ÿm™m¨IÓ6ˆÕr0   c           	      óž   — t        | j                  j                  «       D ��cg c]  \  }}|j                  d«      sŒ|‘Œ c}}«      S c c}}w ©Nz<|visual token©Úsortedrâ  ÚitemsÚ
startswith©r„   ÚnameÚvals      r1   r'   z'Emu3ImageVocabularyMapping.image_tokens&  s8   € ä¨D¯N©N×,@Ñ,@Ó,B×h™y˜t SÀdÇoÁoÐVfÕFg’sÓhÓiÐiùÓhó
   £A	
½A	
c           	      óž   — t        | j                  j                  «       D ��cg c]  \  }}|j                  d«      sŒ|‘Œ c}}«      S c c}}w rç  rè  rì  s      r1   Úimage_tokens_strz+Emu3ImageVocabularyMapping.image_tokens_str*  s8   € ä¨T¯^©^×-AÑ-AÓ-C×i¡	  cÀtÇÁÐWgÕGh’tÓiÓjÐjùÓirï  c                 ót   — | j                   D �ci c]  }t        |dd «      | j                  |   “Œ  c}S c c}w )Niøÿÿÿr  )rñ  r”   râ  )r„   Útokens     r1   Úimg2bpez"Emu3ImageVocabularyMapping.img2bpe.  s5   € àFJ×F[ÑF[Ö\¸U”�E˜"˜R�LÓ! 4§>¡>°%Ñ#8Ñ8Ò\Ð\ùÒ\s   �#5c                 ój   — | j                   j                  «       D ��ci c]  \  }}||“Œ
 c}}S c c}}w rº   )rô  rê  )r„   rA   Úvs      r1   Úbpe2imgz"Emu3ImageVocabularyMapping.bpe2img2  s+   € à!%§¡×!3Ñ!3Ó!5×6™˜˜A��1‘Ó6Ð6ùÓ6s   ž/c                 óæ   — t        j                  t        | j                  j	                  «       «      dz   t         j
                  ¬«      }| j                  j                  «       D ]
  \  }}|||<   Œ |S ©Nr!   ©r\   )r,   ÚzerosÚmaxr÷  rJ  r”   rê  ©r„   ÚmappingrA   rö  s       r1   Úbpe2img_mapping_tensorz1Emu3ImageVocabularyMapping.bpe2img_mapping_tensor6  ó[   € ä—+‘+œc $§,¡,×"3Ñ"3Ó"5Ó6¸Ñ:Ä%Ç)Á)ÔLˆØ—L‘L×&Ñ&Ó(ò 	‰DˆAˆqØˆG�AŠJð	àˆr0   c                 óæ   — t        j                  t        | j                  j	                  «       «      dz   t         j
                  ¬«      }| j                  j                  «       D ]
  \  }}|||<   Œ |S rù  )r,   rû  rü  rô  rJ  r”   rê  rý  s       r1   Úimg2bpe_mapping_tensorz1Emu3ImageVocabularyMapping.img2bpe_mapping_tensor=  r   r0   Ú	img_batchrJ   c                 ó,  — |j                   }t        j                  |j                  d   dft        j                  ¬«      | j
                  z  }| j                  |j                  d«         }t        j                  ||gd¬«      }|j                  |«      S )Nr   r!   rú  Úcpur3   r5   )	Údevicer,   rŸ   r7   r”   rä  r  rf   r8   )r„   r  r  Úeol_rowÚ
img_tokenss        r1   Úconvert_img2bpez*Emu3ImageVocabularyMapping.convert_img2bpeD  sw   € Ø×!Ñ!ˆÜ—*‘*˜iŸo™o¨aÑ0°!Ð4¼E¿I¹IÔFÈ×IZÑIZÑZˆØ×0Ñ0°·±¸eÓ1DÑEˆ
Ü—Y‘Y 
¨GÐ4¸"Ô=ˆ
Ø�}‰}˜VÓ$Ð$r0   c                 óŠ   — |j                   }|dd d…f   }| j                  |j                  d«         }|j                  |«      S )N.r3   r  )r  rÿ  rf   )r„   r  r  r  s       r1   Úconvert_bpe2imgz*Emu3ImageVocabularyMapping.convert_bpe2imgK  sG   € Ø×!Ñ!ˆØ˜c 3 B 3˜hÑ'ˆ	Ø×0Ñ0°·±¸eÓ1DÑEˆ
Ø�}‰}˜VÓ$Ð$r0   N)r(   r)   r*   r+   rw   r   r'   rñ  rô  r÷  rÿ  r  Úlistr,   r•   r	  r  r/   r0   r1   rà  rà    sÅ   „ ñò7ð
 ñjó ðjð ñkó ðkð ñ]ó ð]ð ñ7ó ð7ð ñó ðð ñó ðð%¨¨e¯l©lÑ);ð %ÀÇÁó %ð%¨¯©ð %¸%¿,¹,ô %r0   rà  c                   óP   — e Zd ZU eed<   dZdZdZdgZddgZ	dZ
dZdZdZdZeedœZy	)
ÚEmu3PreTrainedModelrp   Úmodel©r¤  ÚtextTr½   r‡   Úcausal_maskr¥  N)r(   r)   r*   r"   r.   rÕ  r×  Úsupports_gradient_checkpointingrÜ  Ú_skip_keys_device_placementrÙ  rØ  Ú_can_compile_fullgraphrÚ  rÛ  r½   rn   rÝ  r/   r0   r1   r  r  R  s]   … àÓØÐØ(ÐØ&*Ð#àðÐð $5°mÐ"DÐØÐØ€Nà!ÐØÐØ"&Ðà)Ø#ñÑr0   r  c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚEmu3RotaryEmbeddingÚinv_freqNrp   c                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultr  F)Ú
persistentÚoriginal_inv_freq)rv   rw   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrp   Úrope_parametersr  Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r„   rp   r  Úrope_init_fnr  r…   s        €r1   rw   zEmu3RotaryEmbedding.__init__k  s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr0   r  ztorch.deviceÚseq_lenrJ   ztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarQ   NrÏ   r   r4   rú  )r  r\   )	r!  rx   ry   rz   r,   ÚarangeÚint64rf   r­   )rp   r  r'  Úbaser6   Úattention_factorr  s          r1   r"  z3Emu3RotaryEmbedding.compute_default_rope_parameters{  s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r0   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   r3   r!   Úmpsr  F)Údevice_typeÚenabledr4   r5   rú  )r  r­   rL   r7   rf   r  r¯  ÚtypeÚstrr   ra   r,   r8   rB   r#  rC   r\   )
r„   r9   rÆ   Úinv_freq_expandedÚposition_ids_expandedr0  ÚfreqsÚembrB   rC   s
             r1   r’   zEmu3RotaryEmbedding.forward™  sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$rº   r“   )r(   r)   r*   r,   r•   r.   r"   rw   Ústaticmethodr   r”   r–   r­   r"  rÞ  r   r’   r—   r˜   s   @r1   r  r  h  s˜   ø… Ø�l‰lÓñV˜zõ Vð  à$(Ø+/Ø"ñ*Ø˜TÑ!ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r0   r  c                   óþ   ‡ — e Zd ZU eed<   defˆ fd„Zeee	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )ÚEmu3TextModelrp   c           	      ó  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t#        |¬«      | _        d| _        | j)                  «        y c c}w )Nr¿   ©rp   F)rv   rw   Úpad_token_idr¶  Ú
vocab_sizerb   rÐ   ry   Úembed_tokensrh  rj  Únum_hidden_layersr½   Úlayersr›   rÂ   Únormr  Ú
rotary_embÚgradient_checkpointingrÃ  rƒ   s      €r1   rw   zEmu3TextModel.__init__­  sÆ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜBGÈ×H`ÑH`ÓBaÖb°YÔ˜f iÕ0Òbó
ˆŒô   × 2Ñ 2¸×8KÑ8KÔLˆŒ	Ü-°VÔ<ˆŒØ&+ˆÔ#ð 	�‰Õùò cs   ÂDNÚ	input_idsrW   rÆ   r‡   Úinputs_embedsrÇ   rZ   rJ   c           
      ó<  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        | j                  ||||¬«      }	|}
| j                  |
|¬«      }| j                  d | j                  j                   D ]  } ||
f|	||||dœ|¤Ž}
Œ | j                  |
«      }
t        |
|¬	«      S )
Nz:You must specify exactly one of input_ids or inputs_embedsr<  r   r!   )r  )rp   rF  rW   r‡   rÆ   )rÆ   )rW   r†   rÆ   r‡   rÇ   )rÆ  r‡   )rE  r?  r   rp   Úget_seq_lengthr,   r*  r7   r  r?   r   rC  rA  r@  rB  r   )r„   rE  rW   rÆ   r‡   rF  rÇ   rZ   Úpast_seen_tokensr  rH   r†   Údecoder_layers                r1   r’   zEmu3TextModel.forward½  sL  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø*.×*;Ñ*;¸IÓ*FˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 		ˆMÙ)Øðà*Ø$7Ø)Ø /Ø#ñð ñ‰Mð		ð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
r0   )NNNNNN)r(   r)   r*   r#   r.   rw   r   r    r   r,   r-   r•   r
   r\  rË   r   r   r   r’   r—   r˜   s   @r1   r:  r:  ©  sÓ   ø… àÓð˜~õ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ2
à×#Ñ# dÑ*ð2
ð Ÿ™ tÑ+ð2
ð ×&Ñ&¨Ñ-ð	2
ð
  ™ð2
ð ×(Ñ(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ñ,ð2
ð 
!ò2
ó ó ó  ô2
r0   r:  c                   óN  ‡ — e Zd ZU ddiZddiZddgdgfiZeed<   ˆ fd„Ze	e
	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                  d	z  ded	z  dej                  d	z  dej                  d	z  ded	z  deej                  z  dee   defd„«       «       Zˆ xZS )ÚEmu3ForCausalLMúlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrH   Úlogitsrp   c                 óè   •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        | j                  «        y ©NFrt   )
rv   rw   r:  r  r>  rb   r}   ry   rN  rÃ  r¸   s     €r1   rw   zEmu3ForCausalLM.__init__ü  sU   ø€ Ü‰Ñ˜Ô Ü" 6Ó*ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Õr0   NrE  rW   rÆ   r‡   rF  ÚlabelsrÇ   Úlogits_to_keeprZ   rJ   c	           
      óx  —  | j                   d||||||dœ|	¤Ž}
|
j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      }d}|�* | j                  d||| j                  j                  dœ|	¤Ž}t        |||
j                  |
j                  |
j                  ¬«      S )aÃ  
        Example:

        ```python
        >>> from transformers import Emu3Processor, Emu3ForConditionalGeneration
        >>> import torch
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image

        >>> model = Emu3ForCausalLM.from_pretrained("BAAI/Emu3-Chat-hf", dtype=torch.bfloat16)
        >>> processor = Emu3Processor.from_pretrained("BAAI/Emu3-Chat-hf")

        >>> inputs = processor(text=["Can you write me a poem about winter."], return_tensors="pt").to(model.device)

        >>> generated_ids = model.generate(**inputs, max_new_tokens=100, do_sample=False)
        >>> processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        ```©rE  rW   rÆ   r‡   rF  rÇ   N©rP  rS  r>  ©ÚlossrP  r‡   rH   r¦  r/   )r  rÆ  r¯  r”   ÚslicerN  Úloss_functionrp   r>  r   r‡   rH   r¦  )r„   rE  rW   rÆ   r‡   rF  rS  rÇ   rT  rZ   ÚoutputsrH   Úslice_indicesrP  rY  s                  r1   r’   zEmu3ForCausalLM.forward  sÚ   € ð@ ,6¨4¯:©:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r0   )NNNNNNNr   )r(   r)   r*   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr#   r.   rw   r   r   r,   r-   r•   r
   r\  rË   r”   r   r   r   r’   r—   r˜   s   @r1   rL  rL  õ  s  ø… à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÓôð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ7
à×#Ñ# dÑ*ð7
ð Ÿ™ tÑ+ð7
ð ×&Ñ&¨Ñ-ð	7
ð
  ™ð7
ð ×(Ñ(¨4Ñ/ð7
ð × Ñ  4Ñ'ð7
ð ˜$‘;ð7
ð ˜eŸl™lÑ*ð7
ð Ð+Ñ,ð7
ð 
 ò7
ó ó ô7
r0   rL  c                   ó’  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zdej                  dej                  dej                  fd„Z	e
 ed¬	«      dej                  dej                  d
ee   deez  fd„«       «       Z ej"                  «       dej                  dedefd„«       Zdej                  dej                  dej                  fd„Ze
e	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej*                  dz  dej*                  dz  dej                  dz  dedz  dej                  dz  dedz  d
ee   deez  fd„«       «       Zˆ xZS )Ú	Emu3Modelc                 óô   •— t         ‰| �  |«       t        j                  |j                  «      | _        t        |j                  «      | _        t        |j                  «      | _        | j                  «        y rº   )rv   rw   r:  Ú_from_configÚtext_configÚ
text_modelr¢  Ú	vq_configÚvqmodelrà  Úvocabulary_mapÚvocabulary_mappingrÃ  r¸   s     €r1   rw   zEmu3Model.__init__B  sY   ø€ Ü‰Ñ˜Ô Ü'×4Ñ4°V×5GÑ5GÓHˆŒÜ  ×!1Ñ!1Ó2ˆŒÜ"<¸V×=RÑ=RÓ"SˆÔð 	�‰Õr0   c                 ó6   — | j                   j                  «       S rº   )rf  Úget_input_embeddingsrª   s    r1   rl  zEmu3Model.get_input_embeddingsK  s   € Ø�‰×3Ñ3Ó5Ð5r0   c                 ó:   — | j                   j                  |«       y rº   )rf  Úset_input_embeddings©r„   rV   s     r1   rn  zEmu3Model.set_input_embeddingsN  s   € Ø�‰×,Ñ,¨UÕ3r0   r“  rÄ  rJ   c                 óö   — | j                   j                  ||d¬«      }|j                  D �cg c]+  }| j                  j	                  |«      j                  «       ‘Œ- }}t        j                  |«      }|S c c}w )a  
        Tokenizes images into discrete tokens with VQGAN module. Converts
        obtained image tokens into BPE tokens and wraps with "boi" and "eoi"
        special tokens.

        Args:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
                The tensors corresponding to the input images.
            image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
                The sizes of the images in the batch, being (height, width) for each image.
        T)Úreturn_dict)rh  rÎ  r'   rj  r	  rÐ  r,   r8   )r„   r“  rÄ  Úvqmodel_outputsÚtokensÚbpe_tokens_listÚ
bpe_tokenss          r1   Úget_image_tokenszEmu3Model.get_image_tokensQ  sv   € ð 15·±×0CÑ0CÀLÐR]ÐkoÐ0CÓ0pˆàTc×TpÑTpö
ØJPˆD×#Ñ#×3Ñ3°FÓ;×CÑCÕEð
ˆð 
ô —Y‘Y˜Ó/ˆ
ØÐùò	
s   ­0A6zbTokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layerrŸ  rZ   c                 óü  —  | j                   j                  ||fddi|¤Ž}|D ��cg c];  \  }}|| j                   j                  z  || j                   j                  z  dz   z  ‘Œ= }}}|j                  D �cg c]+  }| j                  j                  |«      j                  «       ‘Œ- }	}t        j                  |	«      }
 | j                  «       |
«      }t        j                  ||«      }||_        |S c c}}w c c}w )z­
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
            The tensors corresponding to the input images.
        rq  Tr!   )rh  rÎ  r¾  r'   rj  r	  rÐ  r,   r8   rl  ÚsplitÚpooler_output)r„   r“  rÄ  rZ   rr  rß   rà   Úsplit_sizesrs  rt  ru  Úimage_embeddingsÚimage_featuress                r1   Úget_image_featureszEmu3Model.get_image_featuresd  s	  € ð 1D°·±×0CÑ0CØ˜+ñ1
Ø37ð1
Ø;Añ1
ˆð
 "-÷
á�˜ð �t—|‘|×9Ñ9Ñ9¸eÀtÇ|Á|×GiÑGiÑ>iÐlmÑ>mÓnð
ˆñ 
ð
 Ud×TpÑTpö
ØJPˆD×#Ñ#×3Ñ3°FÓ;×CÑCÕEð
ˆð 
ô —Y‘Y˜Ó/ˆ
Ø6˜4×4Ñ4Ó6°zÓBÐÜŸ™Ð%5°{ÓCˆØ(6ˆÔ%àÐùó
ùò
s   ¦A C3Á70C9r'   rß   rà   c                 ó°   — |dd…dd…f   j                  d||dz   «      }| j                  j                  |«      }| j                  j	                  |«      }|S )aå  
        Decodes generated image tokens from language model to continuous pixel values
        with VQGAN module via upsampling.

        Args:
            image_tokens (`torch.LongTensor` of shape `(batch_size, num_of_tokens)`):
                The tensors corresponding to the input images.
            height (`int`):
                Height of the generated image before upsampling.
            width (`int`):
                Width of the generated image before upsampling.
        Néýÿÿÿr3   r!   )rŠ   rj  r  rh  rÔ  )r„   r'   rß   rà   Ú	sequencesr¤  s         r1   Údecode_image_tokenszEmu3Model.decode_image_tokens€  sX   € ð !¢ C R C Ñ(×-Ñ-¨b°&¸%À!¹)ÓDˆ	Ø×.Ñ.×>Ñ>¸yÓIˆØ—‘×#Ñ# LÓ1ˆØˆr0   rE  rF  r|  c                 ó.  — |€m| | j                  «       t        j                  | j                  j                  t        j
                  |j                  ¬«      «      k(  }|j                  d«      }n|| j                  j                  k(  }|j                  «       }|j                  d   |j                  d   z  }|j                  d«      j                  |j                  «      }t        ||j                  d   z  |j                  «       k(  d|› d|› �«       |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        )r\   r  r3   r   r!   z6Image features and image tokens do not match, tokens: z, features: )rl  r,   Útensorrj  rå  Úlongr  ÚallrÚ   r7   r?   rf   r   Únumel)r„   rE  rF  r|  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r1   Úget_placeholder_maskzEmu3Model.get_placeholder_mask“  s  € ð ÐØ!.Ð2M°$×2KÑ2KÓ2MÜ—‘˜T×4Ñ4×CÑCÌ5Ï:É:Ð^k×^rÑ^rÔsó3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¨d×.EÑ.E×.TÑ.TÑ!TÐà+×/Ñ/Ó1ˆØ)×/Ñ/°Ñ2°^×5IÑ5IÈ!Ñ5LÑLÐØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐÜØ˜]×0Ñ0°Ñ4Ñ4¸×8LÑ8LÓ8NÑNØDÀ^ÐDTÐT`ÐaqÐ`rÐsô	
ð "Ð!r0   NrW   rÆ   r‡   rÇ   c	           	      óD  — |du |duz  rt        d«      ‚|€ | j                  «       |«      }|�Y| j                  ||«      j                  }
t	        j
                  |
d¬«      }
| j                  |||
¬«      }|j                  ||
«      } | j                  d|||||dœ|	¤Ž}|S )ap  
        image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
            The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using
            [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses
            [`Emu3ImageProcessor`] for processing images).
        NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either oner   r5   )rF  r|  )rW   rÆ   r‡   rF  rÇ   r/   )	rE  rl  r}  ry  r,   r8   rŠ  Úmasked_scatterrf  )r„   rE  r“  rÄ  rW   rÆ   r‡   rF  rÇ   rZ   r|  r‡  r\  s                r1   r’   zEmu3Model.forward«  sÚ   € ð( ˜Ð -°tÐ";Ò<ÜØsóð ð Ð Ø7˜D×5Ñ5Ó7¸	ÓBˆMàÐ#Ø!×4Ñ4°\À;ÓO×]Ñ]ˆNÜ"ŸY™Y ~¸1Ô=ˆNØ!%×!:Ñ!:Ø¨À~ð ";ó "Ðð *×8Ñ8Ð9KÈ^Ó\ˆMð "�$—/‘/ð 
Ø)Ø%Ø+Ø'Øñ
ð ñ
ˆð ˆr0   )NNNNNNNN)r(   r)   r*   rw   rl  rn  r,   r\  r-   rv  r   r   r   r   r–   r&   r}  rÞ  r”   r�  rŠ  r•   r
   rË   r   r’   r—   r˜   s   @r1   rb  rb  A  sì  ø„ ôò6ò4ð¨U×->Ñ->ð ÈU×M]ÑM]ð Ðbg×brÑbró ð& ÙØyôðØ!×-Ñ-ðØ<A×<LÑ<LðØX^Ð_qÑXrðà	Ð%Ñ	%òóó ðð0 €U‡]�]ƒ_ð°×0@Ñ0@ð È#ð ÐVYò ó ðð$"Ø×)Ñ)ð"Ø:?×:KÑ:Kð"Ø]b×]nÑ]nó"ð0 Øð .2Ø15Ø+/Ø.2Ø04Ø(,Ø26Ø!%ñ,à×#Ñ# dÑ*ð,ð ×'Ñ'¨$Ñ.ð,ð —\‘\ DÑ(ð	,ð
 Ÿ™ tÑ+ð,ð ×&Ñ&¨Ñ-ð,ð  ™ð,ð ×(Ñ(¨4Ñ/ð,ð ˜$‘;ð,ð Ð+Ñ,ð,ð 
Ð'Ñ	'ò,ó ó ô,r0   rb  c                   óÀ  ‡ — e Zd ZdZddiZˆ fd„Zd„ Zd„ Zdej                  fd„Z
d	„ Zee	 	 	 	 	 	 	 	 	 	 ddej                  d
z  dej                   d
z  dej"                  d
z  dej"                  d
z  dej                  d
z  ded
z  dej                   d
z  ded
z  dej                  d
z  deej"                  z  dee   deez  fd„«       «       Z	 	 	 	 	 	 	 dˆ fd„	Zˆ xZS )ÚEmu3ForConditionalGenerationr  rM  z$model.text_model.embed_tokens.weightc                 óî   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  j                  |j                  j                  d¬«      | _	        | j                  «        y rR  )rv   rw   rb  r  rb   r}   re  ry   r>  rN  rÃ  r¸   s     €r1   rw   z%Emu3ForConditionalGeneration.__init__à  sS   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ü—y‘y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÔkˆŒà�‰Õr0   c                 ó6   — | j                   j                  «       S rº   )r  rl  rª   s    r1   rl  z1Emu3ForConditionalGeneration.get_input_embeddingsç  s   € Ø�z‰z×.Ñ.Ó0Ð0r0   c                 ó:   — | j                   j                  |«       y rº   )r  rn  ro  s     r1   rn  z1Emu3ForConditionalGeneration.set_input_embeddingsê  s   € Ø�
‰
×'Ñ'¨Õ.r0   rJ   c                 ó   — | j                   S rº   )rN  rª   s    r1   Úget_output_embeddingsz2Emu3ForConditionalGeneration.get_output_embeddingsí  s   € Ø�|‰|Ðr0   c                 ó:   —  | j                   j                  di |¤ŽS r?  )r  r�  )r„   rZ   s     r1   r�  z0Emu3ForConditionalGeneration.decode_image_tokensð  s   € Ø-ˆt�z‰z×-Ñ-Ñ7°Ñ7Ð7r0   NrE  r“  rÄ  rW   rÆ   r‡   rF  rÇ   rS  rT  rZ   c           
      ó~  —  | j                   d||||||dœ|¤Ž}|d   }t        |
t        «      rt        |
 d«      n|
}| j	                  |dd…|dd…f   «      }d}|	�4 | j
                  d||	| j                  j                  j                  dœ|¤Ž}t        |||j                  |j                  |j                  ¬«      S )aÕ  
        image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`):
            The sizes of the images in the batch, being (height, width) for each image. Image sizes can be obtained using
            [`AutoImageProcessor`]. See [`Emu3ImageProcessor.__call__`] for details ([]`Emu3Processor`] uses
            [`Emu3ImageProcessor`] for processing images).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import Emu3Processor, Emu3ForConditionalGeneration
        >>> import torch
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image

        >>> model = Emu3ForConditionalGeneration.from_pretrained("BAAI/Emu3-Chat-hf", dtype=torch.bfloat16)
        >>> processor = Emu3Processor.from_pretrained("BAAI/Emu3-Chat-hf")

        >>> conversation = [
        ...     {
        ...     "role": "system",
        ...     "content": [
        ...         {"type": "text", "text": "You are a helpful assistant."},
        ...         ],
        ...     },
        ...     {
        ...     "role": "user",
        ...     "content": [
        ...         {"type": "image"},
        ...         {"type": "text", "text": "Please describe the image."},
        ...         ],
        ...     },
        ... ]

        >>> prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=[image], text=[prompt], return_tensors="pt").to(model.device, torch.bfloat16)

        >>> generated_ids = model.generate(**inputs, max_new_tokens=100, do_sample=False)
        >>> processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        ```rV  r   NrW  rX  r/   )r  r¯  r”   rZ  rN  r[  rp   re  r>  r   r‡   rH   r¦  )r„   rE  r“  rÄ  rW   rÆ   r‡   rF  rÇ   rS  rT  rZ   r\  rH   r]  rP  rY  s                    r1   r’   z$Emu3ForConditionalGeneration.forwardó  sç   € ð@ �$—*‘*ð 
ØØ)Ø%Ø+Ø'Øñ
ð ñ
ˆð   ™
ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%ð Ø f¸¿¹×9PÑ9P×9[Ñ9[ñØ_eñˆDô &ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r0   c	                 óJ   •— t        ‰| �  |f|||||||dœ|	¤Ž}
|s|rd |
d<   |
S )N)r‡   rW   rF  rÆ   r“  rÇ   Úis_first_iterationr“  )rv   Úprepare_inputs_for_generation)r„   rE  r‡   rW   rF  rÆ   rÇ   r“  r—  rZ   Úmodel_inputsr…   s              €r1   r˜  z:Emu3ForConditionalGeneration.prepare_inputs_for_generationP  sR   ø€ ô ‘wÑ<Øð

à+Ø)Ø'Ø%Ø%ØØ1ñ

ð ñ

ˆñ "¡iØ+/ˆL˜Ñ(àÐr0   )
NNNNNNNNNr   )NNNNTNF)r(   r)   r*   Úoutput_modalitiesr^  rw   rl  rn  rb   rm  r“  r�  r   r   r,   r-   r\  r•   r
   rË   r”   r   r   r–   r   r’   r˜  r—   r˜   s   @r1   rŽ  rŽ  Ü  s�  ø„ Ø)ÐØ*Ð,RÐSÐôò1ò/ð r§y¡yó ò8ð Øð .2Ø15Ø+/Ø.2Ø04Ø(,Ø26Ø!%Ø*.Ø-.ñY
à×#Ñ# dÑ*ðY
ð ×'Ñ'¨$Ñ.ðY
ð —\‘\ DÑ(ð	Y
ð
 Ÿ™ tÑ+ðY
ð ×&Ñ&¨Ñ-ðY
ð  ™ðY
ð ×(Ñ(¨4Ñ/ðY
ð ˜$‘;ðY
ð × Ñ  4Ñ'ðY
ð ˜eŸl™lÑ*ðY
ð Ð+Ñ,ðY
ð 
Ð'Ñ	'òY
ó ó ðY
ð| ØØØØØØ ÷ñ r0   rŽ  )rŽ  rL  r:  r  r¢  rb  r`  )r‰   )br‹  Úcollections.abcr   Údataclassesr   Ú	functoolsr   Útypingr   r,   Útorch.nnrb   Útorch.nn.functionalrc   r÷   Ú r   r°  Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úintegrationsr   r   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr    Úconfiguration_emu3r"   r#   r$   r&   r<   rG   r•   r”   rR   rm  r­   rl   rn   r›   r¯   r½   rÍ   rç   rú   r  r  r"  r,  r0  r;  rB  r  rM  rS  r^  rz  r‚  r˜  r¢  rà  r  r  r:  rL  rb  rŽ  Ú__all__r/   r0   r1   ú<module>r±     s‚  ðó, Ý $Ý !Ý %Ý ã Ý ß Ð å &Ý !ß .Ý )ß fÑ fÝ /Ý 9ß kÑ kß Kß FÝ &ß aÓ aß GÝ 5ß KÑ Kð Ø
ô1Ð5ó 1ó ó ð1ò(ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ñ2 Ð)Ó*ô@)�B—I‘Ió @)ó +ð@)ñF ˜YÓ'ôJ�"—)‘)ó Jó (ðJô(ˆb�i‰iô ô (Ð1ô (ôV$˜rŸy™yô $ôD	 R§Y¡Yô 	ô 2§9¡9ô ô�b—i‘iô ô:!˜2Ÿ9™9ô !ôH §	¡	ô ô. "§)¡)ô ô&.( 2§9¡9ô .(ôb<(˜2Ÿ9™9ô <(ô~;)˜bŸi™iô ;)ô|V˜Ÿ™ô Vô˜2Ÿ9™9ô ôD8˜Ÿ™ô 8ôv7�r—y‘yô 7ôtC�r—y‘yô CôLC�r—y‘yô CñL ðôôx2�ó x2óðx2÷v3%ñ 3%ðl ô˜/ó ó ðô*><˜"Ÿ)™)ô ><ðB ôH
Ð'ó H
ó ðH
ðV ôH
Ð)¨?ó H
ó ðH
ôVXÐ#ô XôvQÐ#6¸ô Qòh�r0   