Ë
    ÛÓ^jÞ¿  ã                   óð  — d Z ddlmZ ddlmZ ddlmZ ddlmZ ddl	Z	ddl
mc mZ ddl	mZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4  e+jj                  e6«      Z7e)e G d„ de«      «       «       Z8 G d„ dejr                  «      Z: G d„ dejr                  «      Z;d„ Z<dOd„Z= G d „ d!ejr                  «      Z> G d"„ d#ej~                  «      Z@d$e	j‚                  d%eBd&e	j‚                  fd'„ZC	 dPd(ejr                  d)e	j‚                  d*e	j‚                  d+e	j‚                  d,e	j‚                  dz  d-eDd.eDd/e&e(   fd0„ZE G d1„ d2ejr                  «      ZF G d3„ d4e«      ZG G d5„ d6e«      ZH G d7„ d8ejr                  «      ZI G d9„ d:ejr                  «      ZJ G d;„ d<ejr                  «      ZK G d=„ d>ejr                  «      ZL G d?„ d@ejr                  «      ZM G dA„ dB«      ZNe) G dC„ dDe$«      «       ZO e)dE¬F«       G dG„ dHeO«      «       ZPe) G dI„ dJeO«      «       ZQ e)dK¬F«       G dL„ dMeOe«      «       ZRg dN¢ZSy)QzPyTorch Chameleon model.é    )ÚCallable)Ú	dataclass)Úcached_property)ÚOptionalN)Únné   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚChameleonConfigÚChameleonVQVAEConfigc                   óŠ   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   y)ÚChameleonVQVAEModelOutputa­  
    quantized_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
        Quantized last hidden state from the VQ-VAE model.
    image_tokens (`torch.FloatTensor` of shape `(batch_size, config.vocab_size`):
        Indices of the image tokens predicted by the VQ-VAE model.
    embedding_loss (`torch.FloatTensor`):
        The embedding loss computed during quantization.
    NÚquantized_last_hidden_stateÚimage_tokensÚembedding_loss)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r%   ÚtorchÚFloatTensorÚ__annotations__r&   r'   © ó    ú{/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/chameleon/modeling_chameleon.pyr$   r$   2   sJ   … ñð =AÐ ×!2Ñ!2°TÑ!9Ó@Ø-1€L�%×#Ñ# dÑ*Ó1Ø/3€N�E×%Ñ%¨Ñ,Ô3r0   r$   c                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚChameleonRMSNormÚepsÚreturnNc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z?
        ChameleonRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	Parameterr,   ÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer4   Ú	__class__s      €r1   r8   zChameleonRMSNorm.__init__E   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr0   Úhidden_statesc                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor,   Úfloat32ÚpowÚmeanÚrsqrtr<   r;   )r=   r@   Úinput_dtypeÚvariances       r1   ÚforwardzChameleonRMSNorm.forwardM   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r0   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler;   Úshaper<   ©r=   s    r1   Ú
extra_reprzChameleonRMSNorm.extra_reprT   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr0   )ç�íµ ÷Æ°>)
r(   r)   r*   Úfloatr8   r,   ÚTensorrM   rR   Ú__classcell__©r?   s   @r1   r3   r3   D   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJr0   r3   c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚChameleonRotaryEmbeddingÚinv_freqNÚconfigc                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultrZ   F)Ú
persistentÚoriginal_inv_freq)r7   r8   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr[   Úrope_parametersr]   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r=   r[   ÚdeviceÚrope_init_fnrZ   r?   s        €r1   r8   z!ChameleonRotaryEmbedding.__init__\   s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr0   ri   ztorch.deviceÚseq_lenr5   ztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   rB   ©rE   )ri   rE   )	rd   Úgetattrr>   Únum_attention_headsr,   ÚarangeÚint64rF   rT   )r[   ri   rk   ÚbaseÚdimÚattention_factorrZ   s          r1   re   z8ChameleonRotaryEmbedding.compute_default_rope_parametersl   s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r0   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   rC   r    ÚmpsÚcpuF)Údevice_typeÚenabledrB   ©ru   ro   )rZ   rT   ÚexpandrP   rF   ri   Ú
isinstanceÚtypeÚstrr   Ú	transposer,   ÚcatÚcosrf   ÚsinrE   )
r=   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrz   ÚfreqsÚembrƒ   r„   s
             r1   rM   z ChameleonRotaryEmbedding.forwardŠ   sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$©N)NNN)r(   r)   r*   r,   rU   r.   r!   r8   Ústaticmethodr   ÚintrO   rT   re   Úno_gradr   rM   rV   rW   s   @r1   rY   rY   Y   s˜   ø… Ø�l‰lÓñV˜õ Vð  à)-Ø+/Ø"ñ*Ø $Ñ&ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r0   rY   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..NrC   rB   r|   )rP   r,   r‚   )r…   Úx1Úx2s      r1   Úrotate_halfr’   ›   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r0   c                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer’   )ÚqÚkrƒ   r„   Úunsqueeze_dimÚq_embedÚk_embeds          r1   Úapply_rotary_pos_embrš   £   sY   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr0   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚChameleonMLPc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  | j                  | j                  |j                  ¬«      | _	        t        j                  | j                  | j                  |j                  ¬«      | _
        t        |j                     | _        y )N©Úbias)r7   r8   r[   r>   Úintermediate_sizer   ÚLinearÚmlp_biasÚ	gate_projÚup_projÚ	down_projr	   Ú
hidden_actÚact_fn©r=   r[   r?   s     €r1   r8   zChameleonMLP.__init__¾   s¸   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Ñ/ˆ�r0   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S r‹   )r¥   r§   r£   r¤   )r=   r…   r¥   s      r1   rM   zChameleonMLP.forwardÉ   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr0   ©r(   r)   r*   r8   rM   rV   rW   s   @r1   rœ   rœ   ½   s   ø„ ô0ör0   rœ   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚChameleonLayerNorma†  
    LayerNorm but computes stats only over the last dim because Chameleon applies gamma and beta
    from each shard separately to each head, instead of reducing. We can apply each head's own
    gamma/beta by repeat-interleaving weights from each shard, but the stats have to be computed
    in the last dimension. This module applies gamma/beta manually to fulfill this requirement.
    c                 óB   •— t        ‰| �  |g|¢­i |¤Ž |d   f| _        y )NrC   )r7   r8   Únormalized_shape)r=   r>   ÚargsÚkwargsr?   s       €r1   r8   zChameleonLayerNorm.__init__Ö   s)   ø€ Ü‰Ñ˜Ð6 tÒ6¨vÒ6Ø!,¨R¡Ð 2ˆÕr0   c                 ó†   — t        j                  || j                  d d d¬«      }|| j                  z  | j                  z   }|S )Ngñhãˆµøä>©r4   )ÚFÚ
layer_normr®   r;   rŸ   ©r=   r@   s     r1   rM   zChameleonLayerNorm.forwardÚ   s=   € ÜŸ™ ]°D×4IÑ4IÈ4ÐQUÐ[_Ô`ˆØ%¨¯©Ñ3°d·i±iÑ?ˆØÐr0   )r(   r)   r*   r+   r8   rM   rV   rW   s   @r1   r¬   r¬   Î   s   ø„ ñô3ör0   r¬   r@   Ún_repr5   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r    N)rP   r}   Úreshape)r@   r¶   ÚbatchÚnum_key_value_headsÚslenrn   s         r1   Ú	repeat_kvr¼   á   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr0   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutr°   c                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )NrB   r   rC   )ru   rE   )ÚpÚtrainingr    )r¼   Únum_key_value_groupsr,   Úmatmulr�   r   Ú
functionalÚsoftmaxrG   rF   rE   rÃ   rÆ   Ú
contiguous)r½   r¾   r¿   rÀ   rÁ   rÂ   rÃ   r°   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r1   Úeager_attention_forwardrÐ   î   sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r0   c                   ó,  ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 	 	 	 	 ddej                  dej                  dz  dej                  dz  d	e
dz  d
ededej                  dz  deej                  ej                  dz  eej                     dz  f   fd„Zˆ xZS )ÚChameleonAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNr[   Ú	layer_idxc                 ól  •— t         ‰| �  «        || _        || _        |€-t        j                  d| j                  j                  › d�«       |j                  | _        |j                  | _	        |j                  | _        | j                  | j                  z  | _        |j                  | _        | j                  | j                  z  | _        |j                  | _        d| _        |j"                  | _        | j                  dz  | _        | j                  | j                  z  | j                  k7  r&t'        d| j                  › d| j                  › d�«      ‚t)        j*                  | j                  | j                  | j                  z  |j,                  ¬«      | _        t)        j*                  | j                  | j                  | j                  z  |j,                  ¬«      | _        t)        j*                  | j                  | j                  | j                  z  |j,                  ¬«      | _        t)        j*                  | j                  | j                  |j,                  ¬«      | _        t7        | j                  | j                  f«      | _        t7        | j                  | j                  f«      | _        y )	NzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Tç      à¿z?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).rž   )r7   r8   r[   rÓ   ÚloggerÚwarning_oncer?   r(   Úattention_dropoutr>   rq   Ú	num_headsrn   rº   rÇ   ra   Ú	is_causalÚmodel_parallel_sizerÂ   Ú
ValueErrorr   r¡   Úattention_biasÚq_projÚk_projÚv_projÚo_projr¬   Úq_normÚk_norm©r=   r[   rÓ   r?   s      €r1   r8   zChameleonAttention.__init__
  s  ø€ Ü‰ÑÔØˆŒØ"ˆŒØÐÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôð "(×!9Ñ!9ˆÔØ!×-Ñ-ˆÔØ×3Ñ3ˆŒØ×(Ñ(¨D¯N©NÑ:ˆŒØ#)×#=Ñ#=ˆÔ Ø$(§N¡N°d×6NÑ6NÑ$NˆÔ!Ø'-×'EÑ'EˆÔ$àˆŒØ#)×#=Ñ#=ˆÔ Ø—}‘} dÑ*ˆŒà�M‰M˜DŸN™NÑ*¨t×/?Ñ/?Ò?ÜØQÐRV×RbÑRbÐQcØ$ T§^¡^Ð$4°Bð8óð ô
 —i‘i × 0Ñ 0°$·.±.À4Ç=Á=Ñ2PÐW]×WlÑWlÔmˆŒÜ—i‘i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Ñ2ZÐag×avÑavÔwˆŒÜ—i‘i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Ñ2ZÐag×avÑavÔwˆŒÜ—i‘i × 0Ñ 0°$×2BÑ2BÈ×I^ÑI^Ô_ˆŒÜ(¨$¯.©.¸$¿-¹-Ð)HÓIˆŒÜ(¨$×*BÑ*BÀDÇMÁMÐ)RÓSˆ�r0   r@   rÁ   r†   Úpast_key_valuesÚoutput_attentionsÚ	use_cacheÚposition_embeddingsr5   c                 ób  — |j                  «       \  }	}
}| j                  |«      }| j                  |«      }| j                  |«      }|j	                  d| j
                  | j                  «      }| j                  |«      }|j	                  d| j                  | j                  «      }| j                  |«      }|j	                  |	|
| j
                  | j                  «      j                  dd«      }|j	                  |	|
| j                  | j                  «      j                  dd«      }|j                  |	|
| j                  | j                  «      j                  dd«      }|\  }}t        ||||«      \  }}|� |j                  ||| j                  «      \  }}t        j                   | j"                  j$                  t&        «      } || ||||f| j(                  sdn| j*                  | j,                  dœ|¤Ž\  }}|j	                  |	|
d«      j/                  «       }| j1                  |«      }||fS )NrC   r    rB   ç        )rÃ   rÂ   )ÚsizerÞ   rß   rà   r¸   rÙ   rn   râ   rº   rã   r�   Úviewrš   ÚupdaterÓ   r   Úget_interfacer[   Ú_attn_implementationrÐ   rÆ   rØ   rÂ   rË   rá   )r=   r@   rÁ   r†   rå   ræ   rç   rè   r°   ÚbszÚq_lenÚ_Úquery_statesrÌ   rÍ   rƒ   r„   Úattention_interfacerÏ   rÎ   s                       r1   rM   zChameleonAttention.forward.  s  € ð &×*Ñ*Ó,‰ˆˆU�Aà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆà#×+Ñ+¨B°·±ÀÇÁÓNˆØ—{‘{ <Ó0ˆà×'Ñ'¨¨D×,DÑ,DÀdÇmÁmÓTˆ
Ø—[‘[ Ó,ˆ
à#×+Ñ+¨C°¸¿¹ÈÏÉÓV×`Ñ`ÐabÐdeÓfˆØ×'Ñ'¨¨U°D×4LÑ4LÈdÏmÉmÓ\×fÑfÐghÐjkÓlˆ
Ø#×(Ñ(¨¨e°T×5MÑ5MÈtÏ}É}Ó]×gÑgÐhiÐklÓmˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð "×)Ñ)¨#¨u°bÓ9×DÑDÓFˆØ—k‘k +Ó.ˆà˜LÐ(Ð(r0   r‹   ©NNNFFN)r(   r)   r*   r+   r!   r�   r8   r,   rU   Ú
LongTensorr
   ÚboolrO   rM   rV   rW   s   @r1   rÒ   rÒ     sÔ   ø„ ÙGñ"T˜ð "T¸3À¹:õ "TðN /3Ø04Ø(,Ø"'ØØ37ñ3)à—|‘|ð3)ð Ÿ™ tÑ+ð3)ð ×&Ñ&¨Ñ-ð	3)ð
  ™ð3)ð  ð3)ð ð3)ð #Ÿ\™\¨DÑ0ð3)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷3)r0   rÒ   c                   ó(  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
e
dz  dej                  dz  deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚChameleonDecoderLayerr[   rÓ   c                 ó   •— t         ‰| �  «        |j                  | _        t        ||¬«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _	        t        |j                  |j                  ¬«      | _
        y ©N)r[   rÓ   r²   ©r7   r8   r>   rÒ   Ú	self_attnrœ   Úmlpr3   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrä   s      €r1   r8   zChameleonDecoderLayer.__init__f  óm   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä+°6ÀYÔOˆŒä Ó'ˆŒÜ/°×0BÑ0BÈ×H[ÑH[Ô\ˆÔÜ(8¸×9KÑ9KÐQW×QdÑQdÔ(eˆÕ%r0   Nr@   rÁ   r†   rå   ræ   rç   rè   r5   c                 óØ   — |}	| j                  |«      } | j                  d|||||||dœ|¤Ž\  }}
|	|z   }|}	| j                  |«      }| j                  |«      }|	|z   }|f}|r||
fz  }|S )a<  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            kwargs (`dict`, *optional*):
                Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
                into the model
        ©r@   rÁ   r†   rå   ræ   rç   rè   r/   )r   rý   r  rþ   ©r=   r@   rÁ   r†   rå   ræ   rç   rè   r°   ÚresidualÚself_attn_weightsÚoutputss               r1   rM   zChameleonDecoderLayer.forwardp  s®   € ð8 !ˆà×,Ñ,¨]Ó;ˆð ,:¨4¯>©>ð 	,
Ø'Ø)Ø%Ø+Ø/ØØ 3ñ	,
ð ñ	,
Ñ(ˆÐ(ð ! =Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ  =Ñ0ˆà Ð"ˆáØÐ)Ð+Ñ+ˆGàˆr0   rõ   ©r(   r)   r*   r!   r�   r8   r,   rU   rö   r
   r÷   rO   r-   rM   rV   rW   s   @r1   rù   rù   e  sÙ   ø„ ðf˜ð f¸3õ fð /3Ø04Ø(,Ø).Ø!&Ø37ñ8à—|‘|ð8ð Ÿ™ tÑ+ð8ð ×&Ñ&¨Ñ-ð	8ð
  ™ð8ð   $™;ð8ð ˜$‘;ð8ð #Ÿ\™\¨DÑ0ð8ð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷8r0   rù   c                   ó(  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
e
dz  dej                  dz  deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚChameleonSwinDecoderLayerr[   rÓ   c                 ó   •— t         ‰| �  «        |j                  | _        t        ||¬«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _	        t        |j                  |j                  ¬«      | _
        y rû   rü   rä   s      €r1   r8   z"ChameleonSwinDecoderLayer.__init__¬  r  r0   Nr@   rÁ   r†   rå   ræ   rç   rè   r5   c                 óØ   — |}	 | j                   d|||||||dœ|¤Ž\  }}
| j                  |«      }|	|z   }|}	| j                  |«      }| j                  |«      }|	|z   }|f}|r||
fz  }|S )ad  
        Args:
            hidden_states (`torch.FloatTensor`):
                input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Indices of positions of each input sequence tokens in the position embeddings
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
        r  r/   )rý   r   rþ   r  r  s               r1   rM   z!ChameleonSwinDecoderLayer.forward¶  s¬   € ð: !ˆð ,:¨4¯>©>ð 	,
Ø'Ø)Ø%Ø+Ø/ØØ 3ñ	,
ð ñ	,
Ñ(ˆÐ(ð ×,Ñ,¨]Ó;ˆØ  =Ñ0ˆà ˆØŸ™ Ó/ˆØ×5Ñ5°mÓDˆØ  =Ñ0ˆØ Ð"ˆáØÐ)Ð+Ñ+ˆGàˆr0   rõ   r	  rW   s   @r1   r  r  «  sÙ   ø„ ðf˜ð f¸3õ fð /3Ø04Ø(,Ø).Ø!&Ø37ñ6à—|‘|ð6ð Ÿ™ tÑ+ð6ð ×&Ñ&¨Ñ-ð	6ð
  ™ð6ð   $™;ð6ð ˜$‘;ð6ð #Ÿ\™\¨DÑ0ð6ð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷6r0   r  c                   óB   ‡ — e Zd ZdZˆ fd„Zdej                  fd„Zˆ xZS )ÚChameleonVQVAEVectorQuantizeraâ  
    A module for vector quantization using learned embedding vectors.

    This module implements the quantization process similar to te one described in
    the VQ-VAE (Vector Quantized Variational AutoEncoder) paper. It quantizes continuous
    input vectors into discrete codebook vectors, which are learned during training.
    Current implementation improves over previous ones by avoiding costly matrix multiplications
    and allowing for post-hoc remapping of indices.
    c                 óè   •— t         ‰| �  «        |j                  | _        |j                  | _        t        |dd«      | _        t        j                  | j                  | j                  «      | _	        y )NÚbetag      Ð?)
r7   r8   Únum_embeddingsÚ	embed_dimÚembedding_dimrp   r  r   Ú	EmbeddingÚ	embeddingr¨   s     €r1   r8   z&ChameleonVQVAEVectorQuantizer.__init__ú  sX   ø€ Ü‰ÑÔØ$×3Ñ3ˆÔØ#×-Ñ-ˆÔÜ˜F F¨DÓ1ˆŒ	äŸ™ d×&9Ñ&9¸4×;MÑ;MÓNˆ�r0   Úhidden_statec           
      óL  — |j                  dddd«      j                  «       }|j                  d| j                  «      }t	        j
                  |dz  dd¬«      t	        j
                  | j                  j                  dz  d¬«      z   dt	        j                  d	|| j                  j                  j                  dd«      «      z  z
  }t	        j                  |d¬«      }| j                  |«      j                  |j                  «      }t	        j                  |j                  «       |z
  dz  «      | j                  t	        j                  ||j                  «       z
  dz  «      z  z   }|||z
  j                  «       z   }|j                  dddd«      j                  «       }|||fS )
Nr   rB   r   r    rC   T)ru   rD   r|   z	bd,dn->bn)ÚpermuterË   rì   r  r,   Úsumr  r;   Úeinsumr�   ÚargminrP   rI   Údetachr  )r=   r  Úhidden_state_flattenedÚ	distancesÚmin_encoding_indicesÚhidden_state_quantÚlosss          r1   rM   z%ChameleonVQVAEVectorQuantizer.forward  sˆ  € Ø#×+Ñ+¨A¨q°!°QÓ7×BÑBÓDˆØ!-×!2Ñ!2°2°t×7IÑ7IÓ!JÐô �I‰IÐ,¨aÑ/°QÀÔEÜ�i‰i˜Ÿ™×-Ñ-¨qÑ0°aÔ8ñ9à”%—,‘,˜{Ð,BÀDÇNÁN×DYÑDY×DcÑDcÐdeÐghÓDiÓjÑjñkð 	ô  %Ÿ|™|¨I¸1Ô=ÐØ!Ÿ^™^Ð,@ÓA×FÑFÀ|×GYÑGYÓZÐô �z‰zÐ-×4Ñ4Ó6¸ÑEÈ!ÑKÓLÈtÏyÉyÔ[`×[eÑ[eØ ,×"5Ñ"5Ó"7Ñ7¸AÑ=ó\
ñ P
ñ 
ˆð
 *Ð-?À,Ñ-N×,VÑ,VÓ,XÑXÐð 0×7Ñ7¸¸1¸aÀÓC×NÑNÓPÐà! 4Ð)=Ð=Ð=r0   )	r(   r)   r*   r+   r8   r,   rU   rM   rV   rW   s   @r1   r  r  ï  s   ø„ ñôOð> E§L¡L÷ >r0   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú#ChameleonVQVAEEncoderConvDownsamplec                 ó`   •— t         ‰| �  «        t        j                  ||ddd¬«      | _        y )Nr   rB   r   ©Úkernel_sizeÚstrideÚpadding)r7   r8   r   ÚConv2dÚconv©r=   Úin_channelsr?   s     €r1   r8   z,ChameleonVQVAEEncoderConvDownsample.__init__  s'   ø€ Ü‰ÑÔÜ—I‘I˜k¨;ÀAÈaÐYZÔ[ˆ�	r0   c                 óZ   — t        j                  |ddd¬«      }| j                  |«      }|S )N)r   r    r   r    Úconstantr   )ÚpadÚmoderÀ   )r³   r0  r+  rµ   s     r1   rM   z+ChameleonVQVAEEncoderConvDownsample.forward#  s+   € äŸ™˜m°ÀJÐVWÔXˆØŸ	™	 -Ó0ˆØÐr0   rª   rW   s   @r1   r$  r$    s   ø„ ô\ör0   r$  c                   ó*   ‡ — e Zd Z	 	 dˆ fd„	Zd„ Zˆ xZS )Ú ChameleonVQVAEEncoderResnetBlockc                 óæ  •— t         ‰| �  «        || _        |€|n|| _        || _        t
        j                  j                  d|dd¬«      | _        t
        j                  j                  ||ddd¬«      | _
        t
        j                  j                  d|dd¬«      | _        t
        j                  j                  |j                  «      | _        t
        j                  j                  ||ddd¬«      | _        | j                  | j                  k7  r`| j                  r*t
        j                  j                  ||ddd¬«      | _        y t
        j                  j                  ||ddd¬«      | _        y y )	Né    rS   T©Ú
num_groupsÚnum_channelsr4   Úaffiner   r    r&  r   )r7   r8   r-  Úout_channelsÚuse_conv_shortcutr,   r   Ú	GroupNormÚnorm1r*  Úconv1Únorm2ÚDropoutrÃ   Úconv2Úconv_shortcutÚnin_shortcut)r=   r[   r-  r:  rB  r?   s        €r1   r8   z)ChameleonVQVAEEncoderResnetBlock.__init__+  s1  ø€ ô 	‰ÑÔØ&ˆÔØ+7Ð+?™KÀ\ˆÔØ!.ˆÔä—X‘X×'Ñ'°2ÀKÐUYÐbfÐ'ÓgˆŒ
Ü—X‘X—_‘_ [°,ÈAÐVWÐab�_ÓcˆŒ
Ü—X‘X×'Ñ'°2ÀLÐVZÐcgÐ'ÓhˆŒ
Ü—x‘x×'Ñ'¨¯©Ó7ˆŒÜ—X‘X—_‘_ \°<ÈQÐWXÐbc�_ÓdˆŒ
Ø×Ñ˜t×0Ñ0Ò0Ø×%Ò%Ü%*§X¡X§_¡_°[À,Ð\]ÐfgÐqr _Ó%s�Õ"ä$)§H¡H§O¡O°KÀÐ[\ÐefÐpq OÓ$r�Õ!ð	 1r0   c                 ó²  — |}| j                  |«      }|t        j                  |«      z  }| j                  |«      }| j	                  |«      }|t        j                  |«      z  }| j                  |«      }| j                  |«      }| j                  | j                  k7  r3| j                  r| j                  |«      }||z   S | j                  |«      }||z   S r‹   )r=  r,   Úsigmoidr>  r?  rÃ   rA  r-  r:  r;  rB  rC  )r=   r@   r  s      r1   rM   z(ChameleonVQVAEEncoderResnetBlock.forwardB  sÊ   € Ø ˆØŸ
™
 =Ó1ˆØœŸ™ }Ó5Ñ5ˆØŸ
™
 =Ó1ˆàŸ
™
 =Ó1ˆØœŸ™ }Ó5Ñ5ˆØŸ™ ]Ó3ˆØŸ
™
 =Ó1ˆà×Ñ˜t×0Ñ0Ò0Ø×%Ò%Ø×-Ñ-¨hÓ7�ð ˜-Ñ'Ð'ð  ×,Ñ,¨XÓ6�à˜-Ñ'Ð'r0   )NFrª   rW   s   @r1   r3  r3  *  s   ø„ ð
 Øõsö.(r0   r3  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚChameleonVQVAEEncoderAttnBlockc                 óÈ  •— t         ‰| �  «        || _        t        j                  j                  d|dd¬«      | _        t        j                  j                  ||ddd¬«      | _        t        j                  j                  ||ddd¬«      | _	        t        j                  j                  ||ddd¬«      | _
        t        j                  j                  ||ddd¬«      | _        y )Nr5  rS   Tr6  r    r   r&  )r7   r8   r-  r,   r   r<  Únormr*  r•   r–   ÚvÚproj_outr,  s     €r1   r8   z'ChameleonVQVAEEncoderAttnBlock.__init__W  s·   ø€ Ü‰ÑÔØ&ˆÔä—H‘H×&Ñ&°"À;ÐTXÐaeÐ&ÓfˆŒ	Ü—‘—‘ ¨kÀqÐQRÐ\]�Ó^ˆŒÜ—‘—‘ ¨kÀqÐQRÐ\]�Ó^ˆŒÜ—‘—‘ ¨kÀqÐQRÐ\]�Ó^ˆŒÜŸ™Ÿ™¨°[ÈaÐXYÐcd˜Óeˆ�r0   c                 ót  — |}| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|j                  \  }}}}	|j                  ||||	z  «      j                  ddd«      }|j                  ||||	z  «      }t        j                  ||«      }
|
t        |«      dz  z  }
t        j                  |
d¬«      }
|j                  ||||	z  «      }|
j                  ddd«      }
t        j                  ||
«      j                  ||||	«      }| j                  |«      }||z   S )Nr   rB   r    rÕ   r|   )rI  r•   r–   rJ  rP   r¸   r  r,   Úbmmr�   r³   rÊ   rK  )r=   r@   r  ró   rÌ   rÍ   Ú
batch_sizeÚchannelsÚheightÚwidthrÎ   rÏ   s               r1   rM   z&ChameleonVQVAEEncoderAttnBlock.forwarda  s5  € Ø ˆØŸ	™	 -Ó0ˆØ—v‘v˜mÓ,ˆØ—V‘V˜MÓ*ˆ
Ø—v‘v˜mÓ,ˆð /;×.@Ñ.@Ñ+ˆ
�H˜f eØ#×+Ñ+¨J¸À&È5Á.ÓQ×YÑYÐZ[Ð]^Ð`aÓbˆØ×'Ñ'¨
°H¸fÀu¹nÓMˆ
Ü—y‘y ¨zÓ:ˆØ#¤s¨8£}¸Ñ'>Ñ?ˆÜ—y‘y °1Ô5ˆð $×+Ñ+¨J¸À&È5Á.ÓQˆØ#×+Ñ+¨A¨q°!Ó4ˆÜ—i‘i ¨lÓ;×CÑCÀJÐPXÐZ`ÐbgÓhˆà—m‘m KÓ0ˆØ˜+Ñ%Ð%r0   rª   rW   s   @r1   rG  rG  V  s   ø„ ôfö&r0   rG  c                   ó>   ‡ — e Zd Zˆ fd„Zdej
                  fd„Zˆ xZS )ÚChameleonVQVAEEncoderc           	      óø  •— t         ‰| �  «        t        |j                  «      | _        |j
                  | _        |j                  }|j                  }|j                  }|j                  }|j                  }|j                  }t        j                  j                  ||ddd¬«      | _        |}dt        |«      z   }	|	| _        t        j"                  «       | _        t'        | j                  «      D �]  }
t        j"                  «       }t        j"                  «       }||	|
   z  }|||
   z  }t'        | j
                  «      D ]g  }|j)                  t+        |||¬«      «       |}|j,                  €Œ/||j,                  v sŒ>|j.                  dk(  sŒN|j)                  t1        |«      «       Œi t        j2                  «       }||_        ||_        |
| j                  dz
  k7  rt9        |«      |_        |dz  }| j$                  j)                  |«       �Œ! t        j2                  «       | _        t+        ||¬«      | j<                  _        |j.                  dk(  rt1        |«      nt        j@                  «       | j<                  _!        t+        |||¬«      | j<                  _"        t        j                  jG                  d|d	d
¬«      | _$        t        j                  j                  ||rd|z  n|ddd¬«      | _%        y )Nr   r    r&  ©r    )r[   r-  r:  ÚvanillarB   r5  rS   Tr6  )&r7   r8   ÚlenÚchannel_multiplierÚnum_resolutionsÚnum_res_blocksÚbase_channelsÚ
resolutionr-  Údouble_latentÚlatent_channelsr,   r   r*  Úconv_inrO   Úin_channel_multiplierÚ
ModuleListÚdownÚrangeÚappendr3  Úattn_resolutionsÚ	attn_typerG  ÚModuleÚblockÚattnr$  Ú
downsampleÚmidÚblock_1ÚIdentityÚattn_1Úblock_2r<  Únorm_outÚconv_out)r=   r[   r[  r\  r-  r]  r^  rX  Úcurr_resr`  Úi_levelrh  ri  Úblock_inÚ	block_outÚi_blockrb  r?   s                    €r1   r8   zChameleonVQVAEEncoder.__init__z  s¢  ø€ Ü‰ÑÔä" 6×#<Ñ#<Ó=ˆÔØ$×3Ñ3ˆÔØ×,Ñ,ˆØ×&Ñ&ˆ
Ø×(Ñ(ˆØ×,Ñ,ˆØ ×0Ñ0ˆØ#×6Ñ6Ðä—x‘x—‘ {°MÈqÐYZÐde�ÓfˆŒàˆØ $¤uÐ-?Ó'@Ñ @ÐØ%:ˆÔ"Ü—M‘M“OˆŒ	Ü˜T×1Ñ1Ó2ó 	#ˆGÜ—M‘M“OˆEÜ—=‘=“?ˆDØ$Ð'<¸WÑ'EÑEˆHØ%Ð(:¸7Ñ(CÑCˆIÜ  ×!4Ñ!4Ó5ò J�Ø—‘Ü4Ø%Ø$,Ø%.ôôð %�à×+Ñ+Ñ7Ø  F×$;Ñ$;Ò;Ø×(Ñ(¨IÓ5à—K‘KÔ >¸xÓ HÕIðJô  —9‘9“;ˆDØˆDŒJØˆDŒIØ˜$×.Ñ.°Ñ2Ò2Ü"EÀhÓ"O�”Ø# q™=�Ø�I‰I×Ñ˜TÖ"ð7	#ô: —9‘9“;ˆŒÜ;ØØ Ø!ô
ˆ�‰Ôð
 GM×FVÑFVÐZcÒFcÔ8¸ÔBÔik×itÑitÓivˆ�‰ŒÜ;ØØ Ø!ô
ˆ�‰Ôô Ÿ™×*Ñ*°bÀxÐUYÐbfÐ*ÓgˆŒÜŸ™Ÿ™ØÙ#0ˆA�Ò°oØØØð (ó 
ˆ�r0   Úpixel_valuesc                 ó2  — | j                  |«      g}t        | j                  «      D ]Ü  }t        | j                  «      D ]  } | j                  |   j
                  |   |d   «      }t        | j                  |   j                  «      dkD  r" | j                  |   j                  |   |«      }|j                  |«       Œ� || j                  dz
  k7  sŒ­|j                  | j                  |   j                  |d   «      «       ŒÞ |d   }| j                  j                  |«      }| j                  j                  |«      }| j                  j                  |«      }| j                  |«      }|t        j                   |«      z  }| j#                  |«      }|S )NrC   r   r    )r_  rc  rY  rZ  rb  rh  rW  ri  rd  rj  rk  rl  rn  ro  rp  r,   rE  rq  )r=   rw  r@   rs  rv  r  Úlast_hidden_states          r1   rM   zChameleonVQVAEEncoder.forward¿  s„  € àŸ™ lÓ3Ð4ˆÜ˜T×1Ñ1Ó2ò 		WˆGÜ  ×!4Ñ!4Ó5ò 3�Ø@˜tŸy™y¨Ñ1×7Ñ7¸Ñ@Ø! "Ñ%ó �ô �t—y‘y Ñ)×.Ñ.Ó/°!Ò3Ø#C 4§9¡9¨WÑ#5×#:Ñ#:¸7Ñ#CÀLÓ#Q�LØ×$Ñ$ \Õ2ð3ð ˜$×.Ñ.°Ñ2Ó2Ø×$Ñ$ T§Y¡Y¨wÑ%7×%BÑ%BÀ=ÐQSÑCTÓ%UÕVð		Wð *¨"Ñ-ÐØ ŸH™H×,Ñ,Ð->Ó?ÐØ ŸH™HŸO™OÐ,=Ó>ÐØ ŸH™H×,Ñ,Ð->Ó?Ðð !ŸM™MÐ*;Ó<ÐØœUŸ]™]Ð+<Ó=Ñ=ÐØ ŸM™MÐ*;Ó<ÐØ Ð r0   )r(   r)   r*   r8   r,   rö   rM   rV   rW   s   @r1   rS  rS  y  s   ø„ ôC
ðJ! E×$4Ñ$4÷ !r0   rS  c                   ó®   — e Zd ZdZd„ Zed„ «       Zed„ «       Zed„ «       Zed„ «       Z	ed„ «       Z
ed„ «       Zd	ej                  d
ej                  fd„Zy)ÚChameleonImageVocabularyMappingzM
    A class for mapping discrete image tokens from VQGAN to BPE tokens.
    c                 ó>   — || _         |j                  d«      | _        y )Nz<image>)Ú	vocab_mapÚgetÚimage_token_id)r=   r}  s     r1   r8   z(ChameleonImageVocabularyMapping.__init__ß  s   € Ø"ˆŒØ'Ÿm™m¨IÓ6ˆÕr0   c                 ój   — | j                   j                  «       D ��ci c]  \  }}||“Œ
 c}}S c c}}w r‹   )r}  Úitems©r=   r–   rJ  s      r1   Úval2namez(ChameleonImageVocabularyMapping.val2nameã  s+   € à!%§¡×!5Ñ!5Ó!7×8™˜˜A��1‘Ó8Ð8ùÓ8ó   ž/c           	      óž   — t        | j                  j                  «       D ��cg c]  \  }}|j                  d«      sŒ|‘Œ c}}«      S c c}}w )NÚIMGIMG)Úsortedr}  r�  Ú
startswith)r=   ÚnameÚvals      r1   r&   z,ChameleonImageVocabularyMapping.image_tokensç  s8   € ä¨D¯N©N×,@Ñ,@Ó,B×`™y˜t SÀdÇoÁoÐV^ÕF_’sÓ`ÓaÐaùÓ`s
   £A	
½A	
c           
      ó  ‡— t        d«      D �ci c]#  }t        t        d«      |z   «      t        |«      “Œ% c}Šdt        dt        fˆfd„}| j                  D �ci c]!  }|t         || j                  |   «      «      “Œ# c}S c c}w c c}w )Né
   ÚAÚold_namer5   c                 óP   •— dj                  ˆfd„| t        d«      d D «       «      S )NÚ c              3   óB   •K  — | ]  }‰j                  ||«      –— Œ y ­wr‹   )r~  )Ú.0ÚcÚimg_tkn_chr_mappings     €r1   ú	<genexpr>zIChameleonImageVocabularyMapping.bpe2img.<locals>.remap.<locals>.<genexpr>ð  s   øè ø€ Ò_¸QÐ.×2Ñ2°1°a×8Ñ_ùs   ƒr†  rC   )ÚjoinrW  )rŽ  r”  s    €r1   Úremapz6ChameleonImageVocabularyMapping.bpe2img.<locals>.remapï  s$   ø€ Ø—7‘7Ó_À(Ì3ÈxË=Ð[]ÐB^Ô_Ó_Ð_r0   )rc  ÚchrÚordr€   r&   r�   rƒ  )r=   Úir—  Útokr”  s       @r1   Úbpe2imgz'ChameleonImageVocabularyMapping.bpe2imgë  sƒ   ø€ äBGÈÃ)ÖL¸Qœs¤3 s£8¨a¡<Ó0´#°a³&Ñ8ÒLÐð	`œCð 	`¤Cõ 	`ð @D×?PÑ?PÖQ¸�”S™˜tŸ}™}¨SÑ1Ó2Ó3Ñ3ÒQÐQùò Mùò
 Rs   �(BÁ&Bc                 ój   — | j                   j                  «       D ��ci c]  \  }}||“Œ
 c}}S c c}}w r‹   )rœ  r�  r‚  s      r1   Úimg2bpez'ChameleonImageVocabularyMapping.img2bpeô  s+   € à!%§¡×!3Ñ!3Ó!5×6™˜˜A��1‘Ó6Ð6ùÓ6r„  c                 óÚ   — t        j                  t        | j                  j	                  «       «      «      t        j                  t        | j                  j                  «       «      «      fS r‹   )r,   Útensorr‡  rœ  ÚkeysÚvaluesrQ   s    r1   Úbpe2img_search_tensorsz6ChameleonImageVocabularyMapping.bpe2img_search_tensorsø  sC   € ä�|‰|œF 4§<¡<×#4Ñ#4Ó#6Ó7Ó8¼%¿,¹,ÄvÈdÏlÉl×NaÑNaÓNcÓGdÓ:eÐeÐer0   c                 óæ   — t        j                  t        | j                  j	                  «       «      dz   t         j
                  ¬«      }| j                  j                  «       D ]
  \  }}|||<   Œ |S )Nr    ro   )r,   ÚzerosÚmaxrž  r¡  r�   r�  )r=   Úmappingr–   rJ  s       r1   Úimg2bpe_mapping_tensorz6ChameleonImageVocabularyMapping.img2bpe_mapping_tensorü  s[   € ä—+‘+œc $§,¡,×"3Ñ"3Ó"5Ó6¸Ñ:Ä%Ç)Á)ÔLˆØ—L‘L×&Ñ&Ó(ò 	‰DˆAˆqØˆG�AŠJð	àˆr0   Ú	img_batchr5   c                 óx   — |j                   }| j                  |j                  d«         }|j                  |«      S )Nry   )ri   r¨  rF   )r=   r©  ri   Ú
img_tokenss       r1   Úconvert_img2bpez/ChameleonImageVocabularyMapping.convert_img2bpe  s5   € Ø×!Ñ!ˆØ×0Ñ0°·±¸eÓ1DÑEˆ
Ø�}‰}˜VÓ$Ð$r0   N)r(   r)   r*   r+   r8   r   rƒ  r&   rœ  rž  r£  r¨  r,   rU   r¬  r/   r0   r1   r{  r{  Ú  s¥   „ ñò7ð ñ9ó ð9ð ñbó ðbð ñRó ðRð ñ7ó ð7ð ñfó ðfð ñó ðð%¨¯©ð %¸%¿,¹,ô %r0   r{  c                   óV   — e Zd ZU eed<   dZdZdZddgZddgZ	dZ
dZdZdZdZeeged	œZy
)ÚChameleonPreTrainedModelr[   Úmodel)ÚimageÚtextTrù   r  rå   Úcausal_mask©r@   Ú
attentionsN)r(   r)   r*   r!   r.   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphÚ_supports_flex_attnÚ_supports_attention_backendrù   r  rÒ   Ú_can_record_outputsr/   r0   r1   r®  r®  	  sb   … àÓØÐØ(ÐØ&*Ð#Ø0Ð2MÐNÐØ#4°mÐ"DÐØÐØ€Nà!ÐØÐØ"&Ðà/Ð1JÐKØ(ñÑr0   r®  aW  
    The VQ-VAE model used in Chameleon for encoding/decoding images into discrete tokens.
    This model follows the "Make-a-scene: Scene-based text-to-image generation with human priors" paper from
    [ Oran Gafni, Adam Polyak, Oron Ashual, Shelly Sheynin, Devi Parikh, and Yaniv
    Taigman](https://huggingface.co/papers/2203.13131).
    ©Úcustom_introc                   ó„   ‡ — e Zd ZU eed<   g d¢ZeedœZdefˆ fd„Z	e
edej                  dee   defd„«       «       Zˆ xZS )	ÚChameleonVQVAEr[   )r  rG  r3  r³  c                 óŒ  •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        t        j                  j                  |j                  |j                  d«      | _        t        j                  j                  |j                  |j                  d«      | _        | j                  «        | j                  «        y )Nr    )r7   r8   rS  Úencoderr  Úquantizer,   r   r*  r^  r  Ú
quant_convÚpost_quant_convÚevalÚ	post_initr¨   s     €r1   r8   zChameleonVQVAE.__init__1  s†   ø€ Ü‰Ñ˜Ô ä,¨VÓ4ˆŒÜ5°fÓ=ˆŒÜŸ(™(Ÿ/™/¨&×*@Ñ*@À&×BRÑBRÐTUÓVˆŒÜ$Ÿx™xŸ™¨v×/?Ñ/?À×AWÑAWÐYZÓ[ˆÔØ�	‰	ŒØ�‰Õr0   rw  r°   r5   c                 óŽ   — | j                  |«      }| j                  |«      }| j                  |«      \  }}}t        ||||¬«      S )N)ry  r%   r&   r'   )rÅ  rÇ  rÆ  r$   )r=   rw  r°   r@   Úconv_hidden_statesr%   Úemb_lossÚindicess           r1   ÚencodezChameleonVQVAE.encode;  sQ   € ð
 Ÿ™ \Ó2ˆØ!Ÿ_™_¨]Ó;ÐØ9=¿¹ÐGYÓ9ZÑ6Ð# X¨wÜ(Ø+Ø(CØ Ø#ô	
ð 	
r0   )r(   r)   r*   r"   r.   r¸  r3  rG  r¿  r8   r   r   r,   rö   r   r   r$   rÏ  rV   rW   s   @r1   rÃ  rÃ    sp   ø… ð !Ó òÐð :Ø4ñÐð
Ð3õ ð  Øð
Ø!×,Ñ,ð
Ø8>Ð?QÑ8Rð
à	"ò
ó ó  ô
r0   rÃ  c                   óÞ  ‡ — e Zd Zdefˆ fd„Zdej                  fd„Ze e	d¬«      dej                  de
e   deez  fd	„«       «       Zd
ej                  dej                  dej                  fd„Zeee		 	 	 	 	 	 	 dd
ej                  dz  dej                  dz  dej&                  dz  dej                  dz  dedz  dej                  dz  dedz  de
e   deez  fd„«       «       «       Zˆ xZS )ÚChameleonModelr[   c           	      óÆ  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        |j                  «      | _        | j                  j                  st        nt        }t        j                   t#        |j$                  «      D �cg c]  } |||«      ‘Œ c}«      | _        t)        |j                  |j*                  ¬«      | _        t.        j1                  |j2                  «      | _        t7        |¬«      | _        d| _        | j=                  «        y c c}w )Nr²   ©r[   F)r7   r8   Úpad_token_idÚpadding_idxÚ
vocab_sizer   r  r>   Úembed_tokensr{  Úvocabulary_mapÚvocabulary_mappingr[   Ú	swin_normrù   r  ra  rc  Únum_hidden_layersÚlayersr3   rÿ   rI  rÃ  Ú_from_configÚ	vq_configÚvqmodelrY   Ú
rotary_embÚgradient_checkpointingrÊ  )r=   r[   Údecoder_layerrÓ   r?   s       €r1   r8   zChameleonModel.__init__M  s  ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ"AÀ&×BWÑBWÓ"XˆÔØ59·[±[×5JÒ5JÕ-ÔPiˆÜ—m‘mÜ?DÀV×E]ÑE]Ó?^Ö_°)‰]˜6 9Õ-Ò_ó
ˆŒô % V×%7Ñ%7¸V×=PÑ=PÔQˆŒ	Ü%×2Ñ2°6×3CÑ3CÓDˆŒÜ2¸&ÔAˆŒØ&+ˆÔ#ð 	�‰Õùò `s   ÃErw  c                 óÌ   — |j                   d   }| j                  j                  |d¬«      }| j                  j	                  |j
                  «      }|j                  |d«      }|S )as  
        Tokenizes images into discrete tokens with VQGAN module. Converts
        obtained image tokens into BPE tokens and wraps with "boi" and "eoi"
        special tokens.

        Args:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
                The tensors corresponding to the input images.
        r   T©Úreturn_dictrC   )rP   rß  rÏ  rÙ  r¬  r&   rì   )r=   rw  rN  Úvqmodel_outputsÚbpe_tokss        r1   Úget_image_tokenszChameleonModel.get_image_tokens`  s_   € ð "×'Ñ'¨Ñ*ˆ
Ø59·\±\×5HÑ5HÈÐcgÐ5HÓ5hˆØ×*Ñ*×:Ñ:¸?×;WÑ;WÓXˆØ—=‘= ¨RÓ0ˆØˆr0   zcTokenizes images into discrete tokens with VQGAN module and embeds them with text embeddings layer.rÀ  r°   r5   c                 ó  — |j                   d   } | j                  j                  |fddi|¤Ž}| j                  j	                  |j
                  «      j                  |d«      } | j                  «       |«      |_        |S )z®
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The tensors corresponding to the input images.
        r   rå  TrC   )	rP   rß  rÏ  rÙ  r¬  r&   rì   Úget_input_embeddingsÚpooler_output)r=   rw  r°   rN  ræ  Ú
bpe_tokenss         r1   Úget_image_featuresz!ChameleonModel.get_image_featuresp  s   € ð "×'Ñ'¨Ñ*ˆ
Ø5H°T·\±\×5HÑ5HÈÑ5rÐcgÐ5rÐkqÑ5rˆØ×,Ñ,×<Ñ<¸_×=YÑ=YÓZ×_Ñ_Ð`jÐlnÓoˆ
Ø(C¨×(AÑ(AÓ(CÀJÓ(OˆÔ%ØÐr0   Ú	input_idsÚinputs_embedsÚimage_featuresc                 ó.  — |€m| | j                  «       t        j                  | j                  j                  t        j
                  |j                  ¬«      «      k(  }|j                  d«      }n|| j                  j                  k(  }|j                  «       }|j                  d   |j                  d   z  }|j                  d«      j                  |j                  «      }t        ||j                  d   z  |j                  «       k(  d|› d|› �«       |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        )rE   ri   rC   r   r    z6Image features and image tokens do not match, tokens: z, features: )rê  r,   r   rÙ  r  Úlongri   Úallr  rP   r”   rF   r   Únumel)r=   rî  rï  rð  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r1   Úget_placeholder_maskz#ChameleonModel.get_placeholder_mask�  s  € ð ÐØ!.Ð2M°$×2KÑ2KÓ2MÜ—‘˜T×4Ñ4×CÑCÌ5Ï:É:Ð^k×^rÑ^rÔsó3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¨d×.EÑ.E×.TÑ.TÑ!TÐà+×/Ñ/Ó1ˆØ)×/Ñ/°Ñ2°^×5IÑ5IÈ!Ñ5LÑLÐØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐÜØ˜]×0Ñ0°Ñ4Ñ4¸×8LÑ8LÓ8NÑNØDÀ^ÐDTÐT`ÐaqÐ`rÐsô	
ð "Ð!r0   NrÁ   r†   rå   rç   c           
      óÞ  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|�C| j                  |d¬«      j                  }	| j	                  |||	¬«      }
|j                  |
|	«      }|r6|€4t        j                  j                  «       st        | j                  ¬«      }|€V|�|j                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t!        | j                  ||||¬	«      }|}| j#                  ||¬
«      }| j$                  D ]  } ||f|||||dœ|¤Ž}|d   }Œ | j'                  |«      }t)        ||¬«      S )Nz:You must specify exactly one of input_ids or inputs_embedsTrä  )rï  rð  rÓ  r   r    )ri   )r[   rï  rÁ   rå   r†   )r†   )rÁ   r†   rå   rç   rè   )ry  rå   )rÜ   r×  rí  rë  rø  Úmasked_scatterr,   ÚjitÚ
is_tracingr   r[   Úget_seq_lengthrr   rP   ri   r”   r   rà  rÜ  rI  r   )r=   rî  rw  rÁ   r†   rå   rï  rç   r°   rð  rõ  Úpast_seen_tokensr²  r@   rè   râ  Úlayer_outputss                    r1   rM   zChameleonModel.forward™  s¬  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMàÐ#Ø!×4Ñ4°\ÈtÐ4ÓT×bÑbˆNØ!%×!:Ñ!:Ø¨À~ð ";ó "Ðð *×8Ñ8Ð9KÈ^Ó\ˆMñ ˜Ð0¼¿¹×9MÑ9MÔ9OÜ*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐð "Ÿ[™[ò 	-ˆMÙ)Øðà*Ø)Ø /Ø#Ø$7ñð ñˆMð *¨!Ñ,‰Mð	-ð Ÿ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
r0   )NNNNNNN)r(   r)   r*   r!   r8   r,   r-   rè  r   r   r   r   rO   r   rí  rö   rø  r   r   rU   r
   r÷   r   r   rM   rV   rW   s   @r1   rÑ  rÑ  K  s…  ø„ ð˜õ ð&¨U×->Ñ->ó ð  ÙØzôðØ!×-Ñ-ðØ9?Ð@RÑ9Sðà	Ð+Ñ	+òóó ðð"Ø×)Ñ)ð"Ø:?×:KÑ:Kð"Ø]b×]nÑ]nó"ð0  ØØð .2Ø15Ø.2Ø04Ø(,Ø26Ø!%ñ@
à×#Ñ# dÑ*ð@
ð ×'Ñ'¨$Ñ.ð@
ð Ÿ™ tÑ+ð	@
ð
 ×&Ñ&¨Ñ-ð@
ð  ™ð@
ð ×(Ñ(¨4Ñ/ð@
ð ˜$‘;ð@
ð Ð-Ñ.ð@
ð 
Ð(Ñ	(ò@
ó ó ó  ô@
r0   rÑ  zb
    Chameleon Model with a head on top used for outputting logits for next token prediction.
    c                   ó®  ‡ — e Zd ZddiZˆ fd„Zd„ Zedej                  de	e
   deez  fd„«       Zee	 	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                   d	z  dej                  d	z  ded	z  dej                  d	z  dej                  d	z  ded	z  deej                   z  de	e
   deez  fd„«       «       Z	 	 	 	 	 	 	 dˆ fd„	Zˆ xZS )Ú!ChameleonForConditionalGenerationzlm_head.weightzmodel.embed_tokens.weightc                 óè   •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        | j                  «        y )NFrž   )
r7   r8   rÑ  r¯  rÖ  r   r¡   r>   Úlm_headrÊ  r¨   s     €r1   r8   z*ChameleonForConditionalGeneration.__init__ç  sU   ø€ Ü‰Ñ˜Ô Ü# FÓ+ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Õr0   c                 ó8   — | j                   j                  |«      S r‹   )r¯  rè  )r=   rw  s     r1   rè  z2ChameleonForConditionalGeneration.get_image_tokensð  s   € Ø�z‰z×*Ñ*¨<Ó8Ð8r0   rw  r°   r5   c                 ó<   —  | j                   j                  |fi |¤ŽS r‹   )r¯  rí  )r=   rw  r°   s      r1   rí  z4ChameleonForConditionalGeneration.get_image_featuresó  s    € ð -ˆt�z‰z×,Ñ,¨\ÑD¸VÑDÐDr0   Nrî  rÁ   r†   rå   rï  Úlabelsrç   Úlogits_to_keepc
                 ó  —  | j                   d|||||||dœ|
¤Ž}|d   }t        |	t        «      rt        |	 d«      n|	}| j	                  |dd…|dd…f   «      }| j                   j
                  j                  }t        j                  |j                  «      j                  |dd…dd…|f<   d}|�* | j                  d||| j                  j                  dœ|
¤Ž}t        |||j                  |j                   |j"                  ¬«      S )a†  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import ChameleonProcessor, ChameleonForConditionalGeneration
        >>> import torch
        >>> import httpx
        >>> from io import BytesIO
        >>> from PIL import Image

        >>> model = ChameleonForConditionalGeneration.from_pretrained("facebook/chameleon-7b", dtype=torch.bfloat16)
        >>> processor = ChameleonProcessor.from_pretrained("facebook/chameleon-7b")

        >>> prompt = "I used to know a lot about constellations when I was younger, but as I grew older, I forgot most of what I knew. These are the only two constellations that I really remember now.<image><image>I would like for you to tell me about 3 more constellations and give me a little bit of history about the constellation."
        >>> url = "https://nineplanets.org/wp-content/uploads/2020/12/the-big-dipper-1.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image1 = Image.open(BytesIO(response.read()))

        >>> url = "https://www.kxan.com/wp-content/uploads/sites/40/2020/10/ORION.jpg"
        >>> with httpx.stream("GET", url) as response:
        ...     image2 = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=[image1, image2], text=prompt, return_tensors="pt").to(model.device, torch.bfloat16)

        >>> generated_ids = model.generate(**inputs, max_new_tokens=100, do_sample=False)
        >>> processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        ```)rî  rw  rÁ   r†   rå   rï  rç   r   N)Úlogitsr  rÖ  )r"  r	  rå   r@   r´  r/   )r¯  r~   r�   Úslicer  rÙ  r&   r,   ÚfinforE   ÚminÚloss_functionr[   rÖ  r   rå   r@   r´  )r=   rî  rw  rÁ   r†   rå   rï  r  rç   r  r°   r  r@   Úslice_indicesr	  r&   r"  s                    r1   rM   z)ChameleonForConditionalGeneration.forwardù  s  € ð^ ,6¨4¯:©:ð 	,
ØØ%Ø)Ø%Ø+Ø'Øñ	,
ð ñ	,
ˆð   ™
ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆð —z‘z×4Ñ4×AÑAˆÜ%*§[¡[°·±Ó%>×%BÑ%BˆŠq’!�\Ð!Ñ"àˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r0   c	                 óJ   •— t        ‰| �  |f|||||||dœ|	¤Ž}
|s|rd |
d<   |
S )N)rw  rå   rÁ   rï  r†   rç   Úis_first_iterationrw  )r7   Úprepare_inputs_for_generation)r=   rî  rw  rå   rÁ   rï  r†   rç   r  r°   Úmodel_inputsr?   s              €r1   r  z?ChameleonForConditionalGeneration.prepare_inputs_for_generationH  sT   ø€ ô ‘wÑ<Øð

à%Ø+Ø)Ø'Ø%ØØ1ñ

ð ñ

ˆñ "¡ið
 ,0ˆL˜Ñ(àÐr0   )	NNNNNNNNr   )NNNNNTF)r(   r)   r*   Ú_tied_weights_keysr8   rè  r   r,   r-   r   r   rO   r   rí  r   rö   rU   r
   r÷   r�   r   rM   r  rV   rW   s   @r1   r  r  ß  s�  ø„ ð +Ð,GÐHÐôò9ð ðEØ!×-Ñ-ðEØ9?Ð@RÑ9SðEà	Ð+Ñ	+òEó ðEð
 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñK
à×#Ñ# dÑ*ðK
ð ×'Ñ'¨$Ñ.ðK
ð Ÿ™ tÑ+ð	K
ð
 ×&Ñ&¨Ñ-ðK
ð  ™ðK
ð ×(Ñ(¨4Ñ/ðK
ð × Ñ  4Ñ'ðK
ð ˜$‘;ðK
ð ˜eŸl™lÑ*ðK
ð Ð+Ñ,ðK
ð 
Ð'Ñ	'òK
ó ó ðK
ð` ØØØØØØ ÷!ñ !r0   r  )r  rÑ  r®  rÃ  rU  )rê   )Tr+   Úcollections.abcr   Údataclassesr   Ú	functoolsr   Útypingr   r,   Útorch.nn.functionalr   rÉ   r³   Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_chameleonr!   r"   Ú
get_loggerr(   rÖ   r$   rg  r3   rY   r’   rš   rœ   Ú	LayerNormr¬   rU   r�   r¼   rT   rÐ   rÒ   rù   r  r  r$  r3  rG  rS  r{  r®  rÃ  rÑ  r  Ú__all__r/   r0   r1   ú<module>r*     s£  ðñ å $Ý !Ý %Ý ã ß Ð Ý å !ß .Ý )Ý /Ý BÝ 9ß kÑ kß Kß FÝ &÷õ ÷ HÝ 5ß Jð 
ˆ×	Ñ	˜HÓ	%€ð Ø
ô4Ð :ó 4ó ó ð4ô J�r—y‘yô Jô*><˜rŸy™yô ><òD(óô4�2—9‘9ô ô"˜Ÿ™ô ð&	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð( ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ô2Z)˜Ÿ™ô Z)ô|CÐ6ô CôLAÐ :ô AôH,> B§I¡Iô ,>ô^	¨"¯)©)ô 	ô)( r§y¡yô )(ôX & R§Y¡Yô  &ôF^!˜BŸI™Iô ^!÷B,%ñ ,%ð^ ô˜ó ó ðñ& ðôô#
Ð-ó #
óð#
ðL ôP
Ð-ó P
ó ðP
ñf ðôô
EÐ(@À/ó Eóð
EòP p�r0   