Ë
    ÛÓ^jBþ  ã                   ó\  — d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlmZ ddl	m
Z ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZ ddlmZmZ ddlmZ ddlmZmZmZm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2m3Z3 ddl4m5Z5m6Z6 d„ Z7 G d„ dejp                  «      Z9 G d„ dejp                  «      Z:d„ Z; ed«      d_d„«       Z<	 	 d`dejp                  d ejz                  d!ejz                  d"ejz                  d#ejz                  dz  d$e>dz  d%e>d&e)e+   fd'„Z? ee<«       G d(„ d)ejp                  «      «       Z@ G d*„ d+ejp                  «      ZA G d,„ d-ejp                  «      ZBd.„ ZC G d/„ d0ejp                  «      ZD G d1„ d2ejp                  «      ZE G d3„ d4e«      ZF G d5„ d6ejp                  «      ZG G d7„ d8ejp                  «      ZHe, G d9„ d:e'«      «       ZI G d;„ d<eI«      ZJ G d=„ d>ejp                  «      ZK G d?„ d@ejp                  «      ZL G dA„ dBejp                  «      ZMe,e G dC„ dDe!«      «       «       ZN G dE„ dFejp                  «      ZO G dG„ dHejp                  «      ZP edI«       G dJ„ dKejp                  «      «       ZQ G dL„ dMejp                  «      ZR G dN„ dOejp                  «      ZSdPejz                  dQeTdRejz                  fdS„ZU ee<«       G dT„ dUejp                  «      «       ZV G dV„ dWe«      ZWe, G dX„ dYe'«      «       ZX G dZ„ d[eX«      ZY G d\„ d]eXe«      ZZg d^¢Z[y)aé    N)ÚCallable)Ú	dataclass)ÚOptional)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚBaseModelOutputWithPastÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚEvollaConfigÚSaProtConfigc                 ó¸   — | j                  |«      j                  «       }t        j                  |d¬«      j	                  |«      |z  }|j                  «       |z   S )a  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        x: torch.Tensor x:

    Returns: torch.Tensor
    r$   ©Údim)ÚneÚintÚtorchÚcumsumÚtype_asÚlong)Ú	input_idsÚpadding_idxÚmaskÚincremental_indicess       úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/evolla/modeling_evolla.pyÚ"create_position_ids_from_input_idsr5   4   sP   € ð �<‰<˜Ó$×(Ñ(Ó*€DÜŸ,™, t°Ô3×;Ñ;¸DÓAÀDÑHÐØ×#Ñ#Ó%¨Ñ3Ð3ó    c                   ó8   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 dd„Zd„ Zˆ xZS )ÚEvollaSaProtEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        |j                  r1t        j                  |j
                  |j                  ¬«      | _        nd | _        t        j                  |j                  «      | _        t        |dd«      | _        | j#                  dt%        j&                  |j(                  «      j+                  d«      d¬«       |j                  | _        | j                   dk(  r;t        j                  |j(                  |j
                  | j,                  ¬«      | _        |j0                  | _        |j2                  | _        d | _        y )	N)r1   ©ÚepsÚposition_embedding_typeÚabsoluteÚposition_ids)r$   éÿÿÿÿF©Ú
persistent)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚemb_layer_norm_beforeÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropoutÚgetattrr<   Úregister_bufferr,   ÚarangeÚmax_position_embeddingsÚexpandr1   Úposition_embeddingsÚtoken_dropoutÚmask_token_idr>   ©ÚselfÚconfigÚ	__class__s     €r4   rC   zEvollaSaProtEmbeddings.__init__I   s2  ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔà×'Ò'Ü Ÿl™l¨6×+=Ñ+=À6×CXÑCXÔYˆD�Oà"ˆDŒOÜ—z‘z &×"<Ñ"<Ó=ˆŒä'.¨vÐ7PÐR\Ó']ˆÔ$Ø×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	ô 	
ð "×.Ñ.ˆÔØ×'Ñ'¨:Ò5Ü')§|¡|Ø×.Ñ.°×0BÑ0BÐPT×P`ÑP`ô(ˆDÔ$ð $×1Ñ1ˆÔØ#×1Ñ1ˆÔà ˆÕr6   c                 óæ  — |€*|�t        || j                  «      }n| j                  |«      }|€| j                  |«      }|}| j                  r²|�°|j                  || j                  k(  j                  d«      d«      }d}|�|j                  d«      n|j                  d   }|| j                  k(  j                  d«      j                  «       |z  }|d|z
  z  d|z
  d d …d d f   z  j                  |j                  «      }| j                  dk(  r| j                  |«      }	||	z   }| j                  �| j                  |«      }|�-||j                  d«      z  j                  |j                  «      }|S )Nr?   ç        g¸…ëQ¸¾?r$   r=   )r5   r1   Ú&create_position_ids_from_inputs_embedsrH   rV   Úmasked_fillrW   Ú	unsqueezeÚsumÚshapeÚfloatÚtoÚdtyper<   rU   rL   )
rY   r0   Úattention_maskr>   Úinputs_embedsÚ
embeddingsÚmask_ratio_trainÚsrc_lengthsÚmask_ratio_observedrU   s
             r4   ÚforwardzEvollaSaProtEmbeddings.forwardb   s”  € ð ÐØÐ$äAÀ)ÈT×M]ÑM]Ó^‘à#×JÑJÈ=ÓY�àÐ Ø ×0Ñ0°Ó;ˆMð #ˆ
ð ×Ò )Ð"7Ø#×/Ñ/°¸d×>PÑ>PÑ1P×0[Ñ0[Ð\^Ó0_ÐadÓeˆJØ)ÐØ4BÐ4N˜.×,Ñ,¨RÔ0ÐT]×TcÑTcÐdeÑTfˆKØ#,°×0BÑ0BÑ#B×"GÑ"GÈÓ"K×"QÑ"QÓ"SÐVaÑ"aÐØ$¨Ð,<Ñ(<Ñ=ÀÐEXÑAXÒZ[Ð]aÐcgÐZgÑ@hÑh×lÑlØ× Ñ óˆJð ×'Ñ'¨:Ò5Ø"&×":Ñ":¸<Ó"HÐØ#Ð&9Ñ9ˆJà�?‰?Ð&ØŸ™¨Ó4ˆJØÐ%Ø$ ~×'?Ñ'?ÀÓ'CÑC×GÑGÈ
×HXÑHXÓYˆJð Ðr6   c                 ó  — |j                  «       dd }|d   }t        j                  | j                  dz   || j                  z   dz   t        j                  |j
                  ¬«      }|j                  d«      j                  |«      S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr?   r$   )re   Údevicer   )Úsizer,   rR   r1   r/   rn   r`   rT   )rY   rg   Úinput_shapeÚsequence_lengthr>   s        r4   r^   z=EvollaSaProtEmbeddings.create_position_ids_from_inputs_embeds“   s€   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a™.ˆä—|‘|Ø×Ñ˜qÑ  /°D×4DÑ4DÑ"DÀqÑ"HÔPU×PZÑPZÐcp×cwÑcwô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<r6   ©NNNN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rC   rl   r^   Ú__classcell__©r[   s   @r4   r8   r8   D   s&   ø„ ñô!ð6 ØØØó/öb=r6   r8   c                   óÄ   ‡ — e Zd ZU dZej
                  ed<   ddefˆ fd„Ze		 	 	 ddedz  ddde
dz  d	ed
ef   fd„«       Z ej                  «       edd„«       «       Zˆ xZS )ÚEvollaSaProtRotaryEmbeddingzú
    Rotary position embeddings.
    Implementation based on [ModernBERT's RotaryEmbedding](https://github.com/huggingface/transformers/blob/aad13b87ed59f2afcfaebc985f403301887a35fc/src/transformers/models/modernbert/modeling_modernbert.py#L94).
    Úinv_freqNrZ   c                 óº   •— t         ‰| �  «        || _        i | _        | j	                  | j                  |«      \  }}| j                  d|«       t        | d|«       y )Nr{   Úattention_scaling)rB   rC   rZ   Ú	rope_typeÚcompute_default_rope_parametersrQ   Úsetattr)rY   rZ   rn   Úcurr_inv_freqÚcurr_attention_scalingr[   s        €r4   rC   z$EvollaSaProtRotaryEmbedding.__init__­   sZ   ø€ Ü‰ÑÔàˆŒØˆŒà04×0TÑ0TÐUY×U`ÑU`ÐbhÓ0iÑ-ˆÐ-Ø×Ñ˜Z¨Ô7Ü�Ð)Ð+AÕBr6   rn   ztorch.device | NoneÚseq_lenÚreturnútorch.Tensorc                 ó  — | j                   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )a©  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Úhead_dimNç      ð?r   é   ©re   ©rn   re   )	Ú
rope_thetarP   rF   Únum_attention_headsr,   rR   Úint64rd   rc   ©rZ   rn   rƒ   Úbaser)   Úattention_factorr{   s          r4   r   z;EvollaSaProtRotaryEmbedding.compute_default_rope_parameters·   s‹   € ð( × Ñ ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r6   c                 óB  — t        | d«      }t        | d«      }|d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬	«      5  |j                  «       |j                  «       z  j                  dd
«      }	t        j                  |	|	fd¬«      }
|
j                  «       |z  }|
j                  «       |z  }d d d «       j	                  |j                  ¬«      j	                  |j                  ¬«      fS # 1 sw Y   ŒAxY w)Nr{   r}   r   r?   r$   ÚmpsÚcpuF©Údevice_typeÚenabledr‰   r(   rŠ   )rP   rc   rT   rb   rd   rn   Ú
isinstanceÚtypeÚstrr    Ú	transposer,   ÚcatÚcosÚsinre   )rY   Úxr>   Ú
layer_typer{   r}   Úinv_freq_expandedÚposition_ids_expandedr–   ÚfreqsÚembr�   rž   s                r4   rl   z#EvollaSaProtRotaryEmbedding.forwardÖ   s^  € ô ˜4 Ó,ˆÜ# DÐ*=Ó>Ðà$ Tª1¨d ]Ñ3×9Ñ9Ó;×BÑBÀ<×CUÑCUÐVWÑCXÐZ\Ð^_Ó`×cÑcÐde×dlÑdlÓmÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	0Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷		0ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	0ð 	0ús   Ã$A1FÆF©N©NNN)rs   rt   ru   rv   r,   ÚTensorÚ__annotations__r&   rC   Ústaticmethodr+   Útuplerc   r   Úno_gradr   rl   rw   rx   s   @r4   rz   rz   ¥   sš   ø… ñð
 �l‰lÓñC˜|õ Cð à&*Ø(,Ø"ñ*Ø˜tÑ#ð*à%ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð< €U‡]�]ƒ_Øò<ó ó ô<r6   rz   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr?   r‰   r(   )rb   r,   rœ   )rŸ   Úx1Úx2s      r4   Úrotate_halfr¯   é   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r6   Úrotary_pos_embc                 ób  — | j                   }|j                  |«      }|j                  |«      }| j                  «       |z  t        | j                  «       «      |z  z   }|j                  «       |z  t        |j                  «       «      |z  z   }|j	                  |«      |j	                  |«      fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )re   r`   rc   r¯   rd   )ÚqÚkr�   rž   Úunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds           r4   Úapply_rotary_pos_embr¸   ð   sŽ   € ð& —W‘W€NØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�w‰w‹y˜3‰¤;¨q¯w©w«yÓ#9¸CÑ#?Ñ@€GØ�w‰w‹y˜3‰¤;¨q¯w©w«yÓ#9¸CÑ#?Ñ@€GØ�:‰:�nÓ% w§z¡z°.Ó'AÐAÐAr6   ÚmoduleÚqueryÚkeyÚvaluerf   ÚscalingrO   Úkwargsc                 óæ  — |€|j                  d«      dz  }t        | dd«      }t        ||«      }t        ||«      }t        j                  ||j                  dd«      «      |z  }	|�|	|z   }	t        j                  j                  |	d¬«      }	t        j                  j                  |	|| j                  ¬«      }	t        j                  |	|«      }
|
j                  dd«      j                  «       }
|
|	fS )	Nr?   ç      à¿Únum_key_value_groupsr$   r‰   r   r(   )ÚpÚtraining)ro   rP   Ú	repeat_kvr,   Úmatmulr›   r   Ú
functionalÚsoftmaxrO   rÃ   Ú
contiguous)r¹   rº   r»   r¼   rf   r½   rO   r¾   Ún_repÚattn_weightsÚattn_outputs              r4   Úeager_attention_forwardrÌ     sä   € ð €Ø—*‘*˜R“. DÑ(ˆô
 �FÐ2°AÓ6€EÜ
�C˜Ó
€CÜ�e˜UÓ#€Eô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r6   c                   óê   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dej
                  dz  dee   d	e	ej
                     fd
„Z
ˆ xZS )ÚEvollaSaProtSelfAttentionNc                 óJ  •— t         ‰| �  «        || _        |j                  |j                  z  dk7  r2t        |d«      s&t        d|j                  › d|j                  › d�«      ‚|j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _	        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        |j                  | _        |xs t#        |dd«      | _        d| _        |j(                  | _        || _        | j(                  xr | | _        y )	Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)r<   r=   rˆ   )rB   rC   rZ   rF   r�   ÚhasattrÚ
ValueErrorr+   Úattention_head_sizeÚall_head_sizer   ÚLinearrº   r»   r¼   Úattention_probs_dropout_probrO   rP   r<   r½   Ú
is_decoderÚ	layer_idxÚ	is_causal)rY   rZ   r<   rÙ   Úis_cross_attentionr[   s        €r4   rC   z"EvollaSaProtSelfAttention.__init__0  sh  ø€ Ü‰ÑÔØˆŒà×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
à×:Ñ:ˆŒà'>ò (
Ä'ØÐ-¨zóC
ˆÔ$ð ˆŒØ ×+Ñ+ˆŒØ"ˆŒØŸ™ÒCÐ1CÐ-Cˆ�r6   Úhidden_statesrf   Úencoder_hidden_statesÚencoder_attention_maskrU   r¾   r„   c                 ó
  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	|d u}
|
r|n|}|
r|n|}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }|	| j                  dz  z  }	| j                  dk(  r|\  }}t        |	|||d¬«      \  }	}t        j                  | j                  j                  t        «      } || |	|||f| j                  sdn| j                  | j                   dœ|¤Ž\  }} |j"                  g |¢d‘­Ž j%                  «       }||fS )	Nr?   r$   r‰   rÀ   Úrotary)r´   r]   ©rO   r½   )rb   rÔ   rº   Úviewr›   r»   r¼   r<   r¸   r   Úget_interfacerZ   Ú_attn_implementationrÌ   rÃ   rO   r½   ÚreshaperÈ   )rY   rÜ   rf   rÝ   rÞ   rU   r¾   rp   Úhidden_shapeÚquery_layerrÛ   Úcurrent_statesÚ	key_layerÚvalue_layerr�   rž   Úattention_interfacerË   rÊ   s                      r4   rl   z!EvollaSaProtSelfAttention.forwardL  s£  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆà—j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆà2¸$Ð>ÐÙ2DÑ.È-ˆÙ3EÑ/È>ˆØ—H‘H˜^Ó,×1Ñ1°,Ó?×IÑIÈ!ÈQÓOˆ	Ø—j‘j Ó0×5Ñ5°lÓC×MÑMÈaÐQRÓSˆð " D×$<Ñ$<¸dÑ$BÑBˆà×'Ñ'¨8Ò3Ø*‰HˆC�Ü%9¸+ÀyÐRUÐWZÐjkÔ%lÑ"ˆK˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$·,±,Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ˜LÐ(Ð(r6   )NNFrr   )rs   rt   ru   rC   r,   r§   ÚFloatTensorr   r   rª   rl   rw   rx   s   @r4   rÎ   rÎ   .  s�   ø„ õDð> 48Ø:>Ø;?Ø37ñ.)à—|‘|ð.)ð ×)Ñ)¨DÑ0ð.)ð  %×0Ñ0°4Ñ7ð	.)ð
 !&× 1Ñ 1°DÑ 8ð.)ð #Ÿ\™\¨DÑ0ð.)ð Ð+Ñ,ð.)ð 
ˆu�|‰|Ñ	÷.)r6   rÎ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEvollaSaProtSelfOutputc                 óÈ   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  «      | _        y r¥   )	rB   rC   r   rÖ   rF   ÚdenserM   rN   rO   rX   s     €r4   rC   zEvollaSaProtSelfOutput.__init__~  sB   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆ�r6   c                 óT   — | j                  |«      }| j                  |«      }||z   }|S r¥   ©rð   rO   ©rY   rÜ   Úinput_tensors      r4   rl   zEvollaSaProtSelfOutput.forwardƒ  ó.   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØ%¨Ñ4ˆØÐr6   ©rs   rt   ru   rC   rl   rw   rx   s   @r4   rî   rî   }  ó   ø„ ô>ö
r6   rî   c                   óZ   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej
                  dz  dee   fd„Zˆ xZ	S )ÚEvollaSaProtAttentionNc                 óÈ   •— t         ‰| �  «        t        |||¬«      | _        t	        |«      | _        t        j                  |j                  |j                  ¬«      | _        y )N)rÙ   rÛ   r:   )
rB   rC   rÎ   rY   rî   Úoutputr   rJ   rF   rK   )rY   rZ   rÙ   rÛ   r[   s       €r4   rC   zEvollaSaProtAttention.__init__‹  sI   ø€ Ü‰ÑÔÜ-¨fÀ	Ð^pÔqˆŒ	Ü,¨VÓ4ˆŒäŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆ�r6   rU   r¾   c                 ó‚   — | j                  |«      } | j                  |f||||dœ|¤Ž\  }}	| j                  ||«      }|S )N©rf   rÝ   rÞ   rU   )rJ   rY   rû   )
rY   rÜ   rf   rÝ   rÞ   rU   r¾   Úhidden_states_lnrË   Ú_s
             r4   rl   zEvollaSaProtAttention.forward’  s]   € ð  Ÿ>™>¨-Ó8ÐØ"˜Ÿ™Øð
à)Ø"7Ø#9Ø 3ñ
ð ñ
‰ˆ�Qð —k‘k +¨}Ó=ˆØÐr6   )NFrr   )
rs   rt   ru   rC   r,   r§   r   r   rl   rw   rx   s   @r4   rù   rù   Š  s?   ø„ õUð Ø"Ø#Ø37ñð #Ÿ\™\¨DÑ0ðð Ð+Ñ,÷r6   rù   c                 ój   — | dz  dt        j                  | t        j                  d«      z  «      z   z  S )zz
    This is the gelu implementation from the original EVOLLA_SA_PROT repo. Using F.gelu yields subtly wrong results.
    g      à?rˆ   g       @)r,   ÚerfÚmathÚsqrt)rŸ   s    r4   Úgelur  ¨  s.   € ð ˆs‰7�cœEŸI™I a¬$¯)©)°C«.Ñ&8Ó9Ñ9Ñ:Ð:r6   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚEvollaSaProtIntermediatec                 ó€   •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        y r¥   )rB   rC   r   rÖ   rF   Úintermediate_sizerð   rX   s     €r4   rC   z!EvollaSaProtIntermediate.__init__°  s,   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆ�
r6   rÜ   r„   c                 ó>   — | j                  |«      }t        |«      }|S r¥   )rð   r  )rY   rÜ   s     r4   rl   z EvollaSaProtIntermediate.forward´  s    € ØŸ
™
 =Ó1ˆÜ˜]Ó+ˆØÐr6   ©rs   rt   ru   rC   r,   r§   rl   rw   rx   s   @r4   r  r  ¯  s$   ø„ ôMð U§\¡\ð °e·l±l÷ r6   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEvollaSaProtOutputc                 óÈ   •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j                  «      | _	        y r¥   )
rB   rC   r   rÖ   r  rF   rð   rM   rN   rO   rX   s     €r4   rC   zEvollaSaProtOutput.__init__»  sB   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆ�r6   c                 óT   — | j                  |«      }| j                  |«      }||z   }|S r¥   rò   ró   s      r4   rl   zEvollaSaProtOutput.forwardÀ  rõ   r6   rö   rx   s   @r4   r  r  º  r÷   r6   r  c                   ó^   ‡ — e Zd Zˆ fd„Z	 	 	 	 ddej
                  dz  dee   fd„Zd„ Z	ˆ xZ
S )ÚEvollaSaProtLayerc                 óÆ  •— t         ‰| �  «        |j                  | _        d| _        t	        |«      | _        |j                  | _        |j                  | _        | j                  r,| j                  st        | › d�«      ‚t	        |d¬«      | _	        t        |«      | _        t        |«      | _        t        j                  |j                   |j"                  ¬«      | _        y )Nr$   z> should be used as a decoder model if cross attention is addedT)rÛ   r:   )rB   rC   Úchunk_size_feed_forwardÚseq_len_dimrù   Ú	attentionrØ   Úadd_cross_attentionÚRuntimeErrorÚcrossattentionr  Úintermediater  rû   r   rJ   rF   rK   rX   s     €r4   rC   zEvollaSaProtLayer.__init__È  sµ   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ.¨vÓ6ˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#Ò#Ø—?’?Ü" d VÐ+iÐ#jÓkÐkÜ"7¸ÐSWÔ"XˆDÔÜ4°VÓ<ˆÔÜ(¨Ó0ˆŒÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆ�r6   NrU   r¾   c                 óÖ   —  | j                   |f||dœ|¤Ž}| j                  r5|�3t        | d«      st        d| › d�«      ‚ | j                  |f||||dœ|¤Ž}| j                  |«      }|S )N©rf   rU   r  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`rý   )r  rØ   rÒ   ÚAttributeErrorr  Úfeed_forward_chunk)	rY   rÜ   rf   rÝ   rÞ   rU   r¾   Úattention_outputÚlayer_outputs	            r4   rl   zEvollaSaProtLayer.forward×  s´   € ð *˜4Ÿ>™>Øð
à)Ø 3ñ
ð ñ	
Ðð �?Š?Ð4Ð@Ü˜4Ð!1Ô2Ü$Ø=¸d¸Vð D`ð `óð ð
  3˜t×2Ñ2Ø ð à-Ø&;Ø'=Ø$7ñ ð ñ Ðð ×.Ñ.Ð/?Ó@ˆØÐr6   c                 ón   — | j                  |«      }| j                  |«      }| j                  ||«      }|S r¥   )rJ   r  rû   )rY   r  Úattention_output_lnÚintermediate_outputr  s        r4   r  z$EvollaSaProtLayer.feed_forward_chunkú  s<   € Ø"Ÿn™nÐ-=Ó>ÐØ"×/Ñ/Ð0CÓDÐØ—{‘{Ð#6Ð8HÓIˆØÐr6   rr   )rs   rt   ru   rC   r,   r§   r   r   rl   r  rw   rx   s   @r4   r  r  Ç  sE   ø„ ôUð$ Ø"Ø#Ø37ñ!ð #Ÿ\™\¨DÑ0ð!ð Ð+Ñ,ó!öFr6   r  c                   ób   ‡ — e Zd Zˆ fd„Ze	 	 	 	 ddej                  dz  dee   fd„«       Z	ˆ xZ
S )ÚEvollaSaProtEncoderc                 ó0  •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        t        j                  |j                  |j                  ¬«      | _        d| _        y c c}w )Nr:   F)rB   rC   rZ   r   Ú
ModuleListÚrangeÚnum_hidden_layersr  ÚlayerrJ   rF   rK   Úemb_layer_norm_afterÚgradient_checkpointing)rY   rZ   rÿ   r[   s      €r4   rC   zEvollaSaProtEncoder.__init__  sn   ø€ Ü‰ÑÔØˆŒÜ—]‘]ÄuÈV×MeÑMeÓGfÖ#gÀ!Ô$5°fÕ$=Ò#gÓhˆŒ
Ü$&§L¡L°×1CÑ1CÈ×I^ÑI^Ô$_ˆÔ!Ø&+ˆÕ#ùò $hs   ½BNrU   r¾   c           	      óª   — t        | j                  «      D ]  \  }} ||f||||dœ|¤Ž}Œ | j                  r| j                  |«      }t        |¬«      S )Nrý   )Úlast_hidden_state)Ú	enumerater(  r)  r   )	rY   rÜ   rf   rÝ   rÞ   rU   r¾   ÚiÚlayer_modules	            r4   rl   zEvollaSaProtEncoder.forward	  sn   € ô  )¨¯©Ó4ò 	‰OˆAˆ|Ù(Øðà-Ø&;Ø'=Ø$7ñð ñ‰Mð	ð ×$Ò$Ø ×5Ñ5°mÓDˆMä1ÀMÔRÐRr6   rr   )rs   rt   ru   rC   r   r,   r§   r   r   rl   rw   rx   s   @r4   r#  r#    sQ   ø„ ô,ð ð Ø"Ø#Ø37ñSð #Ÿ\™\¨DÑ0ðSð Ð+Ñ,òSó ôSr6   r#  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚEvollaSaProtPoolerc                 ó²   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  «       | _        y r¥   )rB   rC   r   rÖ   rF   rð   ÚTanhÚ
activationrX   s     €r4   rC   zEvollaSaProtPooler.__init__$  s9   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�r6   rÜ   r„   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S )Nr   )rð   r4  )rY   rÜ   Úfirst_token_tensorÚpooled_outputs       r4   rl   zEvollaSaProtPooler.forward)  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr6   r
  rx   s   @r4   r1  r1  #  s#   ø„ ô$ð
 U§\¡\ð °e·l±l÷ r6   r1  c                   óœ   ‡ — e Zd ZU eed<   dgZdZdZdZdZ	e
 eedd¬«      g eedd¬«      gdœZ ej                  «       ˆ fd	„«       Zˆ xZS )
ÚEvollaSaProtPreTrainedModelrZ   r  Tr$   r  )ÚindexÚ
layer_namer  )rÜ   Ú
attentionsÚcross_attentionsc                 óÂ   •— t         ‰| �  |«       t        |t        «      r?|j	                  |j
                  «      \  }}t        j                  t        |d«      |«       y y )Nr{   )	rB   Ú_init_weightsr˜   rz   r   rZ   ÚinitÚcopy_rP   )rY   r¹   r�   rÿ   r[   s       €r4   r?  z)EvollaSaProtPreTrainedModel._init_weightsC  sN   ø€ ä‰Ñ˜fÔ%Ü�fÔ9Ô:Ø%×EÑEÀfÇmÁmÓTÑˆM˜1Ü�J‰J”w˜v zÓ2°MÕBð ;r6   )rs   rt   ru   r&   r¨   Ú_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr  r"   rÎ   Ú_can_record_outputsr,   r«   r?  rw   rx   s   @r4   r9  r9  2  sx   ø… àÓØ,Ð-ÐØÐØ€NØÐØ"&Ðð +Ù%Ð&?ÀqÐU`ÔaÐbáÐ4¸AÐJZÔ[ð
ñÐð €U‡]�]ƒ_óCó ôCr6   r9  c            
       ó°   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 d
de	j                  dz  de	j                  dz  dee	j                     ez  fd	„«       «       Zˆ xZS )ÚEvollaSaProtProteinEncoderrZ   c                 ó¦   •— t         ‰| �  |«       t        |«      | _        t	        |¬«      | _        t        |«      | _        | j                  «        y ©N©rZ   )	rB   rC   r8   rh   rz   Úrotary_embeddingsr#  ÚencoderÚ	post_initrX   s     €r4   rC   z#EvollaSaProtProteinEncoder.__init__L  s?   ø€ Ü‰Ñ˜Ô Ü0°Ó8ˆŒÜ!<ÀFÔ!KˆÔÜ*¨6Ó2ˆŒØ�‰Õr6   c                 ó.   — | j                   j                  S r¥   ©rh   rH   ©rY   s    r4   Úget_input_embeddingsz/EvollaSaProtProteinEncoder.get_input_embeddingsS  s   € Ø�‰×.Ñ.Ð.r6   c                 ó&   — || j                   _        y r¥   rQ  ©rY   r¼   s     r4   Úset_input_embeddingsz/EvollaSaProtProteinEncoder.set_input_embeddingsV  s   € Ø*/ˆ�‰Õ'r6   Nr0   rf   r„   c                 óÐ  — |j                  «       }|\  }}|j                  }|€t        j                  ||f|¬«      }| j	                  ||¬«      }t        | j                  ||¬«      }t        j                  ||¬«      j                  d«      }	| j                  ||	«      }
 | j                  |f||
dœ|¤Ž}|d   }t        ||j                  |j                  |j                  ¬«      S )N©rn   ©r0   rf   )rZ   rg   rf   r   r  )r,  rÜ   r<  r=  )ro   rn   r,   Úonesrh   r   rZ   rR   r`   rM  rN  r   rÜ   r<  r=  )rY   r0   rf   r¾   rp   Ú
batch_sizeÚ
seq_lengthrn   rg   r>   rU   Úencoder_outputsÚsequence_outputs                r4   rl   z"EvollaSaProtProteinEncoder.forwardY  sö   € ð  —n‘nÓ&ˆØ!,Ñˆ
�Jà×!Ñ!ˆØÐ!Ü"ŸZ™Z¨*°jÐ)AÈ6ÔRˆNØŸ™°)ÈN˜Ó[ˆä2Ø—;‘;Ø'Ø)ô
ˆô —|‘| J°vÔ>×HÑHÈÓKˆØ"×4Ñ4°]ÀLÓQÐà&˜$Ÿ,™,Øð
Ø*8ÐNañ
Øekñ
ˆð *¨!Ñ,ˆä;Ø-Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ô	
ð 	
r6   r¥   )rs   rt   ru   r&   rC   rS  rV  r!   r#   r,   r§   rª   r   rl   rw   rx   s   @r4   rI  rI  K  sw   ø„ ð˜|õ ò/ò0ð  Øð /3ñ!
à—<‘< $Ñ&ð!
ð Ÿ™ tÑ+ð!
ð
 
ˆu�|‰|Ñ	ÐKÑ	Kò!
ó ó  ô!
r6   rI  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )Ú!EvollaSequenceCompressorAttentionc                 ój  •— t         ‰| �  «        |dz  | _        || _        ||z  }t	        j
                  |«      | _        t	        j
                  |«      | _        t	        j                  ||d¬«      | _	        t	        j                  ||dz  d¬«      | _
        t	        j                  ||d¬«      | _        y )NrÀ   F©Úbiasr‰   )rB   rC   ÚscaleÚheadsr   rJ   Ú
norm_mediaÚnorm_latentsrÖ   Úto_qÚto_kvÚto_out)rY   r)   Údim_headre  Ú	inner_dimr[   s        €r4   rC   z*EvollaSequenceCompressorAttention.__init__€  s‹   ø€ Ü‰ÑÔØ˜t‘^ˆŒ
ØˆŒ
Ø˜uÑ$ˆ	äŸ,™, sÓ+ˆŒÜŸL™L¨Ó-ˆÔä—I‘I˜c 9°5Ô9ˆŒ	Ü—Y‘Y˜s I°¡M¸Ô>ˆŒ
Ü—i‘i 	¨3°UÔ;ˆ�r6   c                 óF  — | j                  |«      }| j                  |«      }| j                  }| j                  |«      }t	        j
                  ||fd¬«      }| j                  |«      j                  dd¬«      \  }}|j                  |j                  d«      |j                  d«      |d«      j                  dddd«      }|j                  |j                  d«      |j                  d«      |d«      j                  dddd«      }|j                  |j                  d«      |j                  d«      |d«      j                  dddd«      }|| j                  z  }t	        j                  ||j                  dd«      «      }	|	|	j                  dd¬	«      j                  «       z
  }	|	j                   \  }
}}}t	        j"                  ||«      j%                  |j&                  «      }|d
d
…d
d
d
d
…f   }|d
d
d
…d
d
…d
f   }||z  }|	j)                  d|z
  j+                  «       d«      }	|	j-                  d¬«      }t	        j                  ||«      }|j                  dddd«      }|j/                  |j                  d«      |j                  d«      d«      }| j1                  |«      S )zÔ
        Args:
            x (torch.Tensor): image features
                shape (b, n1, D)
            latent (torch.Tensor): latent features
                shape (b, n2, D);  n2: num of latent tokens
        éþÿÿÿr(   r‰   r?   r   r$   r   T©r)   ÚkeepdimNg     ˆÃÀ)rf  rg  re  rh  r,   rœ   ri  Úchunkrâ   ro   Úpermuterd  rÅ   r›   ÚamaxÚdetachrb   rZ  rd   rn   r_   ÚboolrÇ   rå   rj  )rY   rŸ   Úlatentsr2   Úhr²   Úkv_inputr³   ÚvÚsimÚbsÚnhÚskdÚokdrZ  Úmask_expÚones_expÚattnÚouts                      r4   rl   z)EvollaSequenceCompressorAttention.forward�  sB  € ð �O‰O˜AÓˆØ×#Ñ# GÓ,ˆà�J‰Jˆà�I‰I�gÓˆÜ—9‘9˜a ˜\¨rÔ2ˆØ�z‰z˜(Ó#×)Ñ)Ø�2ð *ó 
‰ˆˆ1ð �F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ�F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ�F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ�—
‘
‰Nˆô �l‰l˜1˜aŸk™k¨"¨bÓ1Ó2ˆØ�C—H‘H ¨T�HÓ2×9Ñ9Ó;Ñ;ˆØŸ9™9ÑˆˆB��SÜ�z‰z˜"˜cÓ"×%Ñ% d§k¡kÓ2ˆØš˜4 ¢qÐ(Ñ)ˆØ˜ša¢ DÐ(Ñ)ˆØ˜(Ñ"ˆà�o‰o˜q 4™xŸo™oÓ/°Ó6ˆØ�{‰{˜rˆ{Ó"ˆÜ�l‰l˜4 Ó#ˆØ�k‰k˜!˜Q  1Ó%ˆð �k‰k˜#Ÿ(™( 1›+ s§x¡x°£{°BÓ7ˆà�{‰{˜3ÓÐr6   )é@   é   rö   rx   s   @r4   r`  r`    s   ø„ õ<ö) r6   r`  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚEvollaFeedForwardc                 ó  •— t         ‰| �  «        t        ||z  «      }t        j                  |«      | _        t        j                  ||d¬«      | _        t        j                  «       | _	        t        j                  ||d¬«      | _
        y ©NFrb  )rB   rC   r+   r   rJ   ÚnormrÖ   Úfc1ÚGELUr4  Úfc2)rY   r)   Úmultrl  r[   s       €r4   rC   zEvollaFeedForward.__init__º  s`   ø€ Ü‰ÑÔÜ˜˜d™
“Oˆ	ä—L‘L Ó%ˆŒ	Ü—9‘9˜S )°%Ô8ˆŒÜŸ'™'›)ˆŒÜ—9‘9˜Y¨°%Ô8ˆ�r6   c           	      ó~   — | j                  | j                  | j                  | j                  |«      «      «      «      S r¥   )rŒ  r4  rŠ  r‰  )rY   rŸ   s     r4   rl   zEvollaFeedForward.forwardÃ  s+   € Ø�x‰x˜Ÿ™¨¯©°·±¸1³Ó(>Ó?Ó@Ð@r6   )é   rö   rx   s   @r4   r†  r†  ¹  s   ø„ õ9öAr6   r†  c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )Ú!EvollaSequenceCompressorResamplerrZ   c           
      ó¨  •— t         ‰| �  «        |j                  j                  }|j                  | _        t        j                  t        j                  | j
                  |«      d¬«      | _
        t        j                  g «      | _        t        |j                  «      D ]g  }| j                  j                  t        j                  t!        ||j"                  |j$                  ¬«      t'        ||j(                  ¬«      g«      «       Œi t        j*                  |j                  «      | _        t        j.                  ||j                  «      | _        y )NT)Úrequires_grad)r)   rk  re  )r)   r�  )rB   rC   Úprotein_encoder_configrF   Úresampler_num_latentsÚnum_latentsr   Ú	Parameterr,   Úrandnrv  r%  Úlayersr&  Úresampler_depthÚappendr`  Úresampler_dim_headÚresampler_headsr†  Úresampler_ff_multrJ   r‰  rÖ   Úprotein_projector)rY   rZ   Úprotein_repr_dimrÿ   r[   s       €r4   rC   z*EvollaSequenceCompressorResampler.__init__È  s  ø€ Ü‰ÑÔØ!×8Ñ8×DÑDÐØ!×7Ñ7ˆÔÜ—|‘|¤E§K¡K°×0@Ñ0@ÐBRÓ$SÐcgÔhˆŒÜ—m‘m BÓ'ˆŒÜ�v×-Ñ-Ó.ò 
	ˆAØ�K‰K×ÑÜ—‘ä9Ø 0¸6×;TÑ;TÐ\b×\rÑ\rôô *Ð.>ÀV×E]ÑE]Ô^ð	óõ	ð
	ô —L‘L ×!3Ñ!3Ó4ˆŒ	Ü!#§¡Ð+;¸V×=OÑ=OÓ!PˆÕr6   c                 ój  — |j                   d   }|j                   \  }}t        j                  || j                  «      j	                  |j
                  «      }t        j                  ||fd¬«      }t        j                  |«      j	                  | j                  j
                  «      }| j                  d    |j                  ddd«      z  }|j	                  |j                  «      }| j                  D ]  \  }	}
 |	|||«      |z   } |
|«      |z   }Œ | j                  |«      }| j                  |«      S )Nr   r$   r(   r?   )rb   r,   rZ  r–  rd   rn   rœ   rv  râ   re   r™  rŸ  r‰  )rY   Úembedsr2   Úbr{  rÿ   Úlatent_maskrZ  rv  r�  ÚffÚtransformed_features               r4   rl   z)EvollaSequenceCompressorResampler.forwardÝ  s  € Ø�L‰L˜‰Oˆà—
‘
‰ˆˆAÜ—j‘j  T×%5Ñ%5Ó6×9Ñ9¸$¿+¹+ÓFˆÜ�y‰y˜$ Ð,°!Ô4ˆô �z‰z˜!‹}×Ñ §¡× 3Ñ 3Ó4ˆØ—,‘,˜tÑ$ t§y¡y°°Q¸Ó':Ñ:ˆØ—*‘*˜VŸ\™\Ó*ˆØŸ™ò 	,‰HˆD�"Ù˜6 7¨DÓ1°GÑ;ˆGÙ˜“k GÑ+‰Gð	,ð #×4Ñ4°WÓ=Ðà�y‰yÐ,Ó-Ð-r6   )rs   rt   ru   r%   rC   rl   rw   rx   s   @r4   r‘  r‘  Ç  s   ø„ ðQ˜|õ Qö*.r6   r‘  c                   óÆ   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)ÚEvollaProteinEncoderModelOutputaN  
    sequence_compressor_output (`torch.FloatTensor` of shape `(batch_size, compressed_seq_len, hidden_size)`, *optional*):
        Compressed sequence representation produced by the sequence compressor module. The sequence length is
        reduced from the original input length to `compressed_seq_len` via learned compression.
    NÚsequence_compressor_outputr,  .rÜ   r<  )rs   rt   ru   rv   r©  r,   rì   r¨   r,  rÜ   rª   r<  © r6   r4   r¨  r¨  ñ  st   … ñð <@Ð × 1Ñ 1°DÑ 8Ó?Ø26Ð�u×(Ñ(¨4Ñ/Ó6Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;r6   r¨  c                   óf   ‡ — e Zd Zdefˆ fd„Zedej                  dej                  fd„«       Z	ˆ xZ
S )ÚEvollaProteinEncoderrZ   c                 óz   •— t         ‰| �  «        t        |j                  ¬«      | _        t        |¬«      | _        y rK  )rB   rC   rI  r”  Úmodelr‘  Úsequence_compressor_resamplerrX   s     €r4   rC   zEvollaProteinEncoder.__init__  s.   ø€ Ü‰ÑÔÜ/°v×7TÑ7TÔUˆŒ
Ü-NÐV\Ô-]ˆÕ*r6   r0   rf   c                 ó’   — | j                  ||¬«      }|j                  }| j                  ||«      }t        ||j                  ¬«      S )NrY  )r©  r,  )r®  r,  r¯  r¨  )rY   r0   rf   r¾   Úprotein_outputÚprotein_embedsÚsequence_reprs          r4   rl   zEvollaProteinEncoder.forward  sJ   € àŸ™¨iÈ˜ÓWˆØ'×9Ñ9ˆØ×:Ñ:¸>È>ÓZˆä.Ø'4Ø,×>Ñ>ô
ð 	
r6   )rs   rt   ru   r%   rC   r   r,   Ú
LongTensorrì   rl   rw   rx   s   @r4   r¬  r¬     s?   ø„ ð^˜|õ ^ð
 ð
 ×!1Ñ!1ð 
À5×CTÑCTò 
ó ô
r6   r¬  c                   ób   ‡ — e Zd Z	 	 	 ddedz  dedz  dedz  fˆ fd„Zd„ Z	 	 	 	 	 	 	 d	d„Zˆ xZS )
Ú#EvollaSequenceAlignerCrossAttentionNÚprotein_encoder_dimÚstructure_encoder_dimÚmsa_encoder_dimc                 ó˜  •— t         ‰| �  «        |j                  | _        |j                  | _        | j                  dz  | _        t        | j                  | j                  z  «      | _        | j                  | j                  z  | _        |j                  }|j                  }|j                  }t        j                  | j                  | j                  «      | _        |�Kt        j                  || j                  «      | _        t        j                  || j                  «      | _        nd | _        d | _        |�Kt        j                  || j                  «      | _        t        j                  || j                  «      | _        nd | _        d | _        |�Kt        j                  || j                  «      | _        t        j                  || j                  «      | _        nd | _        d | _        t)        | j                  «      | _        t        j,                  |«      | _        t        j                  | j                  | j                  |¬«      | _        t3        | j                  |«      | _        t        j6                  t9        j:                  dg«      «      | _        t        j6                  t9        j:                  dg«      «      | _        y )NrÀ   rb  r]   ) rB   rC   rF   r�   rd  r+   rÔ   rÕ   Ú$aligner_attention_probs_dropout_probÚaligner_enable_biasÚaligner_ffn_multr   rÖ   rº   Úkey_proteinÚvalue_proteinÚkey_structureÚvalue_structureÚkey_msaÚ	value_msaÚEvollaRMSNormÚattention_normrM   rO   Úout_projr†  r¥  r—  r,   ÚtensorÚgate_attentionÚgate_ffw)	rY   rZ   r·  r¸  r¹  r×   Úenable_biasÚffn_multr[   s	           €r4   rC   z,EvollaSequenceAlignerCrossAttention.__init__  s  ø€ ô 	‰ÑÔà!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ Ø×-Ñ-¨tÑ3ˆŒ
Ü#& t×'7Ñ'7¸$×:RÑ:RÑ'RÓ#SˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔà'-×'RÑ'RÐ$Ø×0Ñ0ˆØ×*Ñ*ˆä—Y‘Y˜t×/Ñ/°×1CÑ1CÓDˆŒ
ØÐ*Ü!Ÿy™yÐ)<¸d×>PÑ>PÓQˆDÔÜ!#§¡Ð+>À×@RÑ@RÓ!SˆDÕà#ˆDÔØ!%ˆDÔà Ð,Ü!#§¡Ð+@À$×BTÑBTÓ!UˆDÔÜ#%§9¡9Ð-BÀD×DVÑDVÓ#WˆDÕ à!%ˆDÔØ#'ˆDÔ àÐ&ÜŸ9™9 _°d×6HÑ6HÓIˆDŒLÜŸY™Y ¸×8JÑ8JÓKˆD�NàˆDŒLØ!ˆDŒNä+¨D×,<Ñ,<Ó=ˆÔä—z‘zÐ">Ó?ˆŒäŸ	™	 $×"2Ñ"2°D×4DÑ4DÈ;ÔWˆŒä# D×$4Ñ$4°hÓ?ˆŒÜ Ÿl™l¬5¯<©<¸¸Ó+>Ó?ˆÔÜŸ™¤U§\¡\°3°%Ó%8Ó9ˆ�r6   c	                 óˆ  — |||g}	|	D �
cg c]  }
|
€Œ|
‘Œ	 }	}
|	st        d«      ‚t        j                  |	d¬«      }	| j                  |«      }| j	                  |«      }| j
                  �@| j                  �4|j                  |«      }| j                  |«      }| j                  |«      }nd}d}| j                  �@| j                  �4|j                  |«      }| j                  |«      }| j                  |«      }nd}d}| j                  �@| j                  �4|j                  |«      }| j                  |«      }| j                  |«      }nd}d}|||g}|D �
cg c]  }
|
€Œ|
‘Œ	 }}
t        j                  |d¬«      }|||g}|D �
cg c]  }
|
€Œ|
‘Œ	 }}
t        j                  |d¬«      }|j                  «       dd | j                  | j                  fz   } |j                  |Ž j!                  dddd«      }|j                  «       dd | j                  | j                  fz   } |j                  |Ž j!                  dddd«      }|j                  «       dd | j                  | j                  fz   } |j                  |Ž j!                  dddd«      }|| j"                  z  }|€Mt        j$                  |j                  d«      |j                  d«      «      j                  |j&                  «      }|dd…ddd…df   |	dd…dddd…f   z  }t        j(                  ||j+                  dd	«      «      }||j-                  dd
¬«      j/                  «       z
  }|j1                  d|z
  j3                  «       t        j4                  |j6                  «      j8                  «      } t;        j<                  d¬«      |«      }t        j(                  ||«      }|j!                  dddd«      j?                  «       }|j                  «       dd	 | j@                  fz   } |j                  |Ž }| jC                  |«      }|S c c}
w c c}
w c c}
w )zù
        query_states: text
        key_value_states: protein
        query_states: [bs, query_seq_len, dim]
        key_value_states: [bs, kv_seq_len, dim]
        query_attn_mask: [bs, query_seq_len]
        kv_attn_mask: [bs, kv_seq_len]
        Nz=At least one modality should be provided for cross attention.r$   r(   r?   r   r‰   r   rn  Tro  )"rÓ   r,   rœ   rÅ  rº   r¾  r¿  rd   rÀ  rÁ  rÂ  rÃ  ro   r�   rÔ   râ   rr  rd  rZ  rn   rÅ   r›   rs  rt  r_   ru  Úfinfore   Úminr   ÚSoftmaxrÈ   rÕ   rÆ  )rY   Úquery_statesÚprotein_key_value_statesÚstructure_key_value_statesÚmsa_key_value_statesÚquery_attn_maskÚprotein_kv_attn_maskÚstructure_kv_attn_maskÚmsa_kv_attn_maskÚkv_attn_maskrÿ   rç   Úkey_layer_proteinÚvalue_layer_proteinÚkey_layer_structureÚvalue_layer_structureÚkey_layer_msaÚvalue_layer_msaré   rê   Únew_query_layer_shapeÚnew_key_layer_shapeÚnew_value_layer_shaperf   rÊ   Úattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapes                                r4   Úcross_attentionz3EvollaSequenceAlignerCrossAttention.cross_attentionF  si  € ð* -Ð.DÐFVÐWˆØ#/ÖA˜a°1±=šÐAˆÐAÙÜÐ\Ó]Ð]Ü—y‘y °1Ô5ˆà×)Ñ)¨,Ó7ˆð —j‘j Ó-ˆà×ÑÐ'¨D×,>Ñ,>Ð,JØ'?×'BÑ'BÀ<Ó'PÐ$Ø $× 0Ñ 0Ð1IÓ JÐØ"&×"4Ñ"4Ð5MÓ"NÑà $ÐØ"&Ðà×ÑÐ)¨d×.BÑ.BÐ.NØ)C×)FÑ)FÀ|Ó)TÐ&Ø"&×"4Ñ"4Ð5OÓ"PÐØ$(×$8Ñ$8Ð9SÓ$TÑ!à"&ÐØ$(Ð!à�<‰<Ð#¨¯©Ð(BØ#7×#:Ñ#:¸<Ó#HÐ Ø ŸL™LÐ)=Ó>ˆMØ"Ÿn™nÐ-AÓB‰Oà ˆMØ"ˆOà&Ð(;¸]ÐKˆ	Ø )Ö;˜1¨Q©]’QÐ;ˆ	Ð;Ü—I‘I˜i¨QÔ/ˆ	à*Ð,AÀ?ÐSˆØ"-Ö?˜Q°±’qÐ?ˆÐ?Ü—i‘i °Ô3ˆà +× 0Ñ 0Ó 2°3°BÐ 7Ø×$Ñ$Ø×$Ñ$ð;
ñ !
Ðð '�k×&Ñ&Ð(=Ð>×FÑFÀqÈ!ÈQÐPQÓRˆà'Ÿn™nÓ.¨s°Ð3Ø×$Ñ$Ø×$Ñ$ð7
ñ 
Ðð #�I—N‘NÐ$7Ð8×@Ñ@ÀÀAÀqÈ!ÓLˆ	à +× 0Ñ 0Ó 2°3°BÐ 7Ø×$Ñ$Ø×$Ñ$ð;
ñ !
Ðð '�k×&Ñ&Ð(=Ð>×FÑFÀqÈ!ÈQÐPQÓRˆà! D§J¡JÑ.ˆð Ð"Ü#Ÿj™j¨×):Ñ):¸1Ó)=¸|×?PÑ?PÐQRÓ?SÓT×WÑWÐXd×XkÑXkÓlˆOØ(ª¨D²!°TÐ)9Ñ:¸\Ê!ÈTÐSWÒYZÐJZÑ=[Ñ[ˆä—|‘| K°×1DÑ1DÀRÈÓ1LÓMˆØ# l×&7Ñ&7¸BÈÐ&7Ó&M×&TÑ&TÓ&VÑVˆØ'×3Ñ3Ø�Ñ×%Ñ%Ó'¬¯©°\×5GÑ5GÓ)H×)LÑ)Ló
Ðð -œ"Ÿ*™*¨Ô,Ð-=Ó>ˆô Ÿ™ _°kÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ*˜×*Ñ*Ð,CÐDˆàŸ™ mÓ4ˆàÐùòq BùòL <ùò @s"   ŠP5’P5ÅP:ÅP:ÆP?ÆP?c           
      ó^  — |�z|j                   \  }}}|€jt        j                  ||«      j                  |	j                  «      |	j                  ||f¬«      j                  z  j                  |j                  «      }nd }|�z|j                   \  }}}|€jt        j                  ||«      j                  |	j                  «      |
j                  ||f¬«      j                  z  j                  |j                  «      }nd }|�z|j                   \  }}}|€jt        j                  ||«      j                  |	j                  «      |j                  ||f¬«      j                  z  j                  |j                  «      }nd }|}|�|j                  «       s$|�|j                  «       s|�Š|j                  «       rz|}| j                  ||||||||¬«      }t        j                  | j                  «      |z  }||z   }|}| j                  |«      t        j                  | j                  «      z  }||z   }|S )N)ro   )rÐ  rÑ  rÒ  rÓ  rÔ  rÕ  rÖ  r×  )rb   r,   rZ  rd   rn   rT   ÚTÚanyræ  ÚtanhrÈ  r¥  rÉ  )rY   rÐ  Úprotein_kv_statesÚstructure_kv_statesÚmsa_kv_statesrÔ  rÕ  rÖ  r×  Úprotein_batch_maskÚstructure_batch_maskÚmsa_batch_maskÚpast_key_valuesr{  Úprotein_kv_seq_lenr)   Ústructure_kv_seq_lenÚmsa_kv_seq_lenrÜ   Úresiduals                       r4   rl   z+EvollaSequenceAlignerCrossAttention.forward¶  sL  € ð Ð(Ø*;×*AÑ*AÑ'ˆBÐ" CØ#Ð+ä—J‘J˜rÐ#5Ó6×9Ñ9Ð:L×:SÑ:SÓTØ(×/Ñ/Ð6HÈ"Ð5MÐ/ÓN×PÑPñQç‘"Ð&×-Ñ-Ó.ñ %ð
 $(Ð àÐ*Ø,?×,EÑ,EÑ)ˆBÐ$ cØ%Ð-ä—J‘J˜rÐ#7Ó8×;Ñ;Ð<N×<UÑ<UÓVØ*×1Ñ1Ð8LÈbÐ7QÐ1ÓR×TÑTñUç‘"Ð(×/Ñ/Ó0ñ 'ð
 &*Ð"àÐ$Ø&3×&9Ñ&9Ñ#ˆB� ØÐ'ä—J‘J˜r >Ó2×5Ñ5Ð6H×6OÑ6OÓPØ$×+Ñ+°.À"Ð1EÐ+ÓF×HÑHñIç‘"�]×)Ñ)Ó*ñ !ð
  $ÐØ$ˆð Ð*Ð/C×/GÑ/GÔ/IØ#Ð/Ð4J×4NÑ4NÔ4PØÐ)Ð.>×.BÑ.BÔ.Dà$ˆHØ ×0Ñ0Ø*Ø):Ø+>Ø%2Ø /Ø%9Ø'=Ø!1ð 1ó 	ˆMô "ŸJ™J t×':Ñ':Ó;¸mÑKˆMà$ }Ñ4ˆMà$ˆHØ ŸG™G MÓ2´U·Z±ZÀÇÁÓ5NÑNˆMØ$ }Ñ4ˆMàÐr6   r¦   )NNNNNNN)rs   rt   ru   r+   rC   ræ  rl   rw   rx   s   @r4   r¶  r¶    sb   ø„ ð +/Ø,0Ø&*ñ1:ð ! 4™Zð1:ð  # T™zð	1:ð
 ˜t™õ1:òfnðn "Ø#ØØØ!ØØ÷Gr6   r¶  ÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	rÄ  r;   r„   Nc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z<
        EvollaRMSNorm is equivalent to T5LayerNorm
        N)rB   rC   r   r—  r,   rZ  ÚweightÚvariance_epsilon)rY   rF   r;   r[   s      €r4   rC   zEvollaRMSNorm.__init__  s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr6   rÜ   c                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Nr‰   r?   T)rp  )	re   rd   r,   Úfloat32ÚpowÚmeanÚrsqrtrú  rù  )rY   rÜ   Úinput_dtypeÚvariances       r4   rl   zEvollaRMSNorm.forward
  sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r6   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)rª   rù  rb   rú  rR  s    r4   Ú
extra_reprzEvollaRMSNorm.extra_repr  s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr6   )g�íµ ÷Æ°>)
rs   rt   ru   rc   rC   r,   r§   rl   r  rw   rx   s   @r4   rÄ  rÄ     s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJr6   rÄ  c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚEvollaRotaryEmbeddingr{   NrZ   c                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )Nr~   Údefaultr{   Fr@   Úoriginal_inv_freq)rB   rC   rS   Úmax_seq_len_cachedÚoriginal_max_seq_lenrZ   Úrope_parametersr~   r   r   r}   rQ   Úclone)rY   rZ   rn   Úrope_init_fnr{   r[   s        €r4   rC   zEvollaRotaryEmbedding.__init__  s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr6   rn   ztorch.devicerƒ   r„   r…   c                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        rŒ   r‡   Nrˆ   r   r‰   rŠ   r‹   )	r  rP   rF   r�   r,   rR   rŽ   rd   rc   r�   s          r4   r   z5EvollaRotaryEmbedding.compute_default_rope_parameters(  s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r6   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   r?   r$   r“   r”   Fr•   r‰   r(   rŠ   )r{   rc   rT   rb   rd   rn   r˜   r™   rš   r    r›   r,   rœ   r�   r}   rž   re   )
rY   rŸ   r>   r¡   r¢   r–   r£   r¤   r�   rž   s
             r4   rl   zEvollaRotaryEmbedding.forwardF  sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$r¥   r¦   )rs   rt   ru   r,   r§   r¨   r%   rC   r©   r   r+   rª   rc   r   r«   r   rl   rw   rx   s   @r4   r  r    s˜   ø… Ø�l‰lÓñV˜|õ Vð  à&*Ø+/Ø"ñ*Ø˜tÑ#ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r6   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú	EvollaMLPc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  | j                  | j                  |j                  ¬«      | _	        t        j                  | j                  | j                  |j                  ¬«      | _
        t        |j                     | _        y )Nrb  )rB   rC   rZ   rF   r  r   rÖ   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr	   Ú
hidden_actÚact_fnrX   s     €r4   rC   zEvollaMLP.__init__W  s¸   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Ñ/ˆ�r6   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S r¥   )r  r  r  r  )rY   rŸ   r  s      r4   rl   zEvollaMLP.forwarda  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr6   rö   rx   s   @r4   r  r  V  s   ø„ ô0ör6   r  rÜ   rÉ   r„   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r$   N)rb   rT   rå   )rÜ   rÉ   ÚbatchÚnum_key_value_headsÚslenr‡   s         r4   rÄ   rÄ   f  so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr6   c                   óø   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
dz  d
ee   de	ej                  ej                  f   fd„Zˆ xZS )ÚEvollaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrZ   rÙ   c                 ód  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                  ¬«      | _        y )Nr‡   rÀ   Trb  )rB   rC   rZ   rÙ   rP   rF   r�   r‡   r  rÁ   r½   Úattention_dropoutrÚ   r   rÖ   Úattention_biasÚq_projÚk_projÚv_projÚo_proj©rY   rZ   rÙ   r[   s      €r4   rC   zEvollaAttention.__init__v  sM  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆ�r6   NrÜ   rU   rf   rñ  r¾   r„   c                 ó
  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  sdn| j                   | j"                  dœ|¤Ž\  }} |j$                  g |¢d‘­Ž j'                  «       }| j)                  |«      }||fS )Nr?   r$   r‰   r]   rá   )rb   r‡   r#  râ   r›   r$  r%  r¸   ÚupdaterÙ   r   rã   rZ   rä   rÌ   rÃ   r!  r½   rå   rÈ   r&  )rY   rÜ   rU   rf   rñ  r¾   rp   ræ   rÐ  Ú
key_statesÚvalue_statesr�   rž   rë   rË   rÊ   s                   r4   rl   zEvollaAttention.forward�  s‘  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r6   r¦   )rs   rt   ru   rv   r%   r+   rC   r,   r§   rª   r
   r   r   rl   rw   rx   s   @r4   r  r  r  s¥   ø„ áGð
˜|ð 
¸õ 
ð4 IMØ.2Ø(,ñ&)à—|‘|ð&)ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð&)ð Ÿ™ tÑ+ð	&)ð
  ™ð&)ð Ð+Ñ,ð&)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷&)r6   r  c                   óØ  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  fd„Zˆ xZS )ÚEvollaDecoderLayerrZ   rÙ   c                 ó²  •— t         ‰| �  «        |j                  | _        t        ||¬«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _	        t        |j                  |j                  ¬«      | _
        |dz   t        |j                  |j                  z  d«      z  dk(  rt        ||j                  ¬«      | _        y y )N©rZ   rÙ   r:   r$   r   )r·  )rB   rC   rF   r  Ú	self_attnr  ÚmlprÄ  Úrms_norm_epsÚinput_layernormÚpost_attention_layernormÚmaxr'  Úaligner_num_add_layersr¶  Úadapterr'  s      €r4   rC   zEvollaDecoderLayer.__init__·  sµ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä(°À)ÔLˆŒä˜VÓ$ˆŒÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÔ%Ø˜‰MœS ×!9Ñ!9¸V×=ZÑ=ZÑ!ZÐ\]Ó^Ñ^ÐbcÒcÜ>ØØ$*×$6Ñ$6ôˆD�Lð dr6   NrÜ   rU   rf   r>   rñ  Ú	use_cacherë  rì  rí  rî  rï  rð  rÔ  r„   c           
      ó
  — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}||z   }|}| j                  |«      }| j                  |«      }||z   }t	        | d«      r| j                  ||||	||
||¬«      }|S )N)rÜ   rf   r>   rñ  r8  rU   r7  )rÐ  rë  rì  rí  rÔ  rî  rï  rð  rª  )r3  r0  r4  r1  rÒ   r7  )rY   rÜ   rU   rf   r>   rñ  r8  rë  rì  rí  rî  rï  rð  rÔ  r¾   rõ  rÿ   s                    r4   rl   zEvollaDecoderLayer.forwardÆ  sÉ   € ð" !ˆà×,Ñ,¨]Ó;ˆð *˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ  =Ñ0ˆä�4˜Ô#Ø ŸL™LØ*Ø"3Ø$7Ø+Ø /Ø#5Ø%9Ø-ð )ó 	ˆMð Ðr6   )NNNNFNNNNNNN)rs   rt   ru   r%   r+   rC   r,   r§   rª   r´  r
   ru  rl   rw   rx   s   @r4   r-  r-  ¶  sG  ø„ ð˜|ð ¸õ ð$ IMØ.2Ø04Ø(,Ø!&Ø15Ø37Ø-1Ø26Ø48Ø.2Ø/3ñ3à—|‘|ð3ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð3ð Ÿ™ tÑ+ð	3ð
 ×&Ñ&¨Ñ-ð3ð  ™ð3ð ˜$‘;ð3ð !Ÿ<™<¨$Ñ.ð3ð #Ÿ\™\¨DÑ0ð3ð —|‘| dÑ*ð3ð "ŸL™L¨4Ñ/ð3ð $Ÿl™l¨TÑ1ð3ð Ÿ™ tÑ+ð3ð Ÿ™¨Ñ,ð3ð  
�‰÷!3r6   r-  c                   ó†   ‡ — e Zd ZU eed<   dZdZg d¢ZdgZdZ	dZ
dZdZdZeedœZ ej$                  «       ˆ fd„«       Zˆ xZS )	ÚEvollaPreTrainedModelrZ   r®  T)r-  r  r‘  r¶  rñ  F)rÜ   r<  c                 ó¦  •— | j                   j                  }t        ‰| �  |«       t	        |t
        «      rht        j                  |j                  «       t        j                  |j                  «       t        j                  |j                  j                  «       y t	        |t        «      r#t        j                  |j                  d|¬«       y y )Nr]   )rþ  Ústd)rZ   Úinitializer_rangerB   r?  r˜   r¶  r@  Úzeros_rÈ  rÉ  Úones_rÅ  rù  r‘  Únormal_rv  )rY   r¹   r=  r[   s      €r4   r?  z#EvollaPreTrainedModel._init_weights  s‰   ø€ à�k‰k×+Ñ+ˆÜ‰Ñ˜fÔ%Ü�fÔAÔBÜ�K‰K˜×-Ñ-Ô.Ü�K‰K˜Ÿ™Ô(Ü�J‰J�v×,Ñ,×3Ñ3Õ4Ü˜Ô AÔBÜ�L‰L˜Ÿ™¨c°sÖ;ð Cr6   )rs   rt   ru   r%   r¨   Úbase_model_prefixÚsupports_gradient_checkpointingrB  Ú_skip_keys_device_placementrC  rD  rE  Ú_can_compile_fullgraphrF  r-  r  rG  r,   r«   r?  rw   rx   s   @r4   r;  r;  ü  so   ø… àÓØÐØ&*Ð#òÐð $5Ð"5ÐØ ÐØ€NØÐà!ÐØ"'Ðà+Ø%ñÐð
 €U‡]�]ƒ_ó<ó ô<r6   r;  c                   óº  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zeee		 	 	 	 	 	 	 	 	 	 	 	 dde
j                  dz  de
j                  dz  de
j                  dz  d	edz  d
e
j                  dz  dedz  de
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  deez  fd„«       «       «       Zˆ xZS )ÚEvollaModelrZ   c           	      óF  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  | j                  |j                  | j                  «      | _        t        |¬«      | _
        t        j                  t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}«      | _        t!        |j                  |j"                  ¬«      | _        t'        |dd«      | _        t+        |¬«      | _        | j/                  «        y c c}w )NrL  r/  r:   r*  F)rB   rC   rG   r1   rE   r   rD   rF   Úembed_tokensr¬  Úprotein_encoderr%  r&  r'  r-  r™  rÄ  r2  r‰  rP   r*  r  Ú
rotary_embrO  r'  s      €r4   rC   zEvollaModel.__init__   sç   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒÜŸL™L¨¯©¸&×:LÑ:LÈd×N^ÑN^Ó_ˆÔÜ3¸6ÔBˆÔÜ—m‘mô "' v×'?Ñ'?Ó!@öð
 ô	 #Ø!Ø'öòó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü&-¨fÐ6NÐPUÓ&VˆÔ#Ü/°vÔ>ˆŒØ�‰Õùòs   Â$Dc                 ó   — | j                   S r¥   ©rI  rR  s    r4   rS  z EvollaModel.get_input_embeddings5  s   € Ø× Ñ Ð r6   c                 ó   — || _         y r¥   rM  rU  s     r4   rV  z EvollaModel.set_input_embeddings8  s
   € Ø!ˆÕr6   Nr0   rf   r>   rñ  rg   r8  Úprotein_input_idsÚprotein_attention_maskÚstructure_featsÚ	msa_featsrï  rð  r„   c                 óæ  — |du |duz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }d}d}|�^|�\| j                  ||¬«      }|j                  }t        j                  |j                  d   |j                  t
        j                  ¬«      }t        | j                  |||¬	«      }|}| j                  ||¬
«      }| j                   D ]  } ||f||||||	|
|||||dœ|¤Ž}Œ | j#                  |«      }t%        ||¬«      }|S )a;  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
        structure_feats (torch.FloatTensor):
            The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        msa_feats (torch.FloatTensor):
            The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        structure_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummpy input for now.
        msa_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummpy input for now.
        Nz:You must specify exactly one of input_ids or inputs_embedsrL  r   r$   rX  rY  r‹   )rZ   rg   rf   rñ  )r>   )rf   r>   rñ  r8  rë  rì  rí  rî  rï  rð  rÔ  rU   )r,  rñ  )rÓ   rI  r   rZ   Úget_seq_lengthr,   rR   rb   rn   r`   rJ  r©  rZ  ru  r   rK  r™  r‰  r   )rY   r0   rf   r>   rñ  rg   r8  rO  rP  rQ  rR  rï  rð  r¾   Úpast_seen_tokensÚprotein_featsrî  Úprotein_outputsÚcausal_maskrÜ   rU   Údecoder_layerrû   s                          r4   rl   zEvollaModel.forward;  s¿  € ðB ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLàˆØ!ÐàÐ(Ð-CÐ-OØ"×2Ñ2Ø+Ø5ð 3ó ˆOð ,×FÑFˆMÜ!&§¡Ø!×'Ñ'¨Ñ*Ø(×/Ñ/Ü—j‘jô"Ðô )Ø—;‘;Ø'Ø)Ø+ô	
ˆð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[ò 	ˆMÙ)Øðà*Ø)Ø /Ø#Ø"/Ø$3Ø'Ø#5Ø%9Ø-Ø .Ø$7ñð ñ‰Mð	ð$ Ÿ	™	 -Ó0ˆä(Ø+Ø+ô
ˆð ˆr6   )NNNNNNNNNNNN)rs   rt   ru   r%   rC   rS  rV  r   r!   r#   r,   r´  r§   r
   rì   ru  rª   r   rl   rw   rx   s   @r4   rG  rG    sf  ø„ ð˜|õ ò*!ò"ð ØØð .2Ø.2Ø04Ø(,Ø26Ø!%Ø59Ø6:Ø48Ø.2Ø48Ø.2ñ]à×#Ñ# dÑ*ð]ð Ÿ™ tÑ+ð]ð ×&Ñ&¨Ñ-ð	]ð
  ™ð]ð ×(Ñ(¨4Ñ/ð]ð ˜$‘;ð]ð !×+Ñ+¨dÑ2ð]ð !&§¡¨tÑ 3ð]ð ×*Ñ*¨TÑ1ð]ð ×$Ñ$ tÑ+ð]ð $Ÿl™l¨TÑ1ð]ð Ÿ™ tÑ+ð]ð 
Ð(Ñ	(ò]ó ó  ó ô]r6   rG  c                   ó4  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dedz  deej                  z  fd„«       «       Zˆ xZS )ÚEvollaForProteinText2Textc                 óè   •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  | j                  d¬«      | _        | j                  «        y rˆ  )
rB   rC   rG  r®  rE   r   rÖ   rF   Úlm_headrO  rX   s     €r4   rC   z"EvollaForProteinText2Text.__init__Ÿ  sQ   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°T·_±_È5ÔQˆŒà�‰Õr6   c                 ó6   — | j                   j                  «       S r¥   )r®  rS  rR  s    r4   rS  z.EvollaForProteinText2Text.get_input_embeddings§  s   € Ø�z‰z×.Ñ.Ó0Ð0r6   c                 ó8   — | j                   j                  |«      S r¥   )r®  rV  rU  s     r4   rV  z.EvollaForProteinText2Text.set_input_embeddingsª  s   € Ø�z‰z×.Ñ.¨uÓ5Ð5r6   Nr0   rf   rg   ÚlabelsrO  rP  r8  Úlogits_to_keepc	           
      óh  —  | j                   d||||||dœ|	¤Ž}
|
j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      }d}|�  | j                  d||| j                  dœ|	¤Ž}t        |||
j                  |
j                  |
j                  ¬«      }|S )a,  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

        Example:

        ```python
        >>> from transformers import EvollaProcessor, EvollaForProteinText2Text
        >>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
        >>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

        >>> protein_information = {
            "aa_seq": "your amino acid sequence",
            "foldseek": "your foldseek sequence",
        }
        >>> question = "What is the function of this protein?"
        >>> message = [
            {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
            {"role": "user", "content": question},
        ]

        >>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
        >>> outputs = model.generate(**inputs)

        >>> print(processor.batch_decode(outputs, skip_special_tokens=True))
        ```)r0   rf   rg   rO  rP  r8  N)Úlogitsr`  rE   )Úlossrc  rñ  rÜ   r<  rª  )r®  r,  r˜   r+   Úslicer]  Úloss_functionrE   r   rñ  rÜ   r<  )rY   r0   rf   rg   r`  rO  rP  r8  ra  r¾   ÚoutputsrÜ   Úslice_indicesrc  rd  Ú
lm_outputss                   r4   rl   z!EvollaForProteinText2Text.forward­  sÙ   € ðT ,6¨4¯:©:ð ,
ØØ)Ø'Ø/Ø#9Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%Ði¨V¸FÈtÏÉÑiÐbhÑiˆDä+ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ˆ
ð Ðr6   )NNNNNNNr   )rs   rt   ru   rC   rS  rV  r   r   r,   r´  r§   rì   ru  r+   rl   rw   rx   s   @r4   r[  r[  ž  së   ø„ ôò1ò6ð Øð .2Ø.2Ø26Ø*.Ø59Ø6:Ø!%Ø-.ñBà×#Ñ# dÑ*ðBð Ÿ™ tÑ+ðBð ×(Ñ(¨4Ñ/ð	Bð
 × Ñ  4Ñ'ðBð !×+Ñ+¨dÑ2ðBð !&§¡¨tÑ 3ðBð ˜$‘;ðBð ˜eŸl™lÑ*òBó ó ôBr6   r[  )r[  rG  r;  )r$   )Nr]   )\r  Úcollections.abcr   Údataclassesr   Útypingr   r,   r   Ú r   r@  Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úintegrationsr   r   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr    r!   Úutils.output_capturingr"   r#   Úconfiguration_evollar%   r&   r5   ÚModuler8   rz   r¯   r¸   r§   rc   rÌ   rÎ   rî   rù   r  r  r  r  r#  r1  r9  rI  r`  r†  r‘  r¨  r¬  r¶  rÄ  r  r  r+   rÄ   r  r-  r;  rG  r[  Ú__all__rª  r6   r4   ú<module>r~     sR  ðó* Ý $Ý !Ý ã Ý å &Ý !ß .Ý )ß fÑ fß JÝ 9÷õ ÷ Lß FÝ &ß IÑ Iß Gß Eß <ò4ô ^=˜RŸY™Yô ^=ôBA< "§)¡)ô A<òH(ñ Ð*Ó+òBó ,ðBð@ !Øñ %Ø�I‰Ið %à�<‰<ð %ð 
�‰ð %ð �<‰<ð	 %ð
 —L‘L 4Ñ'ð %ð �T‰\ð %ð ð %ð Ð'Ñ(ó %ñF Ð)Ó*ôK) §	¡	ó K)ó +ðK)ô\
˜RŸY™Yô 
ô˜BŸI™Iô ò<;ô˜rŸy™yô ô
˜Ÿ™ô 
ô7Ð2ô 7ôtS˜"Ÿ)™)ô SôD˜Ÿ™ô ð ôC /ó Có ðCô01
Ð!<ô 1
ôh7 ¨¯	©	ô 7 ôtA˜Ÿ	™	ô Aô'.¨¯	©	ô '.ðT Ø
ô
< kó 
<ó ó ð
<ô
˜2Ÿ9™9ô 
ô$k¨"¯)©)ô kñ\ ˜YÓ'ôJ�B—I‘Ió Jó (ðJô(><˜BŸI™Iô ><ôB�—	‘	ô ð 	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uñ Ð)Ó*ô@)�b—i‘ió @)ó +ð@)ôFCÐ3ô CðL ô<˜Oó <ó ð<ôD|Ð'ô |ô~SÐ 5°ô Sòl P�r6   