Ë
    ÛÓ^js�  ã                   ó´  — d dl mZ d dlZd dlmZ ddlmZ ddlmZm	Z	 ddl
mZ ddlmZmZ dd	lmZmZmZmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZ ddlm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7  ejp                  e9«      Z:	 	 dRdejv                  dejx                  dejx                  dejx                  dejx                  dz  de=dz  de=dee   fd„Z> G d„ de$«      Z? G d„ d e*«      Z@ G d!„ d"e+«      ZA G d#„ d$e,«      ZB G d%„ d&e#«      ZC G d'„ d(e&«      ZD G d)„ d*e(«      ZE G d+„ d,e'«      ZF G d-„ d.e%«      ZG G d/„ d0e)«      ZHe G d1„ d2e«      «       ZI G d3„ d4eI«      ZJ G d5„ d6ejv                  «      ZK G d7„ d8ejv                  «      ZL G d9„ d:ejv                  «      ZMee G d;„ d<e«      «       «       ZN G d=„ d>ejv                  «      ZO G d?„ d@ejv                  «      ZP G dA„ dBe2«      ZQ G dC„ dDe3«      ZR G dE„ dFe0«      ZS G dG„ dHe.«      ZT G dI„ dJe/«      ZU G dK„ dLe1«      ZV G dM„ dNeV«      ZW G dO„ dPeVe«      ZXg dQ¢ZYy)Sé    )Ú	dataclassN)Únné   )Úinitialization)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚBaseModelOutputWithPastÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚModelOutput)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )
ÚEsmAttentionÚEsmEmbeddingsÚ
EsmEncoderÚEsmIntermediateÚEsmLayerÚ	EsmOutputÚ	EsmPoolerÚEsmRotaryEmbeddingÚEsmSelfAttentionÚEsmSelfOutput)ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaMLPÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚ	repeat_kvé   )ÚEvollaConfigÚSaProtConfigÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 óæ  — |€|j                  d«      dz  }t        | dd«      }t        ||«      }t        ||«      }t        j                  ||j                  dd«      «      |z  }	|�|	|z   }	t        j                  j                  |	d¬«      }	t        j                  j                  |	|| j                  ¬«      }	t        j                  |	|«      }
|
j                  dd«      j                  «       }
|
|	fS )	Néÿÿÿÿç      à¿Únum_key_value_groupsr+   r   r   ©Údim)ÚpÚtraining)ÚsizeÚgetattrr*   ÚtorchÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr4   r=   Ú
contiguous)r.   r/   r0   r1   r2   r3   r4   r5   Ún_repÚattn_weightsÚattn_outputs              út/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/evolla/modular_evolla.pyÚeager_attention_forwardrJ   C   sä   € ð €Ø—*‘*˜R“. DÑ(ˆô
 �FÐ2°AÓ6€EÜ
�C˜Ó
€CÜ�e˜UÓ#€Eô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$ó    c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚEvollaSaProtEmbeddingsc                 ó2   •— t         ‰| �  |«       d | _        y ©N)ÚsuperÚ__init__Úposition_ids©ÚselfÚconfigÚ	__class__s     €rI   rQ   zEvollaSaProtEmbeddings.__init__g   s   ø€ Ü‰Ñ˜Ô à ˆÕrK   )Ú__name__Ú
__module__Ú__qualname__rQ   Ú__classcell__©rV   s   @rI   rM   rM   f   s   ø„ ÷!ð !rK   rM   c                   ój   ‡ — e Zd Zd
defˆ fd„Ze	 	 	 ddedz  dddedz  dedef   fˆ fd	„«       Z	ˆ xZ
S )ÚEvollaSaProtRotaryEmbeddingNrU   c                 ó&   •— t         ‰| �  ||«       y rO   )rP   rQ   )rT   rU   ÚdevicerV   s      €rI   rQ   z$EvollaSaProtRotaryEmbedding.__init__n   s   ø€ Ü‰Ñ˜ Õ(rK   r_   ztorch.device | NoneÚseq_lenÚreturnztorch.Tensorc                 ó&   •— t         ‰| �  | ||«      S rO   )rP   Úcompute_default_rope_parameters)rU   r_   r`   rV   s      €rI   rc   z;EvollaSaProtRotaryEmbedding.compute_default_rope_parametersq   s   ø€ ô ‰wÑ6°v¸vÀwÓOÐOrK   rO   ©NNN)rW   rX   rY   r-   rQ   ÚstaticmethodÚintÚtupleÚfloatrc   rZ   r[   s   @rI   r]   r]   m   si   ø„ ñ)˜|õ )ð à&*Ø(,Ø"ñPØ˜tÑ#ðPà%ðPð �t‘ðPð 
ˆ~˜uÐ$Ñ	%ô	Pó ôPrK   r]   c                   ó   — e Zd Zy)ÚEvollaSaProtSelfAttentionN©rW   rX   rY   © rK   rI   rj   rj   z   ó   „ ØrK   rj   c                   ó   — e Zd Zy)ÚEvollaSaProtSelfOutputNrk   rl   rK   rI   ro   ro   ~   rm   rK   ro   c                   ó   — e Zd Zy)ÚEvollaSaProtAttentionNrk   rl   rK   rI   rq   rq   ‚   rm   rK   rq   c                   ó   — e Zd Zy)ÚEvollaSaProtIntermediateNrk   rl   rK   rI   rs   rs   †   rm   rK   rs   c                   ó   — e Zd Zy)ÚEvollaSaProtOutputNrk   rl   rK   rI   ru   ru   Š   rm   rK   ru   c                   ó   — e Zd Zy)ÚEvollaSaProtLayerNrk   rl   rK   rI   rw   rw   Ž   rm   rK   rw   c                   ó   — e Zd Zy)ÚEvollaSaProtEncoderNrk   rl   rK   rI   ry   ry   ’   rm   rK   ry   c                   ó   — e Zd Zy)ÚEvollaSaProtPoolerNrk   rl   rK   rI   r{   r{   –   rm   rK   r{   c                   óœ   ‡ — e Zd ZU eed<   dgZdZdZdZdZ	e
 eedd¬«      g eedd¬«      gdœZ ej                  «       ˆ fd	„«       Zˆ xZS )
ÚEvollaSaProtPreTrainedModelrU   rw   Tr+   Ú	attention)ÚindexÚ
layer_nameÚcrossattention)Úhidden_statesÚ
attentionsÚcross_attentionsc                 óÂ   •— t         ‰| �  |«       t        |t        «      r?|j	                  |j
                  «      \  }}t        j                  t        |d«      |«       y y )NÚinv_freq)	rP   Ú_init_weightsÚ
isinstancer]   rc   rU   ÚinitÚcopy_r?   )rT   r.   Úcurr_inv_freqÚ_rV   s       €rI   r‡   z)EvollaSaProtPreTrainedModel._init_weights«   sN   ø€ ä‰Ñ˜fÔ%Ü�fÔ9Ô:Ø%×EÑEÀfÇmÁmÓTÑˆM˜1Ü�J‰J”w˜v zÓ2°MÕBð ;rK   )rW   rX   rY   r-   Ú__annotations__Ú_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrw   r   rj   Ú_can_record_outputsr@   Úno_gradr‡   rZ   r[   s   @rI   r}   r}   š   sx   ø… àÓØ,Ð-ÐØÐØ€NØÐØ"&Ðð +Ù%Ð&?ÀqÐU`ÔaÐbáÐ4¸AÐJZÔ[ð
ñÐð €U‡]�]ƒ_óCó ôCrK   r}   c            
       ó°   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 d
de	j                  dz  de	j                  dz  dee	j                     ez  fd	„«       «       Zˆ xZS )ÚEvollaSaProtProteinEncoderrU   c                 ó¦   •— t         ‰| �  |«       t        |«      | _        t	        |¬«      | _        t        |«      | _        | j                  «        y ©N©rU   )	rP   rQ   rM   Ú
embeddingsr]   Úrotary_embeddingsry   ÚencoderÚ	post_initrS   s     €rI   rQ   z#EvollaSaProtProteinEncoder.__init__´   s?   ø€ Ü‰Ñ˜Ô Ü0°Ó8ˆŒÜ!<ÀFÔ!KˆÔÜ*¨6Ó2ˆŒØ�‰ÕrK   c                 ó.   — | j                   j                  S rO   ©rš   Úword_embeddings©rT   s    rI   Úget_input_embeddingsz/EvollaSaProtProteinEncoder.get_input_embeddings»   s   € Ø�‰×.Ñ.Ð.rK   c                 ó&   — || j                   _        y rO   rŸ   ©rT   r1   s     rI   Úset_input_embeddingsz/EvollaSaProtProteinEncoder.set_input_embeddings¾   s   € Ø*/ˆ�‰Õ'rK   NÚ	input_idsr2   ra   c                 óÐ  — |j                  «       }|\  }}|j                  }|€t        j                  ||f|¬«      }| j	                  ||¬«      }t        | j                  ||¬«      }t        j                  ||¬«      j                  d«      }	| j                  ||	«      }
 | j                  |f||
dœ|¤Ž}|d   }t        ||j                  |j                  |j                  ¬«      S )N©r_   ©r¦   r2   )rU   Úinputs_embedsr2   r   )r2   Úposition_embeddings)Úlast_hidden_stater‚   rƒ   r„   )r>   r_   r@   Úonesrš   r
   rU   ÚarangeÚ	unsqueezer›   rœ   r   r‚   rƒ   r„   )rT   r¦   r2   r5   Úinput_shapeÚ
batch_sizeÚ
seq_lengthr_   rª   rR   r«   Úencoder_outputsÚsequence_outputs                rI   Úforwardz"EvollaSaProtProteinEncoder.forwardÁ   sö   € ð  —n‘nÓ&ˆØ!,Ñˆ
�Jà×!Ñ!ˆØÐ!Ü"ŸZ™Z¨*°jÐ)AÈ6ÔRˆNØŸ™°)ÈN˜Ó[ˆä2Ø—;‘;Ø'Ø)ô
ˆô —|‘| J°vÔ>×HÑHÈÓKˆØ"×4Ñ4°]ÀLÓQÐà&˜$Ÿ,™,Øð
Ø*8ÐNañ
Øekñ
ˆð *¨!Ñ,ˆä;Ø-Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ô	
ð 	
rK   rO   )rW   rX   rY   r-   rQ   r¢   r¥   r   r   r@   ÚTensorrg   r   rµ   rZ   r[   s   @rI   r–   r–   ³   sw   ø„ ð˜|õ ò/ò0ð  Øð /3ñ!
à—<‘< $Ñ&ð!
ð Ÿ™ tÑ+ð!
ð
 
ˆu�|‰|Ñ	ÐKÑ	Kò!
ó ó  ô!
rK   r–   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )Ú!EvollaSequenceCompressorAttentionc                 ój  •— t         ‰| �  «        |dz  | _        || _        ||z  }t	        j
                  |«      | _        t	        j
                  |«      | _        t	        j                  ||d¬«      | _	        t	        j                  ||dz  d¬«      | _
        t	        j                  ||d¬«      | _        y )Nr8   F©Úbiasr   )rP   rQ   ÚscaleÚheadsr   Ú	LayerNormÚ
norm_mediaÚnorm_latentsÚLinearÚto_qÚto_kvÚto_out)rT   r;   Údim_headr½   Ú	inner_dimrV   s        €rI   rQ   z*EvollaSequenceCompressorAttention.__init__è   s‹   ø€ Ü‰ÑÔØ˜t‘^ˆŒ
ØˆŒ
Ø˜uÑ$ˆ	äŸ,™, sÓ+ˆŒÜŸL™L¨Ó-ˆÔä—I‘I˜c 9°5Ô9ˆŒ	Ü—Y‘Y˜s I°¡M¸Ô>ˆŒ
Ü—i‘i 	¨3°UÔ;ˆ�rK   c                 óF  — | j                  |«      }| j                  |«      }| j                  }| j                  |«      }t	        j
                  ||fd¬«      }| j                  |«      j                  dd¬«      \  }}|j                  |j                  d«      |j                  d«      |d«      j                  dddd«      }|j                  |j                  d«      |j                  d«      |d«      j                  dddd«      }|j                  |j                  d«      |j                  d«      |d«      j                  dddd«      }|| j                  z  }t	        j                  ||j                  dd«      «      }	|	|	j                  dd¬	«      j                  «       z
  }	|	j                   \  }
}}}t	        j"                  ||«      j%                  |j&                  «      }|d
d
…d
d
d
d
…f   }|d
d
d
…d
d
…d
f   }||z  }|	j)                  d|z
  j+                  «       d«      }	|	j-                  d¬«      }t	        j                  ||«      }|j                  dddd«      }|j/                  |j                  d«      |j                  d«      d«      }| j1                  |«      S )zÔ
        Args:
            x (torch.Tensor): image features
                shape (b, n1, D)
            latent (torch.Tensor): latent features
                shape (b, n2, D);  n2: num of latent tokens
        éþÿÿÿr:   r   r7   r   r+   r   T©r;   ÚkeepdimNg     ˆÃÀ)r¿   rÀ   r½   rÂ   r@   ÚcatrÃ   ÚchunkÚviewr>   Úpermuter¼   rA   rB   ÚamaxÚdetachÚshaper­   Útor_   Úmasked_fillÚboolrD   ÚreshaperÄ   )rT   ÚxÚlatentsÚmaskÚhÚqÚkv_inputÚkÚvÚsimÚbsÚnhÚskdÚokdr­   Úmask_expÚones_expÚattnÚouts                      rI   rµ   z)EvollaSequenceCompressorAttention.forwardõ   sB  € ð �O‰O˜AÓˆØ×#Ñ# GÓ,ˆà�J‰Jˆà�I‰I�gÓˆÜ—9‘9˜a ˜\¨rÔ2ˆØ�z‰z˜(Ó#×)Ñ)Ø�2ð *ó 
‰ˆˆ1ð �F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ�F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ�F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ�—
‘
‰Nˆô �l‰l˜1˜aŸk™k¨"¨bÓ1Ó2ˆØ�C—H‘H ¨T�HÓ2×9Ñ9Ó;Ñ;ˆØŸ9™9ÑˆˆB��SÜ�z‰z˜"˜cÓ"×%Ñ% d§k¡kÓ2ˆØš˜4 ¢qÐ(Ñ)ˆØ˜ša¢ DÐ(Ñ)ˆØ˜(Ñ"ˆà�o‰o˜q 4™xŸo™oÓ/°Ó6ˆØ�{‰{˜rˆ{Ó"ˆÜ�l‰l˜4 Ó#ˆØ�k‰k˜!˜Q  1Ó%ˆð �k‰k˜#Ÿ(™( 1›+ s§x¡x°£{°BÓ7ˆà�{‰{˜3ÓÐrK   )é@   é   ©rW   rX   rY   rQ   rµ   rZ   r[   s   @rI   r¸   r¸   ç   s   ø„ õ<ö) rK   r¸   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚEvollaFeedForwardc                 ó  •— t         ‰| �  «        t        ||z  «      }t        j                  |«      | _        t        j                  ||d¬«      | _        t        j                  «       | _	        t        j                  ||d¬«      | _
        y ©NFrº   )rP   rQ   rf   r   r¾   ÚnormrÁ   Úfc1ÚGELUÚ
activationÚfc2)rT   r;   ÚmultrÆ   rV   s       €rI   rQ   zEvollaFeedForward.__init__"  s`   ø€ Ü‰ÑÔÜ˜˜d™
“Oˆ	ä—L‘L Ó%ˆŒ	Ü—9‘9˜S )°%Ô8ˆŒÜŸ'™'›)ˆŒÜ—9‘9˜Y¨°%Ô8ˆ�rK   c           	      ó~   — | j                  | j                  | j                  | j                  |«      «      «      «      S rO   )rò   rñ   rï   rî   )rT   rÖ   s     rI   rµ   zEvollaFeedForward.forward+  s+   € Ø�x‰x˜Ÿ™¨¯©°·±¸1³Ó(>Ó?Ó@Ð@rK   )é   ré   r[   s   @rI   rë   rë   !  s   ø„ õ9öArK   rë   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )Ú!EvollaSequenceCompressorResamplerrU   c           
      ó¨  •— t         ‰| �  «        |j                  j                  }|j                  | _        t        j                  t        j                  | j
                  |«      d¬«      | _
        t        j                  g «      | _        t        |j                  «      D ]g  }| j                  j                  t        j                  t!        ||j"                  |j$                  ¬«      t'        ||j(                  ¬«      g«      «       Œi t        j*                  |j                  «      | _        t        j.                  ||j                  «      | _        y )NT)Úrequires_grad)r;   rÅ   r½   )r;   ró   )rP   rQ   Úprotein_encoder_configÚhidden_sizeÚresampler_num_latentsÚnum_latentsr   Ú	Parameterr@   Úrandnr×   Ú
ModuleListÚlayersÚrangeÚresampler_depthÚappendr¸   Úresampler_dim_headÚresampler_headsrë   Úresampler_ff_multr¾   rî   rÁ   Úprotein_projector)rT   rU   Úprotein_repr_dimrŒ   rV   s       €rI   rQ   z*EvollaSequenceCompressorResampler.__init__0  s  ø€ Ü‰ÑÔØ!×8Ñ8×DÑDÐØ!×7Ñ7ˆÔÜ—|‘|¤E§K¡K°×0@Ñ0@ÐBRÓ$SÐcgÔhˆŒÜ—m‘m BÓ'ˆŒÜ�v×-Ñ-Ó.ò 
	ˆAØ�K‰K×ÑÜ—‘ä9Ø 0¸6×;TÑ;TÐ\b×\rÑ\rôô *Ð.>ÀV×E]ÑE]Ô^ð	óõ	ð
	ô —L‘L ×!3Ñ!3Ó4ˆŒ	Ü!#§¡Ð+;¸V×=OÑ=OÓ!PˆÕrK   c                 ój  — |j                   d   }|j                   \  }}t        j                  || j                  «      j	                  |j
                  «      }t        j                  ||fd¬«      }t        j                  |«      j	                  | j                  j
                  «      }| j                  d    |j                  ddd«      z  }|j	                  |j                  «      }| j                  D ]  \  }	}
 |	|||«      |z   } |
|«      |z   }Œ | j                  |«      }| j                  |«      S )Nr   r+   r:   r7   )rÑ   r@   r­   rý   rÒ   r_   rË   r×   rÍ   Údtyper  r  rî   )rT   ÚembedsrØ   Úbrß   rŒ   Úlatent_maskr­   r×   rå   ÚffÚtransformed_features               rI   rµ   z)EvollaSequenceCompressorResampler.forwardE  s  € Ø�L‰L˜‰Oˆà—
‘
‰ˆˆAÜ—j‘j  T×%5Ñ%5Ó6×9Ñ9¸$¿+¹+ÓFˆÜ�y‰y˜$ Ð,°!Ô4ˆô �z‰z˜!‹}×Ñ §¡× 3Ñ 3Ó4ˆØ—,‘,˜tÑ$ t§y¡y°°Q¸Ó':Ñ:ˆØ—*‘*˜VŸ\™\Ó*ˆØŸ™ò 	,‰HˆD�"Ù˜6 7¨DÓ1°GÑ;ˆGÙ˜“k GÑ+‰Gð	,ð #×4Ñ4°WÓ=Ðà�y‰yÐ,Ó-Ð-rK   )rW   rX   rY   r,   rQ   rµ   rZ   r[   s   @rI   r÷   r÷   /  s   ø„ ðQ˜|õ Qö*.rK   r÷   c                   óÆ   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)ÚEvollaProteinEncoderModelOutputaN  
    sequence_compressor_output (`torch.FloatTensor` of shape `(batch_size, compressed_seq_len, hidden_size)`, *optional*):
        Compressed sequence representation produced by the sequence compressor module. The sequence length is
        reduced from the original input length to `compressed_seq_len` via learned compression.
    NÚsequence_compressor_outputr¬   .r‚   rƒ   )rW   rX   rY   Ú__doc__r  r@   ÚFloatTensorr�   r¬   r‚   rg   rƒ   rl   rK   rI   r  r  Y  st   … ñð <@Ð × 1Ñ 1°DÑ 8Ó?Ø26Ð�u×(Ñ(¨4Ñ/Ó6Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;rK   r  c                   óf   ‡ — e Zd Zdefˆ fd„Zedej                  dej                  fd„«       Z	ˆ xZ
S )ÚEvollaProteinEncoderrU   c                 óz   •— t         ‰| �  «        t        |j                  ¬«      | _        t        |¬«      | _        y r˜   )rP   rQ   r–   rú   Úmodelr÷   Úsequence_compressor_resamplerrS   s     €rI   rQ   zEvollaProteinEncoder.__init__i  s.   ø€ Ü‰ÑÔÜ/°v×7TÑ7TÔUˆŒ
Ü-NÐV\Ô-]ˆÕ*rK   r¦   r2   c                 ó’   — | j                  ||¬«      }|j                  }| j                  ||«      }t        ||j                  ¬«      S )Nr©   )r  r¬   )r  r¬   r  r  )rT   r¦   r2   r5   Úprotein_outputÚprotein_embedsÚsequence_reprs          rI   rµ   zEvollaProteinEncoder.forwardn  sJ   € àŸ™¨iÈ˜ÓWˆØ'×9Ñ9ˆØ×:Ñ:¸>È>ÓZˆä.Ø'4Ø,×>Ñ>ô
ð 	
rK   )rW   rX   rY   r,   rQ   r   r@   Ú
LongTensorr  rµ   rZ   r[   s   @rI   r  r  h  s?   ø„ ð^˜|õ ^ð
 ð
 ×!1Ñ!1ð 
À5×CTÑCTò 
ó ô
rK   r  c                   ób   ‡ — e Zd Z	 	 	 ddedz  dedz  dedz  fˆ fd„Zd„ Z	 	 	 	 	 	 	 d	d„Zˆ xZS )
Ú#EvollaSequenceAlignerCrossAttentionNÚprotein_encoder_dimÚstructure_encoder_dimÚmsa_encoder_dimc                 ó˜  •— t         ‰| �  «        |j                  | _        |j                  | _        | j                  dz  | _        t        | j                  | j                  z  «      | _        | j                  | j                  z  | _        |j                  }|j                  }|j                  }t        j                  | j                  | j                  «      | _        |�Kt        j                  || j                  «      | _        t        j                  || j                  «      | _        nd | _        d | _        |�Kt        j                  || j                  «      | _        t        j                  || j                  «      | _        nd | _        d | _        |�Kt        j                  || j                  «      | _        t        j                  || j                  «      | _        nd | _        d | _        t)        | j                  «      | _        t        j,                  |«      | _        t        j                  | j                  | j                  |¬«      | _        t3        | j                  |«      | _        t        j6                  t9        j:                  dg«      «      | _        t        j6                  t9        j:                  dg«      «      | _        y )Nr8   rº   ç        ) rP   rQ   rû   Únum_attention_headsr¼   rf   Úattention_head_sizeÚall_head_sizeÚ$aligner_attention_probs_dropout_probÚaligner_enable_biasÚaligner_ffn_multr   rÁ   r/   Úkey_proteinÚvalue_proteinÚkey_structureÚvalue_structureÚkey_msaÚ	value_msaÚEvollaRMSNormÚattention_normÚDropoutr4   Úout_projrë   r  rþ   r@   ÚtensorÚgate_attentionÚgate_ffw)	rT   rU   r"  r#  r$  Úattention_probs_dropout_probÚenable_biasÚffn_multrV   s	           €rI   rQ   z,EvollaSequenceAlignerCrossAttention.__init__{  s  ø€ ô 	‰ÑÔà!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ Ø×-Ñ-¨tÑ3ˆŒ
Ü#& t×'7Ñ'7¸$×:RÑ:RÑ'RÓ#SˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔà'-×'RÑ'RÐ$Ø×0Ñ0ˆØ×*Ñ*ˆä—Y‘Y˜t×/Ñ/°×1CÑ1CÓDˆŒ
ØÐ*Ü!Ÿy™yÐ)<¸d×>PÑ>PÓQˆDÔÜ!#§¡Ð+>À×@RÑ@RÓ!SˆDÕà#ˆDÔØ!%ˆDÔà Ð,Ü!#§¡Ð+@À$×BTÑBTÓ!UˆDÔÜ#%§9¡9Ð-BÀD×DVÑDVÓ#WˆDÕ à!%ˆDÔØ#'ˆDÔ àÐ&ÜŸ9™9 _°d×6HÑ6HÓIˆDŒLÜŸY™Y ¸×8JÑ8JÓKˆD�NàˆDŒLØ!ˆDŒNä+¨D×,<Ñ,<Ó=ˆÔä—z‘zÐ">Ó?ˆŒäŸ	™	 $×"2Ñ"2°D×4DÑ4DÈ;ÔWˆŒä# D×$4Ñ$4°hÓ?ˆŒÜ Ÿl™l¬5¯<©<¸¸Ó+>Ó?ˆÔÜŸ™¤U§\¡\°3°%Ó%8Ó9ˆ�rK   c	                 óˆ  — |||g}	|	D �
cg c]  }
|
€Œ|
‘Œ	 }	}
|	st        d«      ‚t        j                  |	d¬«      }	| j                  |«      }| j	                  |«      }| j
                  �@| j                  �4|j                  |«      }| j                  |«      }| j                  |«      }nd}d}| j                  �@| j                  �4|j                  |«      }| j                  |«      }| j                  |«      }nd}d}| j                  �@| j                  �4|j                  |«      }| j                  |«      }| j                  |«      }nd}d}|||g}|D �
cg c]  }
|
€Œ|
‘Œ	 }}
t        j                  |d¬«      }|||g}|D �
cg c]  }
|
€Œ|
‘Œ	 }}
t        j                  |d¬«      }|j                  «       dd | j                  | j                  fz   } |j                  |Ž j!                  dddd«      }|j                  «       dd | j                  | j                  fz   } |j                  |Ž j!                  dddd«      }|j                  «       dd | j                  | j                  fz   } |j                  |Ž j!                  dddd«      }|| j"                  z  }|€Mt        j$                  |j                  d«      |j                  d«      «      j                  |j&                  «      }|dd…ddd…df   |	dd…dddd…f   z  }t        j(                  ||j+                  dd	«      «      }||j-                  dd
¬«      j/                  «       z
  }|j1                  d|z
  j3                  «       t        j4                  |j6                  «      j8                  «      } t;        j<                  d¬«      |«      }t        j(                  ||«      }|j!                  dddd«      j?                  «       }|j                  «       dd	 | j@                  fz   } |j                  |Ž }| jC                  |«      }|S c c}
w c c}
w c c}
w )zù
        query_states: text
        key_value_states: protein
        query_states: [bs, query_seq_len, dim]
        key_value_states: [bs, kv_seq_len, dim]
        query_attn_mask: [bs, query_seq_len]
        kv_attn_mask: [bs, kv_seq_len]
        Nz=At least one modality should be provided for cross attention.r+   r:   r7   r   r   r   rÈ   TrÉ   )"Ú
ValueErrorr@   rË   r4  r/   r-  r.  rÒ   r/  r0  r1  r2  r>   r'  r(  rÍ   rÎ   r¼   r­   r_   rA   rB   rÏ   rÐ   rÓ   rÔ   Úfinfor  Úminr   ÚSoftmaxrE   r)  r6  )rT   Úquery_statesÚprotein_key_value_statesÚstructure_key_value_statesÚmsa_key_value_statesÚquery_attn_maskÚprotein_kv_attn_maskÚstructure_kv_attn_maskÚmsa_kv_attn_maskÚkv_attn_maskrŒ   Úquery_layerÚkey_layer_proteinÚvalue_layer_proteinÚkey_layer_structureÚvalue_layer_structureÚkey_layer_msaÚvalue_layer_msaÚ	key_layerÚvalue_layerÚnew_query_layer_shapeÚnew_key_layer_shapeÚnew_value_layer_shaper2   rG   Úattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapes                                rI   Úcross_attentionz3EvollaSequenceAlignerCrossAttention.cross_attention®  si  € ð* -Ð.DÐFVÐWˆØ#/ÖA˜a°1±=šÐAˆÐAÙÜÐ\Ó]Ð]Ü—y‘y °1Ô5ˆà×)Ñ)¨,Ó7ˆð —j‘j Ó-ˆà×ÑÐ'¨D×,>Ñ,>Ð,JØ'?×'BÑ'BÀ<Ó'PÐ$Ø $× 0Ñ 0Ð1IÓ JÐØ"&×"4Ñ"4Ð5MÓ"NÑà $ÐØ"&Ðà×ÑÐ)¨d×.BÑ.BÐ.NØ)C×)FÑ)FÀ|Ó)TÐ&Ø"&×"4Ñ"4Ð5OÓ"PÐØ$(×$8Ñ$8Ð9SÓ$TÑ!à"&ÐØ$(Ð!à�<‰<Ð#¨¯©Ð(BØ#7×#:Ñ#:¸<Ó#HÐ Ø ŸL™LÐ)=Ó>ˆMØ"Ÿn™nÐ-AÓB‰Oà ˆMØ"ˆOà&Ð(;¸]ÐKˆ	Ø )Ö;˜1¨Q©]’QÐ;ˆ	Ð;Ü—I‘I˜i¨QÔ/ˆ	à*Ð,AÀ?ÐSˆØ"-Ö?˜Q°±’qÐ?ˆÐ?Ü—i‘i °Ô3ˆà +× 0Ñ 0Ó 2°3°BÐ 7Ø×$Ñ$Ø×$Ñ$ð;
ñ !
Ðð '�k×&Ñ&Ð(=Ð>×FÑFÀqÈ!ÈQÐPQÓRˆà'Ÿn™nÓ.¨s°Ð3Ø×$Ñ$Ø×$Ñ$ð7
ñ 
Ðð #�I—N‘NÐ$7Ð8×@Ñ@ÀÀAÀqÈ!ÓLˆ	à +× 0Ñ 0Ó 2°3°BÐ 7Ø×$Ñ$Ø×$Ñ$ð;
ñ !
Ðð '�k×&Ñ&Ð(=Ð>×FÑFÀqÈ!ÈQÐPQÓRˆà! D§J¡JÑ.ˆð Ð"Ü#Ÿj™j¨×):Ñ):¸1Ó)=¸|×?PÑ?PÐQRÓ?SÓT×WÑWÐXd×XkÑXkÓlˆOØ(ª¨D²!°TÐ)9Ñ:¸\Ê!ÈTÐSWÒYZÐJZÑ=[Ñ[ˆä—|‘| K°×1DÑ1DÀRÈÓ1LÓMˆØ# l×&7Ñ&7¸BÈÐ&7Ó&M×&TÑ&TÓ&VÑVˆØ'×3Ñ3Ø�Ñ×%Ñ%Ó'¬¯©°\×5GÑ5GÓ)H×)LÑ)Ló
Ðð -œ"Ÿ*™*¨Ô,Ð-=Ó>ˆô Ÿ™ _°kÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ*˜×*Ñ*Ð,CÐDˆàŸ™ mÓ4ˆàÐùòq BùòL <ùò @s"   ŠP5’P5ÅP:ÅP:ÆP?ÆP?c           
      ó^  — |�z|j                   \  }}}|€jt        j                  ||«      j                  |	j                  «      |	j                  ||f¬«      j                  z  j                  |j                  «      }nd }|�z|j                   \  }}}|€jt        j                  ||«      j                  |	j                  «      |
j                  ||f¬«      j                  z  j                  |j                  «      }nd }|�z|j                   \  }}}|€jt        j                  ||«      j                  |	j                  «      |j                  ||f¬«      j                  z  j                  |j                  «      }nd }|}|�|j                  «       s$|�|j                  «       s|�Š|j                  «       rz|}| j                  ||||||||¬«      }t        j                  | j                  «      |z  }||z   }|}| j                  |«      t        j                  | j                  «      z  }||z   }|S )N)r>   )rB  rC  rD  rE  rF  rG  rH  rI  )rÑ   r@   r­   rÒ   r_   ÚexpandÚTÚanyr[  Útanhr8  r  r9  )rT   rB  Úprotein_kv_statesÚstructure_kv_statesÚmsa_kv_statesrF  rG  rH  rI  Úprotein_batch_maskÚstructure_batch_maskÚmsa_batch_maskÚpast_key_valuesrß   Úprotein_kv_seq_lenr;   Ústructure_kv_seq_lenÚmsa_kv_seq_lenr‚   Úresiduals                       rI   rµ   z+EvollaSequenceAlignerCrossAttention.forward  sL  € ð Ð(Ø*;×*AÑ*AÑ'ˆBÐ" CØ#Ð+ä—J‘J˜rÐ#5Ó6×9Ñ9Ð:L×:SÑ:SÓTØ(×/Ñ/Ð6HÈ"Ð5MÐ/ÓN×PÑPñQç‘"Ð&×-Ñ-Ó.ñ %ð
 $(Ð àÐ*Ø,?×,EÑ,EÑ)ˆBÐ$ cØ%Ð-ä—J‘J˜rÐ#7Ó8×;Ñ;Ð<N×<UÑ<UÓVØ*×1Ñ1Ð8LÈbÐ7QÐ1ÓR×TÑTñUç‘"Ð(×/Ñ/Ó0ñ 'ð
 &*Ð"àÐ$Ø&3×&9Ñ&9Ñ#ˆB� ØÐ'ä—J‘J˜r >Ó2×5Ñ5Ð6H×6OÑ6OÓPØ$×+Ñ+°.À"Ð1EÐ+ÓF×HÑHñIç‘"�]×)Ñ)Ó*ñ !ð
  $ÐØ$ˆð Ð*Ð/C×/GÑ/GÔ/IØ#Ð/Ð4J×4NÑ4NÔ4PØÐ)Ð.>×.BÑ.BÔ.Dà$ˆHØ ×0Ñ0Ø*Ø):Ø+>Ø%2Ø /Ø%9Ø'=Ø!1ð 1ó 	ˆMô "ŸJ™J t×':Ñ':Ó;¸mÑKˆMà$ }Ñ4ˆMà$ˆHØ ŸG™G MÓ2´U·Z±ZÀÇÁÓ5NÑNˆMØ$ }Ñ4ˆMàÐrK   rd   )NNNNNNN)rW   rX   rY   rf   rQ   r[  rµ   rZ   r[   s   @rI   r!  r!  z  sb   ø„ ð +/Ø,0Ø&*ñ1:ð ! 4™Zð1:ð  # T™zð	1:ð
 ˜t™õ1:òfnðn "Ø#ØØØ!ØØ÷GrK   r!  c                   ó   — e Zd Zy)r3  Nrk   rl   rK   rI   r3  r3  h  rm   rK   r3  c                   ó   — e Zd Zy)ÚEvollaRotaryEmbeddingNrk   rl   rK   rI   rn  rn  l  rm   rK   rn  c                   ó   — e Zd Zy)Ú	EvollaMLPNrk   rl   rK   rI   rp  rp  p  rm   rK   rp  c                   ó   — e Zd Zy)ÚEvollaAttentionNrk   rl   rK   rI   rr  rr  t  rm   rK   rr  c                   óÀ  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  fd„Zˆ xZS )ÚEvollaDecoderLayerrU   Ú	layer_idxc                 ó¸   •— t         ‰| �  ||«       |dz   t        |j                  |j                  z  d«      z  dk(  rt        ||j                  ¬«      | _        y y )Nr+   r   )r"  )rP   rQ   ÚmaxÚnum_hidden_layersÚaligner_num_add_layersr!  rû   Úadapter©rT   rU   ru  rV   s      €rI   rQ   zEvollaDecoderLayer.__init__y  s[   ø€ Ü‰Ñ˜ Ô+Ø˜‰MœS ×!9Ñ!9¸V×=ZÑ=ZÑ!ZÐ\]Ó^Ñ^ÐbcÒcÜ>ØØ$*×$6Ñ$6ôˆD�Lð drK   Nr‚   r«   r2   rR   rg  Ú	use_cachera  rb  rc  rd  re  rf  rF  c           
      ó
  — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}||z   }|}| j                  |«      }| j                  |«      }||z   }t	        | d«      r| j                  ||||	||
||¬«      }|S )N)r‚   r2   rR   rg  r|  r«   rz  )rB  ra  rb  rc  rF  rd  re  rf  rl   )Úinput_layernormÚ	self_attnÚpost_attention_layernormÚmlpÚhasattrrz  )rT   r‚   r«   r2   rR   rg  r|  ra  rb  rc  rd  re  rf  rF  r5   rk  rŒ   s                    rI   rµ   zEvollaDecoderLayer.forward�  sÉ   € ð" !ˆà×,Ñ,¨]Ó;ˆð *˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ  =Ñ0ˆä�4˜Ô#Ø ŸL™LØ*Ø"3Ø$7Ø+Ø /Ø#5Ø%9Ø-ð )ó 	ˆMð ÐrK   )NNNNFNNNNNNN)rW   rX   rY   r,   rf   rQ   r@   r¶   rg   r  r   rÔ   rµ   rZ   r[   s   @rI   rt  rt  x  s9  ø„ ð˜|ð ¸õ ð IMØ.2Ø04Ø(,Ø!&Ø15Ø37Ø-1Ø26Ø48Ø.2Ø/3ñ3à—|‘|ð3ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð3ð Ÿ™ tÑ+ð	3ð
 ×&Ñ&¨Ñ-ð3ð  ™ð3ð ˜$‘;ð3ð !Ÿ<™<¨$Ñ.ð3ð #Ÿ\™\¨DÑ0ð3ð —|‘| dÑ*ð3ð "ŸL™L¨4Ñ/ð3ð $Ÿl™l¨TÑ1ð3ð Ÿ™ tÑ+ð3ð Ÿ™¨Ñ,÷3rK   rt  c                   óN   — e Zd ZdZdZdZg d¢Z ej                  «       d„ «       Z	y)ÚEvollaPreTrainedModelF)rt  rw   r÷   r!  c                 ó²  — | j                   j                  }t        j                  | |«       t	        |t
        «      rht        j                  |j                  «       t        j                  |j                  «       t        j                  |j                  j                  «       y t	        |t        «      r#t        j                  |j                  d|¬«       y y )Nr&  )ÚmeanÚstd)rU   Úinitializer_ranger   r‡   rˆ   r!  r‰   Úzeros_r8  r9  Úones_r4  Úweightr÷   Únormal_r×   )rT   r.   r‡  s      rI   r‡   z#EvollaPreTrainedModel._init_weightsÂ  s‹   € à�k‰k×+Ñ+ˆÜ×%Ñ% d¨FÔ3Ü�fÔAÔBÜ�K‰K˜×-Ñ-Ô.Ü�K‰K˜Ÿ™Ô(Ü�J‰J�v×,Ñ,×3Ñ3Õ4Ü˜Ô AÔBÜ�L‰L˜Ÿ™¨c°sÖ;ð CrK   N)
rW   rX   rY   r�   r‘   r’   rŽ   r@   r”   r‡   rl   rK   rI   r„  r„  ·  s8   „ Ø ÐØÐØ"'ÐòÐð €U‡]�]ƒ_ñ<ó ñ<rK   r„  c                   óº  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zeee		 	 	 	 	 	 	 	 	 	 	 	 dde
j                  dz  de
j                  dz  de
j                  dz  d	edz  d
e
j                  dz  dedz  de
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  deez  fd„«       «       «       Zˆ xZS )ÚEvollaModelrU   c           	      óF  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  | j                  |j                  | j                  «      | _        t        |¬«      | _
        t        j                  t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}«      | _        t!        |j                  |j"                  ¬«      | _        t'        |dd«      | _        t+        |¬«      | _        | j/                  «        y c c}w )Nr™   )rU   ru  )ÚepsÚgradient_checkpointingF)rP   rQ   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingrû   Úembed_tokensr  Úprotein_encoderr   r  rx  rt  r  r3  Úrms_norm_epsrî   r?   r‘  rn  Ú
rotary_embr�   r{  s      €rI   rQ   zEvollaModel.__init__Ï  sç   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒÜŸL™L¨¯©¸&×:LÑ:LÈd×N^ÑN^Ó_ˆÔÜ3¸6ÔBˆÔÜ—m‘mô "' v×'?Ñ'?Ó!@öð
 ô	 #Ø!Ø'öòó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü&-¨fÐ6NÐPUÓ&VˆÔ#Ü/°vÔ>ˆŒØ�‰Õùòs   Â$Dc                 ó   — | j                   S rO   ©r–  r¡   s    rI   r¢   z EvollaModel.get_input_embeddingsä  s   € Ø× Ñ Ð rK   c                 ó   — || _         y rO   r›  r¤   s     rI   r¥   z EvollaModel.set_input_embeddingsç  s
   € Ø!ˆÕrK   Nr¦   r2   rR   rg  rª   r|  Úprotein_input_idsÚprotein_attention_maskÚstructure_featsÚ	msa_featsre  rf  ra   c                 óæ  — |du |duz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }d}d}|�^|�\| j                  ||¬«      }|j                  }t        j                  |j                  d   |j                  t
        j                  ¬«      }t        | j                  |||¬	«      }|}| j                  ||¬
«      }| j                   D ]  } ||f||||||	|
|||||dœ|¤Ž}Œ | j#                  |«      }t%        ||¬«      }|S )a;  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
        structure_feats (torch.FloatTensor):
            The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        msa_feats (torch.FloatTensor):
            The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        structure_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummpy input for now.
        msa_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummpy input for now.
        Nz:You must specify exactly one of input_ids or inputs_embedsr™   r   r+   r¨   r©   )r_   r  )rU   rª   r2   rg  )rR   )r2   rR   rg  r|  ra  rb  rc  rd  re  rf  rF  r«   )r¬   rg  )r>  r–  r   rU   Úget_seq_lengthr@   r®   rÑ   r_   r¯   r—  r  r­   rÔ   r   r™  r  rî   r   )rT   r¦   r2   rR   rg  rª   r|  r�  rž  rŸ  r   re  rf  r5   Úpast_seen_tokensÚprotein_featsrd  Úprotein_outputsÚcausal_maskr‚   r«   Údecoder_layerÚoutputs                          rI   rµ   zEvollaModel.forwardê  s¿  € ðB ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLàˆØ!ÐàÐ(Ð-CÐ-OØ"×2Ñ2Ø+Ø5ð 3ó ˆOð ,×FÑFˆMÜ!&§¡Ø!×'Ñ'¨Ñ*Ø(×/Ñ/Ü—j‘jô"Ðô )Ø—;‘;Ø'Ø)Ø+ô	
ˆð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[ò 	ˆMÙ)Øðà*Ø)Ø /Ø#Ø"/Ø$3Ø'Ø#5Ø%9Ø-Ø .Ø$7ñð ñ‰Mð	ð$ Ÿ	™	 -Ó0ˆä(Ø+Ø+ô
ˆð ˆrK   )NNNNNNNNNNNN)rW   rX   rY   r,   rQ   r¢   r¥   r   r   r   r@   r  r¶   r   r  rÔ   rg   r   rµ   rZ   r[   s   @rI   rŽ  rŽ  Î  sf  ø„ ð˜|õ ò*!ò"ð ØØð .2Ø.2Ø04Ø(,Ø26Ø!%Ø59Ø6:Ø48Ø.2Ø48Ø.2ñ]à×#Ñ# dÑ*ð]ð Ÿ™ tÑ+ð]ð ×&Ñ&¨Ñ-ð	]ð
  ™ð]ð ×(Ñ(¨4Ñ/ð]ð ˜$‘;ð]ð !×+Ñ+¨dÑ2ð]ð !&§¡¨tÑ 3ð]ð ×*Ñ*¨TÑ1ð]ð ×$Ñ$ tÑ+ð]ð $Ÿl™l¨TÑ1ð]ð Ÿ™ tÑ+ð]ð 
Ð(Ñ	(ò]ó ó  ó ô]rK   rŽ  c                   ó4  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dedz  deej                  z  fd„«       «       Zˆ xZS )ÚEvollaForProteinText2Textc                 óè   •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  | j                  d¬«      | _        | j                  «        y rí   )
rP   rQ   rŽ  r  r”  r   rÁ   rû   Úlm_headr�   rS   s     €rI   rQ   z"EvollaForProteinText2Text.__init__N  sQ   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°T·_±_È5ÔQˆŒà�‰ÕrK   c                 ó6   — | j                   j                  «       S rO   )r  r¢   r¡   s    rI   r¢   z.EvollaForProteinText2Text.get_input_embeddingsV  s   € Ø�z‰z×.Ñ.Ó0Ð0rK   c                 ó8   — | j                   j                  |«      S rO   )r  r¥   r¤   s     rI   r¥   z.EvollaForProteinText2Text.set_input_embeddingsY  s   € Ø�z‰z×.Ñ.¨uÓ5Ð5rK   Nr¦   r2   rª   Úlabelsr�  rž  r|  Úlogits_to_keepc	           
      óh  —  | j                   d||||||dœ|	¤Ž}
|
j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      }d}|�  | j                  d||| j                  dœ|	¤Ž}t        |||
j                  |
j                  |
j                  ¬«      }|S )a,  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

        Example:

        ```python
        >>> from transformers import EvollaProcessor, EvollaForProteinText2Text
        >>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
        >>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

        >>> protein_information = {
            "aa_seq": "your amino acid sequence",
            "foldseek": "your foldseek sequence",
        }
        >>> question = "What is the function of this protein?"
        >>> message = [
            {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
            {"role": "user", "content": question},
        ]

        >>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
        >>> outputs = model.generate(**inputs)

        >>> print(processor.batch_decode(outputs, skip_special_tokens=True))
        ```)r¦   r2   rª   r�  rž  r|  N)Úlogitsr¯  r”  )Úlossr²  rg  r‚   rƒ   rl   )r  r¬   rˆ   rf   Úslicer¬  Úloss_functionr”  r   rg  r‚   rƒ   )rT   r¦   r2   rª   r¯  r�  rž  r|  r°  r5   Úoutputsr‚   Úslice_indicesr²  r³  Ú
lm_outputss                   rI   rµ   z!EvollaForProteinText2Text.forward\  sÙ   € ðT ,6¨4¯:©:ð ,
ØØ)Ø'Ø/Ø#9Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%Ði¨V¸FÈtÏÉÑiÐbhÑiˆDä+ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ˆ
ð ÐrK   )NNNNNNNr   )rW   rX   rY   rQ   r¢   r¥   r   r   r@   r  r¶   r  rÔ   rf   rµ   rZ   r[   s   @rI   rª  rª  M  së   ø„ ôò1ò6ð Øð .2Ø.2Ø26Ø*.Ø59Ø6:Ø!%Ø-.ñBà×#Ñ# dÑ*ðBð Ÿ™ tÑ+ðBð ×(Ñ(¨4Ñ/ð	Bð
 × Ñ  4Ñ'ðBð !×+Ñ+¨dÑ2ðBð !&§¡¨tÑ 3ðBð ˜$‘;ðBð ˜eŸl™lÑ*òBó ó ôBrK   rª  )rª  rŽ  r„  )Nr&  )ZÚdataclassesr   r@   r   Ú r   r‰   Úcache_utilsr   r   Ú
generationr	   Úmasking_utilsr
   r   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úesm.modeling_esmr   r   r   r   r   r   r    r!   r"   r#   Úllama.modeling_llamar$   r%   r&   r'   r(   r)   r*   Úconfiguration_evollar,   r-   Ú
get_loggerrW   ÚloggerÚModuler¶   rh   rJ   rM   r]   rj   ro   rq   rs   ru   rw   ry   r{   r}   r–   r¸   rë   r÷   r  r  r!  r3  rn  rp  rr  rt  r„  rŽ  rª  Ú__all__rl   rK   rI   ú<module>rË     sŒ  ðõ "ã Ý å &ß .Ý )ß J÷ó õ .Ý &÷ó õ 8ß E÷÷ ÷ ÷÷ ñ ÷ =ð 
ˆ×	Ñ	˜HÓ	%€ð !Øñ %Ø�I‰Ið %à�<‰<ð %ð 
�‰ð %ð �<‰<ð	 %ð
 —L‘L 4Ñ'ð %ð �T‰\ð %ð ð %ð Ð'Ñ(ó %ôF!˜]ô !ô
PÐ"4ô 
Pô	Ð 0ô 	ô	˜]ô 	ô	˜Lô 	ô	˜ô 	ô	˜ô 	ô	˜ô 	ô	˜*ô 	ô	˜ô 	ð ôC /ó Có ðCô01
Ð!<ô 1
ôh7 ¨¯	©	ô 7 ôtA˜Ÿ	™	ô Aô'.¨¯	©	ô '.ðT Ø
ô
< kó 
<ó ó ð
<ô
˜2Ÿ9™9ô 
ô$k¨"¯)©)ô kô\	�Lô 	ô	Ð0ô 	ô	�ô 	ô	�nô 	ô<Ð*ô <ô~<Ð0ô <ô.|Ð'ô |ô~SÐ 5°ô Sòl P�rK   