
    ^js                        d dl mZ d dlZd dlmZ ddlmZ ddlmZm	Z	 ddl
mZ ddlmZmZ dd	lmZmZmZmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZ ddlm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7  ejp                  e9      Z:	 	 dRdejv                  dejx                  dejx                  dejx                  dejx                  dz  de=dz  de=dee   fdZ> G d de$      Z? G d d e*      Z@ G d! d"e+      ZA G d# d$e,      ZB G d% d&e#      ZC G d' d(e&      ZD G d) d*e(      ZE G d+ d,e'      ZF G d- d.e%      ZG G d/ d0e)      ZHe G d1 d2e             ZI G d3 d4eI      ZJ G d5 d6ejv                        ZK G d7 d8ejv                        ZL G d9 d:ejv                        ZMee G d; d<e                    ZN G d= d>ejv                        ZO G d? d@ejv                        ZP G dA dBe2      ZQ G dC dDe3      ZR G dE dFe0      ZS G dG dHe.      ZT G dI dJe/      ZU G dK dLe1      ZV G dM dNeV      ZW G dO dPeVe      ZXg dQZYy)S    )	dataclassN)nn   )initialization)CacheDynamicCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)BaseModelOutputWithPast,BaseModelOutputWithPoolingAndCrossAttentionsCausalLMOutputWithPastModelOutput)PreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)merge_with_config_defaults)OutputRecordercapture_outputs   )
EsmAttentionEsmEmbeddings
EsmEncoderEsmIntermediateEsmLayer	EsmOutput	EsmPoolerEsmRotaryEmbeddingEsmSelfAttentionEsmSelfOutput)LlamaAttentionLlamaDecoderLayerLlamaMLPLlamaPreTrainedModelLlamaRMSNormLlamaRotaryEmbedding	repeat_kv   )EvollaConfigSaProtConfigmodulequerykeyvalueattention_maskscalingdropoutkwargsc                    ||j                  d      dz  }t        | dd      }t        ||      }t        ||      }t        j                  ||j                  dd            |z  }	||	|z   }	t        j                  j                  |	d      }	t        j                  j                  |	|| j                        }	t        j                  |	|      }
|
j                  dd      j                         }
|
|	fS )	N      num_key_value_groupsr+   r   r   dim)ptraining)sizegetattrr*   torchmatmul	transposer   
functionalsoftmaxr4   r=   
contiguous)r.   r/   r0   r1   r2   r3   r4   r5   n_repattn_weightsattn_outputs              t/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/evolla/modular_evolla.pyeager_attention_forwardrJ   C   s     **R.D(
 F2A6E
C
CeU#E <<s}}Q':;gEL!#n4==((2(>L==((6??([L,,|U3K''1-88:K$$    c                        e Zd Z fdZ xZS )EvollaSaProtEmbeddingsc                 2    t         |   |       d | _        y N)super__init__position_idsselfconfig	__class__s     rI   rQ   zEvollaSaProtEmbeddings.__init__g   s      rK   )__name__
__module____qualname__rQ   __classcell__rV   s   @rI   rM   rM   f   s    ! !rK   rM   c                   j     e Zd Zd
def fdZe	 	 	 ddedz  dddedz  dedef   f fd	       Z	 xZ
S )EvollaSaProtRotaryEmbeddingNrU   c                 &    t         |   ||       y rO   )rP   rQ   )rT   rU   devicerV   s      rI   rQ   z$EvollaSaProtRotaryEmbedding.__init__n   s    (rK   r_   ztorch.device | Noneseq_lenreturnztorch.Tensorc                 &    t         |   | ||      S rO   )rP   compute_default_rope_parameters)rU   r_   r`   rV   s      rI   rc   z;EvollaSaProtRotaryEmbedding.compute_default_rope_parametersq   s     w6vvwOOrK   rO   NNN)rW   rX   rY   r-   rQ   staticmethodinttuplefloatrc   rZ   r[   s   @rI   r]   r]   m   si    )| ) &*(,"Pt#P%P tP 
~u$	%	P PrK   r]   c                       e Zd Zy)EvollaSaProtSelfAttentionNrW   rX   rY    rK   rI   rj   rj   z       rK   rj   c                       e Zd Zy)EvollaSaProtSelfOutputNrk   rl   rK   rI   ro   ro   ~   rm   rK   ro   c                       e Zd Zy)EvollaSaProtAttentionNrk   rl   rK   rI   rq   rq      rm   rK   rq   c                       e Zd Zy)EvollaSaProtIntermediateNrk   rl   rK   rI   rs   rs      rm   rK   rs   c                       e Zd Zy)EvollaSaProtOutputNrk   rl   rK   rI   ru   ru      rm   rK   ru   c                       e Zd Zy)EvollaSaProtLayerNrk   rl   rK   rI   rw   rw      rm   rK   rw   c                       e Zd Zy)EvollaSaProtEncoderNrk   rl   rK   rI   ry   ry      rm   rK   ry   c                       e Zd Zy)EvollaSaProtPoolerNrk   rl   rK   rI   r{   r{      rm   rK   r{   c                        e Zd ZU eed<   dgZdZdZdZdZ	e
 eedd      g eedd      gdZ ej                          fd	       Z xZS )
EvollaSaProtPreTrainedModelrU   rw   Tr+   	attention)index
layer_namecrossattention)hidden_states
attentionscross_attentionsc                     t         |   |       t        |t              r?|j	                  |j
                        \  }}t        j                  t        |d      |       y y )Ninv_freq)	rP   _init_weights
isinstancer]   rc   rU   initcopy_r?   )rT   r.   curr_inv_freq_rV   s       rI   r   z)EvollaSaProtPreTrainedModel._init_weights   sN    f%f9:%EEfmmTM1JJwvz2MB ;rK   )rW   rX   rY   r-   __annotations___no_split_modules_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrw   r   rj   _can_record_outputsr@   no_gradr   rZ   r[   s   @rI   r}   r}      sx    ,-N"& +%&?qU`ab4AJZ[
 U]]_C CrK   r}   c            
            e Zd Zdef fdZd Zd Zee	 d
de	j                  dz  de	j                  dz  dee	j                     ez  fd	              Z xZS )EvollaSaProtProteinEncoderrU   c                     t         |   |       t        |      | _        t	        |      | _        t        |      | _        | j                          y NrU   )	rP   rQ   rM   
embeddingsr]   rotary_embeddingsry   encoder	post_initrS   s     rI   rQ   z#EvollaSaProtProteinEncoder.__init__   s?     08!<F!K*62rK   c                 .    | j                   j                  S rO   r   word_embeddingsrT   s    rI   get_input_embeddingsz/EvollaSaProtProteinEncoder.get_input_embeddings   s    ...rK   c                 &    || j                   _        y rO   r   rT   r1   s     rI   set_input_embeddingsz/EvollaSaProtProteinEncoder.set_input_embeddings   s    */'rK   N	input_idsr2   ra   c                    |j                         }|\  }}|j                  }|t        j                  ||f|      }| j	                  ||      }t        | j                  ||      }t        j                  ||      j                  d      }	| j                  ||	      }
 | j                  |f||
d|}|d   }t        ||j                  |j                  |j                        S )Nr_   r   r2   )rU   inputs_embedsr2   r   )r2   position_embeddings)last_hidden_stater   r   r   )r>   r_   r@   onesr   r
   rU   arange	unsqueezer   r   r   r   r   r   )rT   r   r2   r5   input_shape
batch_size
seq_lengthr_   r   rR   r   encoder_outputssequence_outputs                rI   forwardz"EvollaSaProtProteinEncoder.forward   s      nn&!,
J!!!"ZZ*j)A6RN)N[2;;')
 ||Jv>HHK"44]LQ&$,,
*8Na
ek
 *!,;-)77&11,==	
 	
rK   rO   )rW   rX   rY   r-   rQ   r   r   r   r   r@   Tensorrg   r   r   rZ   r[   s   @rI   r   r      sw    | /0   /3!
<<$&!
 t+!

 
u||	K	K!
   !
rK   r   c                   &     e Zd Zd fd	Zd Z xZS )!EvollaSequenceCompressorAttentionc                 j   t         |           |dz  | _        || _        ||z  }t	        j
                  |      | _        t	        j
                  |      | _        t	        j                  ||d      | _	        t	        j                  ||dz  d      | _
        t	        j                  ||d      | _        y )Nr8   Fbiasr   )rP   rQ   scaleheadsr   	LayerNorm
norm_medianorm_latentsLinearto_qto_kvto_out)rT   r;   dim_headr   	inner_dimrV   s        rI   rQ   z*EvollaSequenceCompressorAttention.__init__   s    t^

u$	,,s+LL-IIc959	YYsIM>
ii	3U;rK   c                 F   | j                  |      }| j                  |      }| j                  }| j                  |      }t	        j
                  ||fd      }| j                  |      j                  dd      \  }}|j                  |j                  d      |j                  d      |d      j                  dddd      }|j                  |j                  d      |j                  d      |d      j                  dddd      }|j                  |j                  d      |j                  d      |d      j                  dddd      }|| j                  z  }t	        j                  ||j                  dd            }	|	|	j                  dd	      j                         z
  }	|	j                   \  }
}}}t	        j"                  ||      j%                  |j&                        }|d
d
d
d
d
d
f   }|d
d
d
d
d
d
f   }||z  }|	j)                  d|z
  j+                         d      }	|	j-                  d      }t	        j                  ||      }|j                  dddd      }|j/                  |j                  d      |j                  d      d      }| j1                  |      S )z
        Args:
            x (torch.Tensor): image features
                shape (b, n1, D)
            latent (torch.Tensor): latent features
                shape (b, n2, D);  n2: num of latent tokens
        r:   r   r7   r   r+   r   Tr;   keepdimNg     )r   r   r   r   r@   catr   chunkviewr>   permuter   rA   rB   amaxdetachshaper   tor_   masked_fillboolrD   reshaper   )rT   xlatentsmaskhqkv_inputkvsimbsnhskdokdr   mask_expones_expattnouts                      rI   r   z)EvollaSequenceCompressorAttention.forward   sB    OOA##G,JJIIg99a\r2zz(#))2 * 
1 FF166!9affQiB/771aCFF166!9affQiB/771aCFF166!9affQiB/771aC

N ll1akk"b12CHHTH299;;99BSzz"c"%%dkk24q()aD()("ooq4xoo/6{{r{"ll4#kk!Q1% kk#((1+sxx{B7{{3rK   )@      rW   rX   rY   rQ   r   rZ   r[   s   @rI   r   r      s    <) rK   r   c                   &     e Zd Zd fd	Zd Z xZS )EvollaFeedForwardc                    t         |           t        ||z        }t        j                  |      | _        t        j                  ||d      | _        t        j                         | _	        t        j                  ||d      | _
        y NFr   )rP   rQ   rf   r   r   normr   fc1GELU
activationfc2)rT   r;   multr   rV   s       rI   rQ   zEvollaFeedForward.__init__"  s`    d
O	LL%	99S)%8'')99Y%8rK   c           	      ~    | j                  | j                  | j                  | j                  |                        S rO   )r   r   r   r   )rT   r   s     rI   r   zEvollaFeedForward.forward+  s+    xx1(>?@@rK   )   r   r[   s   @rI   r   r   !  s    9ArK   r   c                   *     e Zd Zdef fdZd Z xZS )!EvollaSequenceCompressorResamplerrU   c           
         t         |           |j                  j                  }|j                  | _        t        j                  t        j                  | j
                  |      d      | _
        t        j                  g       | _        t        |j                        D ]g  }| j                  j                  t        j                  t!        ||j"                  |j$                        t'        ||j(                        g             i t        j*                  |j                        | _        t        j.                  ||j                        | _        y )NT)requires_grad)r;   r   r   )r;   r   )rP   rQ   protein_encoder_confighidden_sizeresampler_num_latentsnum_latentsr   	Parameterr@   randnr   
ModuleListlayersrangeresampler_depthappendr   resampler_dim_headresampler_headsr   resampler_ff_multr   r   r   protein_projector)rT   rU   protein_repr_dimr   rV   s       rI   rQ   z*EvollaSequenceCompressorResampler.__init__0  s   !88DD!77||EKK0@0@BR$ScghmmB'v--. 
	AKK9 06;T;T\b\r\r *.>VE]E]^		
	 LL!3!34	!#+;V=O=O!PrK   c                 j   |j                   d   }|j                   \  }}t        j                  || j                        j	                  |j
                        }t        j                  ||fd      }t        j                  |      j	                  | j                  j
                        }| j                  d    |j                  ddd      z  }|j	                  |j                        }| j                  D ]  \  }	}
 |	|||      |z   } |
|      |z   } | j                  |      }| j                  |      S )Nr   r+   r:   r7   )r   r@   r   r   r   r_   r   r   r   dtyper  r  r   )rT   embedsr   br   r   latent_maskr   r   r   fftransformed_features               rI   r   z)EvollaSequenceCompressorResampler.forwardE  s   LLO

AjjT%5%5699$++Fyy$,!4 zz!} 3 34,,t$tyyQ'::**V\\* 	,HD"67D1G;GkG+G	, #44W=yy,--rK   )rW   rX   rY   r,   rQ   r   rZ   r[   s   @rI   r   r   /  s    Q| Q*.rK   r   c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)EvollaProteinEncoderModelOutputaN  
    sequence_compressor_output (`torch.FloatTensor` of shape `(batch_size, compressed_seq_len, hidden_size)`, *optional*):
        Compressed sequence representation produced by the sequence compressor module. The sequence length is
        reduced from the original input length to `compressed_seq_len` via learned compression.
    Nsequence_compressor_outputr   .r   r   )rW   rX   rY   __doc__r  r@   FloatTensorr   r   r   rg   r   rl   rK   rI   r  r  Y  st     <@ 1 1D 8?26u((4/6:>M5**C/047>7;Je'',-4;rK   r  c                   f     e Zd Zdef fdZedej                  dej                  fd       Z	 xZ
S )EvollaProteinEncoderrU   c                 z    t         |           t        |j                        | _        t        |      | _        y r   )rP   rQ   r   r   modelr   sequence_compressor_resamplerrS   s     rI   rQ   zEvollaProteinEncoder.__init__i  s.    /v7T7TU
-NV\-]*rK   r   r2   c                     | j                  ||      }|j                  }| j                  ||      }t        ||j                        S )Nr   )r  r   )r  r   r  r  )rT   r   r2   r5   protein_outputprotein_embedssequence_reprs          rI   r   zEvollaProteinEncoder.forwardn  sJ    iW'99::>>Z.'4,>>
 	
rK   )rW   rX   rY   r,   rQ   r   r@   
LongTensorr  r   rZ   r[   s   @rI   r  r  h  s?    ^| ^
 
!1!1 
5CTCT 
 
rK   r  c                   b     e Zd Z	 	 	 ddedz  dedz  dedz  f fdZd Z	 	 	 	 	 	 	 d	dZ xZS )
#EvollaSequenceAlignerCrossAttentionNprotein_encoder_dimstructure_encoder_dimmsa_encoder_dimc                    t         |           |j                  | _        |j                  | _        | j                  dz  | _        t        | j                  | j                  z        | _        | j                  | j                  z  | _        |j                  }|j                  }|j                  }t        j                  | j                  | j                        | _        |Kt        j                  || j                        | _        t        j                  || j                        | _        nd | _        d | _        |Kt        j                  || j                        | _        t        j                  || j                        | _        nd | _        d | _        |Kt        j                  || j                        | _        t        j                  || j                        | _        nd | _        d | _        t)        | j                        | _        t        j,                  |      | _        t        j                  | j                  | j                  |      | _        t3        | j                  |      | _        t        j6                  t9        j:                  dg            | _        t        j6                  t9        j:                  dg            | _        y )Nr8   r           ) rP   rQ   r   num_attention_headsr   rf   attention_head_sizeall_head_size$aligner_attention_probs_dropout_probaligner_enable_biasaligner_ffn_multr   r   r/   key_proteinvalue_proteinkey_structurevalue_structurekey_msa	value_msaEvollaRMSNormattention_normDropoutr4   out_projr   r  r   r@   tensorgate_attentiongate_ffw)	rT   rU   r"  r#  r$  attention_probs_dropout_probenable_biasffn_multrV   s	           rI   rQ   z,EvollaSequenceAlignerCrossAttention.__init__{  s    	!--#)#=#= --t3
#&t'7'7$:R:R'R#S !558P8PP'-'R'R$00**YYt//1C1CD
*!yy)<d>P>PQD!#+>@R@R!SD#D!%D ,!#+@$BTBT!UD#%99-BDDVDV#WD !%D#'D &99_d6H6HIDLYY8J8JKDNDL!DN+D,<,<=zz">?		$"2"2D4D4D;W#D$4$4h? ll5<<+>?U\\3%%89rK   c	                    |||g}	|	D 
cg c]  }
|
|
	 }	}
|	st        d      t        j                  |	d      }	| j                  |      }| j	                  |      }| j
                  @| j                  4|j                  |      }| j                  |      }| j                  |      }nd}d}| j                  @| j                  4|j                  |      }| j                  |      }| j                  |      }nd}d}| j                  @| j                  4|j                  |      }| j                  |      }| j                  |      }nd}d}|||g}|D 
cg c]  }
|
|
	 }}
t        j                  |d      }|||g}|D 
cg c]  }
|
|
	 }}
t        j                  |d      }|j                         dd | j                  | j                  fz   } |j                  | j!                  dddd      }|j                         dd | j                  | j                  fz   } |j                  | j!                  dddd      }|j                         dd | j                  | j                  fz   } |j                  | j!                  dddd      }|| j"                  z  }|Mt        j$                  |j                  d      |j                  d            j                  |j&                        }|ddddddf   |	ddddddf   z  }t        j(                  ||j+                  dd	            }||j-                  dd
      j/                         z
  }|j1                  d|z
  j3                         t        j4                  |j6                        j8                        } t;        j<                  d      |      }t        j(                  ||      }|j!                  dddd      j?                         }|j                         dd	 | j@                  fz   } |j                  | }| jC                  |      }|S c c}
w c c}
w c c}
w )z
        query_states: text
        key_value_states: protein
        query_states: [bs, query_seq_len, dim]
        key_value_states: [bs, kv_seq_len, dim]
        query_attn_mask: [bs, query_seq_len]
        kv_attn_mask: [bs, kv_seq_len]
        Nz=At least one modality should be provided for cross attention.r+   r:   r7   r   r   r   r   Tr   )"
ValueErrorr@   r   r4  r/   r-  r.  r   r/  r0  r1  r2  r>   r'  r(  r   r   r   r   r_   rA   rB   r   r   r   r   finfor  minr   SoftmaxrE   r)  r6  )rT   query_statesprotein_key_value_statesstructure_key_value_statesmsa_key_value_statesquery_attn_maskprotein_kv_attn_maskstructure_kv_attn_maskmsa_kv_attn_maskkv_attn_maskr   query_layerkey_layer_proteinvalue_layer_proteinkey_layer_structurevalue_layer_structurekey_layer_msavalue_layer_msa	key_layervalue_layernew_query_layer_shapenew_key_layer_shapenew_value_layer_shaper2   rG   attention_scoresattention_probscontext_layernew_context_layer_shapes                                rI   cross_attentionz3EvollaSequenceAlignerCrossAttention.cross_attention  si   * -.DFVW#/Aa1=AA\]]yy15)),7 jj-'D,>,>,J'?'B'B<'P$ $ 0 01I J"&"4"45M"N $"&)d.B.B.N)C)F)F|)T&"&"4"45O"P$($8$89S$T!"&$(!<<#(B#7#:#:<#H  LL)=>M"nn-ABO M"O&(;]K	 );1Q]Q;	;IIiQ/	*,A?S"-?Qq??ii3 + 0 0 23B 7$$$$;
 !
 'k&&(=>FFq!QPQR'nn.s3$$$$7
 
 #INN$78@@Aq!L	 + 0 0 23B 7$$$$;
 !
 'k&&(=>FFq!QPQR!DJJ. "#jj):):1)=|?P?PQR?STWWXdXkXklO(D!T)9:\!TSWYZJZ=[[||K1D1DR1LM#l&7&7B&7&M&T&T&VV'33%%'\5G5G)H)L)L
 -"**,-=> _kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S***,CDm4q BL < @s"   P5P5P:P:P?P?c           
      ^   |z|j                   \  }}}|jt        j                  ||      j                  |	j                        |	j                  ||f      j                  z  j                  |j                        }nd }|z|j                   \  }}}|jt        j                  ||      j                  |	j                        |
j                  ||f      j                  z  j                  |j                        }nd }|z|j                   \  }}}|jt        j                  ||      j                  |	j                        |j                  ||f      j                  z  j                  |j                        }nd }|}||j                         s$||j                         s||j                         rz|}| j                  ||||||||      }t        j                  | j                        |z  }||z   }|}| j                  |      t        j                  | j                        z  }||z   }|S )N)r>   )rB  rC  rD  rE  rF  rG  rH  rI  )r   r@   r   r   r_   expandTanyr[  tanhr8  r  r9  )rT   rB  protein_kv_statesstructure_kv_statesmsa_kv_statesrF  rG  rH  rI  protein_batch_maskstructure_batch_maskmsa_batch_maskpast_key_valuesr   protein_kv_seq_lenr;   structure_kv_seq_lenmsa_kv_seq_lenr   residuals                       rI   r   z+EvollaSequenceAlignerCrossAttention.forward  sL    (*;*A*A'B"C#+JJr#5699:L:S:ST(//6H"5M/NPPQ"&--. %
 $( *,?,E,E)B$c%-JJr#78;;<N<U<UV*118Lb7Q1RTTU"(//0 '
 &*"$&3&9&9#B'JJr>2556H6O6OP$++."1E+FHHI"]))* !
  $$ */C/G/G/I#/4J4N4N4P).>.B.B.D$H 00*):+>%2 /%9'=!1 1 	M "JJt':':;mKM$}4M$H GGM2UZZ5NNM$}4MrK   rd   )NNNNNNN)rW   rX   rY   rf   rQ   r[  r   rZ   r[   s   @rI   r!  r!  z  sb     +/,0&*1: !4Z1:  #Tz	1:
 t1:fnn "#!GrK   r!  c                       e Zd Zy)r3  Nrk   rl   rK   rI   r3  r3  h  rm   rK   r3  c                       e Zd Zy)EvollaRotaryEmbeddingNrk   rl   rK   rI   rn  rn  l  rm   rK   rn  c                       e Zd Zy)	EvollaMLPNrk   rl   rK   rI   rp  rp  p  rm   rK   rp  c                       e Zd Zy)EvollaAttentionNrk   rl   rK   rI   rr  rr  t  rm   rK   rr  c                       e Zd Zdedef fdZ	 	 	 	 	 	 	 	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  fdZ xZS )EvollaDecoderLayerrU   	layer_idxc                     t         |   ||       |dz   t        |j                  |j                  z  d      z  dk(  rt        ||j                        | _        y y )Nr+   r   )r"  )rP   rQ   maxnum_hidden_layersaligner_num_add_layersr!  r   adapterrT   rU   ru  rV   s      rI   rQ   zEvollaDecoderLayer.__init__y  s[    +MS!9!9V=Z=Z!Z\]^^bcc>$*$6$6DL drK   Nr   r   r2   rR   rg  	use_cachera  rb  rc  rd  re  rf  rF  c           
      
   |}| j                  |      } | j                  d||||||d|\  }}||z   }|}| j                  |      }| j                  |      }||z   }t	        | d      r| j                  ||||	||
||      }|S )N)r   r2   rR   rg  r|  r   rz  )rB  ra  rb  rc  rF  rd  re  rf  rl   )input_layernorm	self_attnpost_attention_layernormmlphasattrrz  )rT   r   r   r2   rR   rg  r|  ra  rb  rc  rd  re  rf  rF  r5   rk  r   s                    rI   r   zEvollaDecoderLayer.forward  s    " !,,]; *4>> 
')%+ 3
 
q !=0 !55mD/ =04# LL*"3$7+ /#5%9- ) 	M rK   )NNNNFNNNNNNN)rW   rX   rY   r,   rf   rQ   r@   r   rg   r  r   r   r   rZ   r[   s   @rI   rt  rt  x  s9   |   IM.204(,!&1537-12648.2/33||3 #5<<#=>E3 t+	3
 &&-3 3 $;3 !<<$.3 #\\D03 ||d*3 "LL4/3 $llT13 t+3 ,3rK   rt  c                   N    e Zd ZdZdZdZg dZ ej                         d        Z	y)EvollaPreTrainedModelF)rt  rw   r   r!  c                    | j                   j                  }t        j                  | |       t	        |t
              rht        j                  |j                         t        j                  |j                         t        j                  |j                  j                         y t	        |t              r#t        j                  |j                  d|       y y )Nr&  )meanstd)rU   initializer_ranger   r   r   r!  r   zeros_r8  r9  ones_r4  weightr   normal_r   )rT   r.   r  s      rI   r   z#EvollaPreTrainedModel._init_weights  s    kk++%%dF3fABKK--.KK(JJv,,334 ABLLcs; CrK   N)
rW   rX   rY   r   r   r   r   r@   r   r   rl   rK   rI   r  r    s8     "' U]]_< <rK   r  c                       e Zd Zdef fdZd Zd Zeee		 	 	 	 	 	 	 	 	 	 	 	 dde
j                  dz  de
j                  dz  de
j                  dz  d	edz  d
e
j                  dz  dedz  de
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  deez  fd                     Z xZS )EvollaModelrU   c           	      F   t         |   |       |j                  | _        |j                  | _        t        j                  | j                  |j                  | j                        | _        t        |      | _
        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t!        |j                  |j"                        | _        t'        |dd      | _        t+        |      | _        | j/                          y c c}w )Nr   )rU   ru  )epsgradient_checkpointingF)rP   rQ   pad_token_idpadding_idx
vocab_sizer   	Embeddingr   embed_tokensr  protein_encoderr   r  rx  rt  r  r3  rms_norm_epsr   r?   r  rn  
rotary_embr   r{  s      rI   rQ   zEvollaModel.__init__  s     !.. ++LL&:L:LdN^N^_36Bmm "'v'?'?!@
 	 #!'
 "&"4"4&:M:MN	&-f6NPU&V#/v>s   $Dc                     | j                   S rO   r  r   s    rI   r   z EvollaModel.get_input_embeddings  s       rK   c                     || _         y rO   r  r   s     rI   r   z EvollaModel.set_input_embeddings  s
    !rK   Nr   r2   rR   rg  r   r|  protein_input_idsprotein_attention_maskstructure_feats	msa_featsre  rf  ra   c                    |du |duz  rt        d      || j                  |      }|r|t        | j                        }|V||j	                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }d}d}|^|\| j                  ||      }|j                  }t        j                  |j                  d   |j                  t
        j                        }t        | j                  |||	      }|}| j                  ||
      }| j                   D ]  } ||f||||||	|
|||||d|} | j#                  |      }t%        ||      }|S )a;  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
        structure_feats (torch.FloatTensor):
            The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        msa_feats (torch.FloatTensor):
            The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
        structure_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummpy input for now.
        msa_batch_mask (torch.Tensor):
            The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummpy input for now.
        Nz:You must specify exactly one of input_ids or inputs_embedsr   r   r+   r   r   )r_   r  )rU   r   r2   rg  )rR   )r2   rR   rg  r|  ra  rb  rc  rd  re  rf  rF  r   )r   rg  )r>  r  r   rU   get_seq_lengthr@   r   r   r_   r   r  r  r   r   r   r  r  r   r   )rT   r   r2   rR   rg  r   r|  r  r  r  r  re  rf  r5   past_seen_tokensprotein_featsrd  protein_outputscausal_maskr   r   decoder_layeroutputs                          rI   r   zEvollaModel.forward  s   B -t";<YZZ  --i8M0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L!(-C-O"22+5 3 O ,FFM!&!''*(//jj" );;')+	
 &"oom,oW![[ 	M)*) /#"/$3'#5%9- .$7 M	$ 		-0(++
 rK   )NNNNNNNNNNNN)rW   rX   rY   r,   rQ   r   r   r   r   r   r@   r  r   r   r  r   rg   r   r   rZ   r[   s   @rI   r  r    sf   | *!"  .2.204(,26!%596:48.248.2]##d*] t+] &&-	]
 ] ((4/] $;] !++d2] !&t 3] **T1] $$t+] $llT1] t+] 
(	(]    ]rK   r  c                   4    e Zd Z fdZd Zd Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dedz  deej                  z  fd              Z xZS )EvollaForProteinText2Textc                     t         |   |       t        |      | _        |j                  | _        t        j                  |j                  | j                  d      | _        | j                          y r   )
rP   rQ   r  r  r  r   r   r   lm_headr   rS   s     rI   rQ   z"EvollaForProteinText2Text.__init__N  sQ      (
 ++yy!3!3T__5QrK   c                 6    | j                   j                         S rO   )r  r   r   s    rI   r   z.EvollaForProteinText2Text.get_input_embeddingsV  s    zz..00rK   c                 8    | j                   j                  |      S rO   )r  r   r   s     rI   r   z.EvollaForProteinText2Text.set_input_embeddingsY  s    zz..u55rK   Nr   r2   r   labelsr  r  r|  logits_to_keepc	           
      h    | j                   d||||||d|	}
|
j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         }d}|  | j                  d||| j                  d|	}t        |||
j                  |
j                  |
j                        }|S )a,  
        protein_input_ids (torch.LongTensor):
            The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
        protein_attention_mask (torch.Tensor):
            The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

        Example:

        ```python
        >>> from transformers import EvollaProcessor, EvollaForProteinText2Text
        >>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
        >>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

        >>> protein_information = {
            "aa_seq": "your amino acid sequence",
            "foldseek": "your foldseek sequence",
        }
        >>> question = "What is the function of this protein?"
        >>> message = [
            {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
            {"role": "user", "content": question},
        ]

        >>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
        >>> outputs = model.generate(**inputs)

        >>> print(processor.batch_decode(outputs, skip_special_tokens=True))
        ```)r   r2   r   r  r  r|  N)logitsr  r  )lossr  rg  r   r   rl   )r  r   r   rf   slicer  loss_functionr  r   rg  r   r   )rT   r   r2   r   r  r  r  r|  r  r5   outputsr   slice_indicesr  r  
lm_outputss                   rI   r   z!EvollaForProteinText2Text.forward\  s    T ,64:: ,
)'/#9,
 ,
  118B>SV8W~ot4]kmA}a,?@A%4%%iVFtibhiD+#33!//))

 rK   )NNNNNNNr   )rW   rX   rY   rQ   r   r   r   r   r@   r  r   r  r   rf   r   rZ   r[   s   @rI   r  r  M  s    16  .2.226*.596:!%-.B##d*B t+B ((4/	B
   4'B !++d2B !&t 3B $;B ell*B  BrK   r  )r  r  r  )Nr&  )Zdataclassesr   r@   r    r   r   cache_utilsr   r   
generationr	   masking_utilsr
   r   modeling_outputsr   r   r   r   modeling_utilsr   processing_utilsr   utilsr   r   r   r   utils.genericr   utils.output_capturingr   r   esm.modeling_esmr   r   r   r   r   r   r    r!   r"   r#   llama.modeling_llamar$   r%   r&   r'   r(   r)   r*   configuration_evollar,   r-   
get_loggerrW   loggerModuler   rh   rJ   rM   r]   rj   ro   rq   rs   ru   rw   ry   r{   r}   r   r   r   r   r  r  r!  r3  rn  rp  rr  rt  r  r  r  __all__rl   rK   rI   <module>r     s   "   & . ) J  . &  8 E     = 
		H	% ! %II %<< % 
 % <<	 %
 LL4' % T\ %  % '( %F!] !
P"4 
P	 0 		] 		L 		 		 		 		* 		 	 C/ C C01
!< 1
h7 		 7 tA		 A'.		 '.T 

<k 
<  
<
299 
$k")) k\	L 		0 		 		n 	<* <~<0 <.|' |~S 5 Sl PrK   