
    ^j                        d Z ddlZddlmZ ddlmZ ddlZddlmZ ddlm	Z	m
Z
mZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0  e(jb                  e2      Z3d Z4 ed      dLd       Z5d Z6d Z7d Z8 G d dejr                        Z: G d dejr                        Z; G d d ejr                        Z<	 	 dMd!ejr                  d"ejz                  d#ejz                  d$ejz                  d%ejz                  dz  d&e>dz  d'e>d(e#e%   fd)Z? ee5       G d* d+ejr                               Z@ G d, d-ejr                        ZA G d. d/ejr                        ZB G d0 d1ejr                        ZC G d2 d3ejr                        ZD G d4 d5e      ZE G d6 d7ejr                        ZF G d8 d9ejr                        ZGe& G d: d;e!             ZHe& G d< d=eH             ZIe& G d> d?eH             ZJ G d@ dAejr                        ZK e&dBC       G dD dEeH             ZLe& G dF dGeH             ZM G dH dIejr                        ZNdJ ZOg dKZPy)NzPyTorch ESM model.    N)Callable)Optional)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )initialization)use_kernel_func_from_hubuse_kernelized_func)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer)"BaseModelOutputWithCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsMaskedLMOutputSequenceClassifierOutputTokenClassifierOutput)dynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)maybe_autocastmerge_with_config_defaults)OutputRecordercapture_outputs   )	EsmConfigc                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..N   dim)shapetorchcat)xx1x2s      o/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/esm/modeling_esm.pyrotate_halfr/   0   sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''    rotary_pos_embc                 b   | j                   }|j                  |      }|j                  |      }| j                         |z  t        | j                               |z  z   }|j                         |z  t        |j                               |z  z   }|j	                  |      |j	                  |      fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )dtype	unsqueezefloatr/   to)qkcossinunsqueeze_dimoriginal_dtypeq_embedk_embeds           r.   apply_rotary_pos_embr?   7   s    & WWN
--
&C
--
&Cwwy3;qwwy#9C#?@Gwwy3;qwwy#9C#?@G::n%wzz.'AAAr0   c                 j    | dz  dt        j                  | t        j                  d      z        z   z  S )zo
    This is the gelu implementation from the original ESM repo. Using F.gelu yields subtly wrong results.
    g      ?      ?g       @)r)   erfmathsqrtr+   s    r.   gelurF   R   s.     s7cEIIa$))C.&899::r0   c                 ,    | | j                  dd      z   S )zJMake layer symmetric in final two dimensions, used for contact prediction.r$   )	transposerE   s    r.   
symmetrizerJ   Y   s    q{{2r"""r0   c                     | j                  dd      }| j                  dd      }| j                  dd      }||z  }|j                  |       | |z
  }|S )z=Perform average product correct, used for contact prediction.r$   T)keepdimsrH   )r$   rH   )sumdiv_)r+   a1a2a12avg
normalizeds         r.   average_product_correctrT   ^   s[    	
rD	!B	
rD	!B
%%4%
(C
r'CHHSMSJr0   c                        e Zd ZU dZej
                  ed<   ddef fdZe		 	 	 ddedz  de
d   dedz  d	ed
ef   fd       Z ej                         edd              Z xZS )EsmRotaryEmbeddingz
    Rotary position embeddings.
    Implementation based on [ModernBERT's RotaryEmbedding](https://github.com/huggingface/transformers/blob/aad13b87ed59f2afcfaebc985f403301887a35fc/src/transformers/models/modernbert/modeling_modernbert.py#L94).
    inv_freqNconfigc                     t         |           || _        i | _        | j	                  | j                  |      \  }}| j                  d|       t        | d|       y )NrW   attention_scaling)super__init__rX   	rope_typecompute_default_rope_parametersregister_buffersetattr)selfrX   devicecurr_inv_freqcurr_attention_scaling	__class__s        r.   r\   zEsmRotaryEmbedding.__init__r   sZ    040T0TUYU`U`bh0i--Z7)+ABr0   rb   ztorch.deviceseq_lenreturnztorch.Tensorc                    | j                   }t        | dd      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                        j                  |t        j                        |z  z  z  }||fS )a  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        head_dimNrA   r   r%   r3   )rb   r3   )	
rope_thetagetattrhidden_sizenum_attention_headsr)   arangeint64r6   r5   )rX   rb   rf   baser'   attention_factorrW   s          r.   r^   z2EsmRotaryEmbedding.compute_default_rope_parameters|   s    (   fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjjk
 )))r0   c                 B   t        | d      }t        | d      }|d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d	      5  |j                         |j                         z  j                  dd
      }	t        j                  |	|	fd      }
|
j                         |z  }|
j                         |z  }d d d        j	                  |j                        j	                  |j                        fS # 1 sw Y   AxY w)NrW   rZ   r   r$   r!   mpscpuF)device_typeenabledr%   r&   rj   )rl   r5   expandr(   r6   rb   
isinstancetypestrr   rI   r)   r*   r9   r:   r3   )ra   r+   position_ids
layer_typerW   rZ   inv_freq_expandedposition_ids_expandedrv   freqsembr9   r:   s                r.   forwardzEsmRotaryEmbedding.forward   s^    4,#D*=>$T1d]399;BB<CUCUVWCXZ\^_`ccdedldlm ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	0&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')//C'')//C		0 vvAGGv$cff177f&;;;	0 	0s   $A1FFN)NNN)__name__
__module____qualname____doc__r)   Tensor__annotations__r"   r\   staticmethodr   inttupler5   r^   no_gradr   r   __classcell__re   s   @r.   rV   rV   j   s    
 llCy C #'+/"*D *(* t* 
~u$	%	* *< U]]_<  <r0   rV   c                   8     e Zd ZdZ	 	 ddedef fdZd Z xZS )EsmContactPredictionHeadzWPerforms symmetrization, apc, and computes a logistic regression on the output featuresin_featureseos_idxc                     t         |           || _        || _        t	        j
                  |d|      | _        t	        j                         | _        y )Nr!   )	r[   r\   r   r   r   Linear
regressionSigmoid
activation)ra   r   biasr   re   s       r.   r\   z!EsmContactPredictionHead.__init__   s@     	&))KD9**,r0   c                 X   |j                  | j                        j                  |      }|j                  d      |j                  d      z  }||d d d d d d d d f   z  }|dd dd df   }|ddd dd f   }|j	                         \  }}}}}|j                  |||z  ||      }|j                  | j                  j                  j                        }t        t        |            }|j                  dddd      }| j                  | j                  |      j                  d            S )Nr!   r%   .r$   r   r	   )ner   r6   r4   sizeviewr   weightrb   rT   rJ   permuter   squeeze)	ra   tokens
attentionseos_mask
batch_sizelayersheadsseqlen_s	            r.   r   z EsmContactPredictionHead.forward   s!   99T\\*--j9%%a(8+=+=a+@@(1dD!Q+>"??
SbS#2#.
QR,
/9/@,
FE61__Z%P
  ]]OO""))

 -Z
-CD
''1a3
tz:BB1EFFr0   )Tr%   )r   r   r   r   r   r\   r   r   r   s   @r.   r   r      s+    a
 	
'
' 	
'Gr0   r   c                   8     e Zd ZdZ fdZ	 	 	 	 ddZd Z xZS )EsmEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                    t         |           t        j                  |j                  |j
                  |j                        | _        |j                  r1t        j                  |j
                  |j                        | _        nd | _        t        j                  |j                        | _        t        |dd      | _        | j#                  dt%        j&                  |j(                        j+                  d      d       |j                  | _        | j                   dk(  r;t        j                  |j(                  |j
                  | j,                        | _        |j0                  | _        |j2                  | _        y )	N)padding_idxepsposition_embedding_typeabsoluter|   r!   r$   F)
persistent)r[   r\   r   	Embedding
vocab_sizerm   pad_token_idword_embeddingsemb_layer_norm_before	LayerNormlayer_norm_eps
layer_normDropouthidden_dropout_probdropoutrl   r   r_   r)   ro   max_position_embeddingsrx   r   position_embeddingstoken_dropoutmask_token_idra   rX   re   s     r.   r\   zEsmEmbeddings.__init__   s*   !||F,=,=v?Q?Q_e_r_rs'' ll6+=+=6CXCXYDO"DOzz&"<"<='.v7PR\']$ELL)G)GHOOPWXej 	 	
 "..'':5')||..0B0BPTP`P`(D$ $11#11r0   c                    |*|t        || j                        }n| j                  |      }|| j                  |      }|}| j                  r||j                  || j                  k(  j                  d      d      }d}||j                  d      n|j                  d   }|| j                  k(  j                  d      j                         |z  }|d|z
  z  d|z
  d d d d f   z  j                  |j                        }| j                  dk(  r| j                  |      }	||	z   }| j                  | j                  |      }|-||j                  d      z  j                  |j                        }|S )Nr$           gQ?r!   r   )"create_position_ids_from_input_idsr   &create_position_ids_from_inputs_embedsr   r   masked_fillr   r4   rM   r(   r5   r6   r3   r   r   r   )
ra   	input_idsattention_maskr|   inputs_embeds
embeddingsmask_ratio_trainsrc_lengthsmask_ratio_observedr   s
             r.   r   zEsmEmbeddings.forward   s    $A)TM]M]^#JJ=Y  00;M #
 )"7#//d>P>P1P0[0[\^0_adeJ)4B4N.,,R0T]TcTcdeTfK#,0B0B#B"G"G"K"Q"Q"SVa"a$,<(<=EXAXZ[]acgZg@hhll  J '':5"&":":<"H#&99J??&4J%$~'?'?'CCGG
HXHXYJ r0   c                    |j                         dd }|d   }t        j                  | j                  dz   || j                  z   dz   t        j                  |j
                        }|j                  d      j                  |      S )z
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr$   r!   )r3   rb   r   )r   r)   ro   r   longrb   r4   rx   )ra   r   input_shapesequence_lengthr|   s        r.   r   z4EsmEmbeddings.create_position_ids_from_inputs_embeds  s     $((*3B/%a.||q /D4D4D"Dq"HPUPZPZcpcwcw
 %%a(//<<r0   NNNN)r   r   r   r   r\   r   r   r   r   s   @r.   r   r      s&    22 /b=r0   r   modulequerykeyvaluer   scalingr   kwargsc                    ||j                  d      dz  }t        j                  ||j                  dd            |z  }|||z   }t        j
                  j                  |d      }t        j
                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nr$         r%   r	   r&   )ptrainingr!   )
r   r)   matmulrI   r   
functionalsoftmaxr   r   
contiguous)
r   r   r   r   r   r   r   r   attn_weightsattn_outputs
             r.   eager_attention_forwardr   1  s     **R.D( <<s}}Q':;gEL!#n4==((2(>L==((6??([L,,|U3K''1-88:K$$r0   c                        e Zd Zd fd	Z	 	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dej
                  dz  dee   d	e	ej
                     fd
Z
 xZS )EsmSelfAttentionNc                 J   t         |           || _        |j                  |j                  z  dk7  r2t        |d      s&t        d|j                   d|j                   d      |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _	        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        |j                  | _        |xs t#        |dd      | _        d| _        |j(                  | _        || _        | j(                  xr | | _        y )	Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r   r   rA   )r[   r\   rX   rm   rn   hasattr
ValueErrorr   attention_head_sizeall_head_sizer   r   r   r   r   attention_probs_dropout_probr   rl   r   r   
is_decoder	layer_idx	is_causal)ra   rX   r   r   is_cross_attentionre   s        r.   r\   zEsmSelfAttention.__init__O  sh    : ::a?PVXhHi#F$6$6#7 8 445Q8 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EF
99V//1C1CDYYv1143E3EF
::'> (
'-zC
$  ++"C1C-Cr0   hidden_statesr   encoder_hidden_statesencoder_attention_maskr   r   rg   c                 
   |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }	|d u}
|
r|n|}|
r|n|}| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }|	| j                  dz  z  }	| j                  dk(  r|\  }}t        |	|||d      \  }	}t        j                  | j                  j                  t              } || |	|||f| j                  sdn| j                  | j                   d|\  }} |j"                  g |d j%                         }||fS )	Nr$   r!   r%   r   rotary)r;   r   )r   r   )r(   r   r   r   rI   r   r   r   r?   r   get_interfacerX   _attn_implementationr   r   r   r   reshaper   )ra   r   r   r   r   r   r   r   hidden_shapequery_layerr   current_states	key_layervalue_layerr9   r:   attention_interfacer   r   s                      r.   r   zEsmSelfAttention.forwardk  s    $))#2.CCbC$*B*BCjj/44\BLLQPQR2$>2D.-3E/>HH^,11,?II!QO	jj055lCMMaQRS "D$<$<d$BB''83*HC%9+yRUWZjk%l"K(?(M(MKK,,.E)
 %8	%
  $}}C$,,LL	%
 	%
!\ *k));;;;FFHL((r0   )NNFr   )r   r   r   r\   r)   r   FloatTensorr   r   r   r   r   r   s   @r.   r   r   M  s    D> 48:>;?37.)||.) ))D0.)  %0047	.)
 !& 1 1D 8.) #\\D0.) +,.) 
u||	.)r0   r   c                   $     e Zd Z fdZd Z xZS )EsmSelfOutputc                     t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        y r   )	r[   r\   r   r   rm   denser   r   r   r   s     r.   r\   zEsmSelfOutput.__init__  sB    YYv1163E3EF
zz&"<"<=r0   c                 T    | j                  |      }| j                  |      }||z   }|S r   r  r   ra   r   input_tensors      r.   r   zEsmSelfOutput.forward  .    

=1]3%4r0   r   r   r   r\   r   r   r   s   @r.   r  r        >
r0   r  c                   Z     e Zd Zd fd	Z	 	 	 	 ddej
                  dz  dee   fdZ xZ	S )EsmAttentionNc                     t         |           t        |||      | _        t	        |      | _        t        j                  |j                  |j                        | _        y )N)r   r   r   )
r[   r\   r   ra   r  outputr   r   rm   r   )ra   rX   r   r   re   s       r.   r\   zEsmAttention.__init__  sI    $VyUgh	#F+f&8&8f>S>STr0   r   r   c                     | j                  |      } | j                  |f||||d|\  }}	| j                  ||      }|S )Nr   r   r   r   )r   ra   r  )
ra   r   r   r   r   r   r   hidden_states_lnr   r   s
             r.   r   zEsmAttention.forward  s]      >>-8"
)"7#9 3
 
Q kk+}=r0   )NFr   )
r   r   r   r\   r)   r   r   r   r   r   r   s   @r.   r  r    s?    U "#37 #\\D0 +,r0   r  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )EsmIntermediatec                     t         |           t        j                  |j                  |j
                        | _        y r   )r[   r\   r   r   rm   intermediate_sizer  r   s     r.   r\   zEsmIntermediate.__init__  s,    YYv1163K3KL
r0   r   rg   c                 >    | j                  |      }t        |      }|S r   )r  rF   )ra   r   s     r.   r   zEsmIntermediate.forward  s     

=1]+r0   r   r   r   r\   r)   r   r   r   r   s   @r.   r  r    s$    MU\\ ell r0   r  c                   $     e Zd Z fdZd Z xZS )	EsmOutputc                     t         |           t        j                  |j                  |j
                        | _        t        j                  |j                        | _	        y r   )
r[   r\   r   r   r  rm   r  r   r   r   r   s     r.   r\   zEsmOutput.__init__  sB    YYv779K9KL
zz&"<"<=r0   c                 T    | j                  |      }| j                  |      }||z   }|S r   r  r  s      r.   r   zEsmOutput.forward  r
  r0   r  r   s   @r.   r  r    r  r0   r  c                   ^     e Zd Z fdZ	 	 	 	 ddej
                  dz  dee   fdZd Z	 xZ
S )EsmLayerc                    t         |           |j                  | _        d| _        t	        |      | _        |j                  | _        |j                  | _        | j                  r,| j                  st        |  d      t	        |d      | _	        t        |      | _        t        |      | _        t        j                  |j                   |j"                        | _        y )Nr!   z> should be used as a decoder model if cross attention is addedT)r   r   )r[   r\   chunk_size_feed_forwardseq_len_dimr  	attentionr   add_cross_attentionRuntimeErrorcrossattentionr  intermediater  r  r   r   rm   r   r   s     r.   r\   zEsmLayer.__init__  s    '-'E'E$%f- ++#)#=#= ##??"dV+i#jkk".v$"OD+F3'f&8&8f>S>STr0   Nr   r   c                      | j                   |f||d|}| j                  r5|3t        | d      st        d|  d       | j                  |f||||d|}| j                  |      }|S )N)r   r   r&  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r  )r#  r   r   AttributeErrorr&  feed_forward_chunk)	ra   r   r   r   r   r   r   attention_outputlayer_outputs	            r.   r   zEsmLayer.forward  s     *4>>
) 3
 	
 ??4@4!12$=dV D` ` 
  3t22  -&;'=$7    ../?@r0   c                 n    | j                  |      }| j                  |      }| j                  ||      }|S r   )r   r'  r  )ra   r+  attention_output_lnintermediate_outputr,  s        r.   r*  zEsmLayer.feed_forward_chunk  s<    "nn-=>"//0CD{{#68HIr0   r   )r   r   r   r\   r)   r   r   r   r   r*  r   r   s   @r.   r  r    sE    U$ "#37! #\\D0! +,!Fr0   r  c                   b     e Zd Z fdZe	 	 	 	 ddej                  dz  dee   fd       Z	 xZ
S )
EsmEncoderc                 0   t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        t        j                  |j                  |j                        | _        d| _        y c c}w )Nr   F)r[   r\   rX   r   
ModuleListrangenum_hidden_layersr  layerr   rm   r   emb_layer_norm_aftergradient_checkpointing)ra   rX   r   re   s      r.   r\   zEsmEncoder.__init__  sm    ]]eFD\D\>]#^HV$4#^_
$&LL1C1CI^I^$_!&+# $_s   BNr   r   c           	          t        | j                        D ]  \  }} ||f||||d|} | j                  r| j                  |      }t        |      S )Nr  )last_hidden_state)	enumerater6  r7  r   )	ra   r   r   r   r   r   r   ilayer_modules	            r.   r   zEsmEncoder.forward!  sn      )4 	OA|(-&;'=$7 M	 $$ 55mDM1MRRr0   r   )r   r   r   r\   r   r)   r   r   r   r   r   r   s   @r.   r1  r1    sQ    ,  "#37S #\\D0S +,S Sr0   r1  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )	EsmPoolerc                     t         |           t        j                  |j                  |j                        | _        t        j                         | _        y r   )r[   r\   r   r   rm   r  Tanhr   r   s     r.   r\   zEsmPooler.__init__=  s9    YYv1163E3EF
'')r0   r   rg   c                 \    |d d df   }| j                  |      }| j                  |      }|S Nr   )r  r   )ra   r   first_token_tensorpooled_outputs       r.   r   zEsmPooler.forwardB  s6     +1a40

#566r0   r  r   s   @r.   r?  r?  <  s#    $
U\\ ell r0   r?  c                        e Zd ZU eed<   dZdZdZg dZdgZ	dZ
dZdZdZe eedd	      g eedd
	      gdZ ej&                          fd       Zd Z xZS )EsmPreTrainedModelrX   esmTF)r  #EsmFoldTriangularSelfAttentionBlockr   zposition_embeddings.weightr!   r#  )index
layer_namer&  )r   r   cross_attentionsc                    t         |   |       t        |t              r t	        j
                  |j                         yt        |t              rZt	        j                  |j                  t        j                  |j                  j                  d         j                  d             yt        |t              r?|j                  |j                         \  }}t	        j                  t#        |d      |       yy)zInitialize the weightsr$   r   rW   N)r[   _init_weightsry   	EsmLMHeadinitzeros_r   r   copy_r|   r)   ro   r(   rx   rV   r^   rX   rl   )ra   r   rc   r   re   s       r.   rN  z EsmPreTrainedModel._init_weights`  s     	f%fi(KK$.JJv**ELL9L9L9R9RSU9V,W,^,^_f,gh 23%EEfmmTM1JJwvz2MB 4r0   c                      y r    ra   s    r.   get_output_embeddingsz(EsmPreTrainedModel.get_output_embeddingsl  s     r0   )r   r   r   r"   r   base_model_prefixsupports_gradient_checkpointingaccepts_loss_kwargs_no_split_modules"_keys_to_ignore_on_load_unexpected_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendr  r   r   _can_record_outputsr)   r   rN  rV  r   r   s   @r.   rG  rG  K  s    &*#\*F)G&N"& "%&6aKXY+1AQR
 U]]_	C 	Cr0   rG  c                   X    e Zd ZdZd fd	Zd Zd Zd Zee	e
	 	 	 	 	 	 ddej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dee   deej                     ez  fd                     Zd Zd Z xZS )EsmModela  

    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    c                    t         |   |       || _        t        |      | _        d| _        t        |dd      | _        | j                  dk(  rt        |      | _        t        |      | _
        |rt        |      nd| _        t        |j                  |j                  z  d      | _        | j#                  | j$                         | j'                          y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        Nr   r   r   )rX   T)r   r   )r[   r\   rX   r   r   rotary_embeddingsrl   r   rV   r1  encoderr?  poolerr   r5  rn   contact_head"_register_load_state_dict_pre_hook	load_hook	post_init)ra   rX   add_pooling_layerre   s      r.   r\   zEsmModel.__init__  s    
 	 '/!%'.v7PR\']$''83%7v%FD"!&)+<i'$40063M3MMTX
 	//?r0   c                      d}||vrCt        fdt        |j                               D              }|r||d      ||<   |D ]  }||=  yy)a  Remap per-layer rotary inv_freq keys from old checkpoints to the new model-level location.

        Old checkpoints stored inv_freq per attention layer at:
            {prefix}encoder.layer.{i}.attention.self.rotary_embeddings.inv_freq
        New code stores a single shared inv_freq at:
            {prefix}rotary_embeddings.inv_freq
        The old checkpoint values must be preserved (not recomputed) because they may
        have been saved in float16, matching the precision used during training.
        zrotary_embeddings.inv_freqc              3   f   K   | ](  }|j                        r|j                  d       r| * yw)z*.attention.self.rotary_embeddings.inv_freqN)
startswithendswith).0r8   prefixs     r.   	<genexpr>z%EsmModel.load_hook.<locals>.<genexpr>  s1      <<'AJJ7c,d s   .1r   N)sortedlistkeys)ra   
state_dictrq  argsnew_keyold_keysr8   s     `    r.   ri  zEsmModel.load_hook  sp     H67*$ joo/0 H
 &0!&=
7# "qM" %r0   c                 .    | j                   j                  S r   r   r   rU  s    r.   get_input_embeddingszEsmModel.get_input_embeddings  s    ...r0   c                 &    || j                   _        y r   r{  )ra   r   s     r.   set_input_embeddingszEsmModel.set_input_embeddings  s    */'r0   Nr   r   r|   r   r   r   r   rg   c                    |du |duz  rt        d      || j                  |||      }| j                  ||||d      \  }}| j                  dk(  rT|?|j                  d   }t        j                  ||j                        j                  d      }| j                  ||      }	nd}	 | j                  |f||||	d	|}
|
d   }| j                  | j                  |      nd}t        ||
      S )aV  
        input_ids (`torch.LongTensor` of shape `((batch_size, sequence_length))`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        position_ids (`torch.LongTensor` of shape `((batch_size, sequence_length))`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `((batch_size, sequence_length), hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nz:You must specify exactly one of input_ids or inputs_embeds)r   r   r|   )r   r   embedding_outputr   past_key_valuesr   r!   )rb   r   r  )r:  pooler_output)r   r   _create_attention_masksr   r(   r)   ro   rb   r4   rd  re  rf  r   )ra   r   r   r|   r   r   r   r   rf   r   encoder_outputssequence_outputrE  s                r.   r   zEsmModel.forward  s;   > -t";<YZZ  !OO#-) , M 261M1M)#9*"7  2N 2
.. ''83#'--a0$||GM<P<PQ[[\]^"&"8"8"U"&&$,,
)"7#9 3
 
 *!,8<8OO4UY;-'
 	
r0   c                     | j                   j                  rt        | j                   |||      }nt        | j                   ||      }|t        | j                   |||      }||fS )N)rX   r   r   r  )rX   r   r   )rX   r   r   r   )rX   r   r   r   )ra   r   r   r  r   r  s         r.   r  z EsmModel._create_attention_masks  su     ;;!!/{{.- /	N 7{{.-N "-%>{{.5&;	&" 555r0   c                 H    | ||dd      j                   }t        j                  |d      }||j                  d      j                  d      j                  d      z  }||j                  d      j                  d      j                  d      z  }| j	                  ||      S )NT)r   return_dictoutput_attentionsr!   r&   r%   r	      )r   r)   stackr4   rg  )ra   r   r   attnss       r.   predict_contactszEsmModel.predict_contacts#  s    VN`deppEq)
 	))!,66q9CCAFF))!,66q9CCAFF  //r0   )T)NNNNNN)r   r   r   r   r\   ri  r|  r~  r   r    r   r)   r   r   r   r   r   r   r  r  r   r   s   @r.   rb  rb  r  s   
6",/0   *..2,0-1596:F
<<$&F
 t+F
 llT)	F

 ||d*F
  %||d2F
 !&t 3F
 +,F
 
u||	K	KF
    F
R6@	0r0   rb  c                   J    e Zd ZddiZ fdZd Zd Zee	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  dee   deez  fd              Zd Z xZS )EsmForMaskedLMzlm_head.decoder.weightz%esm.embeddings.word_embeddings.weightc                     t         |   |       |j                  rt        j	                  d       t        |d      | _        t        |      | _        | j                          y )NzjIf you want to use `EsmForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Frk  )
r[   r\   r   loggerwarningrb  rH  rO  lm_headrj  r   s     r.   r\   zEsmForMaskedLM.__init__3  sP     NN1
 Fe< (r0   c                 .    | j                   j                  S r   r  decoderrU  s    r.   rV  z$EsmForMaskedLM.get_output_embeddingsA  s    ||###r0   c                 &    || j                   _        y r   r  )ra   new_embeddingss     r.   set_output_embeddingsz$EsmForMaskedLM.set_output_embeddingsD  s    -r0   Nr   r   r|   r   r   r   labelsr   rg   c           	      p    | j                   |f|||||d|}	|	d   }
| j                  |
      }d}|at               }|j                  |j                        } ||j                  d| j                  j                        |j                  d            }t        |||	j                  |	j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        )r   r|   r   r   r   r   Nr$   losslogitsr   r   )rH  r  r   r6   rb   r   rX   r   r   r   r   )ra   r   r   r|   r   r   r   r  r   outputsr  prediction_scoresmasked_lm_lossloss_fcts                 r.   r   zEsmForMaskedLM.forwardG  s    ( $((
)%'"7#9
 
 "!* LL9')HYY0778F%&7&<&<RAWAW&XZ`ZeZefhZijN$!//))	
 	
r0   c                 <    | j                   j                  ||      S )N)r   )rH  r  )ra   r   r   s      r.   r  zEsmForMaskedLM.predict_contactsu  s    xx(((OOr0   )NNNNNNN)r   r   r   _tied_weights_keysr\   rV  r  r   r   r)   
LongTensorr   r  r   r   r   r   r   r  r   r   s   @r.   r  r  /  s   24[\$.  .2.20426:>6:*.*
##d**
 t+*
 &&-	*

 ((4/*
  %0047*
 !&t 3*
   4'*
 +,*
 
	*
  *
XPr0   r  c                   (     e Zd ZdZ fdZd Z xZS )rO  z&ESM Head for masked language modeling.c                    t         |           t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                  d      | _
        t        j                  t        j                  |j                              | _        y )Nr   F)r   )r[   r\   r   r   rm   r  r   r   r   r   r  	Parameterr)   zerosr   r   s     r.   r\   zEsmLMHead.__init__|  s    YYv1163E3EF
,,v'9'9v?T?TUyy!3!3V5F5FUSLLV->->!?@	r0   c                     | j                  |      }t        |      }| j                  |      }| j                  |      | j                  z   }|S r   )r  rF   r   r  r   ra   featuresr   r+   s       r.   r   zEsmLMHead.forward  sD    JJx GOOA LLOdii'r0   r   r   r   r   r\   r   r   r   s   @r.   rO  rO  y  s    0Ar0   rO  z
    ESM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )custom_introc                        e Zd Z fdZee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  de
e   d	eez  fd
              Z xZS )EsmForSequenceClassificationc                     t         |   |       |j                  | _        || _        t	        |d      | _        t        |      | _        | j                          y NFr  )	r[   r\   
num_labelsrX   rb  rH  EsmClassificationHead
classifierrj  r   s     r.   r\   z%EsmForSequenceClassification.__init__  sH      ++Fe</7r0   Nr   r   r|   r   r  r   rg   c                     | j                   |f|||d|}|d   }| j                  |      }	d}
||j                  |	j                        }| j                  j
                  | j                  dk(  rd| j                  _        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                  _        nd| j                  _        | j                  j
                  dk(  rIt               }| j                  dk(  r& ||	j                         |j                               }
n ||	|      }
n| j                  j
                  dk(  r=t               } ||	j                  d| j                        |j                  d            }
n,| j                  j
                  dk(  rt               } ||	|      }
t!        |
|	|j"                  |j$                  	      S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        r   r|   r   r   Nr!   r   single_label_classificationmulti_label_classificationr$   r  )rH  r  r6   rb   rX   problem_typer  r3   r)   r   r   r   r   r   r   r   r   r   r   ra   r   r   r|   r   r  r   r  r  r  r  r  s               r.   r   z$EsmForSequenceClassification.forward  s   $ $((
)%'	

 
 "!*1YYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+-B @&++b/R))-II,./'!//))	
 	
r0   NNNNN)r   r   r   r\   r   r   r)   r  r   r  r   r   r   r   r   r   r   s   @r.   r  r    s      .2.20426*.8
##d*8
 t+8
 &&-	8

 ((4/8
   4'8
 +,8
 
)	)8
  8
r0   r  c                        e Zd Z fdZee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  de
e   d	eez  fd
              Z xZS )EsmForTokenClassificationc                 0   t         |   |       |j                  | _        t        |d      | _        t        j                  |j                        | _        t        j                  |j                  |j                        | _        | j                          y r  )r[   r\   r  rb  rH  r   r   r   r   r   rm   r  rj  r   s     r.   r\   z"EsmForTokenClassification.__init__  si      ++Fe<zz&"<"<=))F$6$68I8IJr0   Nr   r   r|   r   r  r   rg   c                 z    | j                   |f|||d|}|d   }| j                  |      }| j                  |      }	d}
|Wt               }|j	                  |	j
                        } ||	j                  d| j                        |j                  d            }
t        |
|	|j                  |j                        S )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        r  r   Nr$   r  )rH  r   r  r   r6   rb   r   r  r   r   r   r  s               r.   r   z!EsmForTokenClassification.forward  s      $((
)%'	

 
 "!*,,71')HYYv}}-FFKKDOO<fkk"oND$!//))	
 	
r0   r  )r   r   r   r\   r   r   r)   r  r   r  r   r   r   r   r   r   r   s   @r.   r  r    s      .2.20426*.'
##d*'
 t+'
 &&-	'

 ((4/'
   4''
 +,'
 
&	&'
  '
r0   r  c                   (     e Zd ZdZ fdZd Z xZS )r  z-Head for sentence-level classification tasks.c                 &   t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        t        j                  |j                  |j                        | _
        y r   )r[   r\   r   r   rm   r  r   r   r   r  out_projr   s     r.   r\   zEsmClassificationHead.__init__  s`    YYv1163E3EF
zz&"<"<=		&"4"4f6G6GHr0   c                     |d d dd d f   }| j                  |      }| j                  |      }t        j                  |      }| j                  |      }| j	                  |      }|S rC  )r   r  r)   tanhr  r  s       r.   r   zEsmClassificationHead.forward  sY    Q1WLLOJJqMJJqMLLOMM!r0   r  r   s   @r.   r  r    s    7Ir0   r  c                     | j                  |      j                         }t        j                  |d      j	                  |      |z  }|j                         |z   S )a  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        x: torch.Tensor x:

    Returns: torch.Tensor
    r!   r&   )r   r   r)   cumsumtype_asr   )r   r   maskincremental_indicess       r.   r   r   '  sP     <<$((*D,,t3;;DADH##%33r0   )r  r  r  rb  rG  )r!   )Nr   )Qr   rC   collections.abcr   typingr   r)   r   torch.nnr   r   r    r
   rP  integrationsr   r   masking_utilsr   r   modeling_layersr   modeling_outputsr   r   r   r   r   modeling_rope_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.genericr   r   utils.output_capturingr   r    configuration_esmr"   
get_loggerr   r  r/   r?   rF   rJ   rT   ModulerV   r   r   r   r5   r   r   r  r  r  r  r  r1  r?  rG  rb  r  rO  r  r  r  r   __all__rT  r0   r.   <module>r     s     $    A A & I J 9  7 F & R R G E ( 
		H	%( *+B ,B4;#
	A< A<H Gryy  GF\=BII \=L !%II%<<% 
% <<	%
 LL4'% T\% % '(%8 )*K)ryy K) +K)\
BII 
299 <bii 
		 
7) 7tS SF		  # # #L y0! y0 y0x FP' FP FPR		 * E
#5 E
E
P 4
 2 4
 4
nBII &4 r0   