Ë
    ÛÓ^jXn  ã                   ó"  — d dl Z d dlmZ d dlZd dlmZ d dlmZmZmZ ddl	m
Z
 ddlmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+  e%jX                  e-«      Z. G d„ dej^                  «      Z0 G d„ dej^                  «      Z1	 	 d2dejd                  dejf                  dejf                  dejf                  dejf                  dz  de4dz  de4de e"   fd„Z5 G d „ d!ejd                  «      Z6 G d"„ d#e«      Z7e# G d$„ d%e«      «       Z8e# G d&„ d'e8«      «       Z9 e#d(¬)«       G d*„ d+e8e«      «       Z:e# G d,„ d-e8«      «       Z; e#d.¬)«       G d/„ d0e8«      «       Z<g d1¢Z=y)3é    N)ÚCallable)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚ SequenceClassifierOutputWithPastÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚBioGptConfigc                   óx   ‡ — e Zd ZdZdedefˆ fd„Z	 	 d
dej                  dedej                  dz  fˆ fd	„Zˆ xZ	S )Ú BioGptLearnedPositionalEmbeddingzN
    This module learns positional embeddings up to a fixed maximum size.
    Únum_embeddingsÚembedding_dimc                 óN   •— d| _         t        ‰| �	  || j                   z   |«       y )Né   )ÚoffsetÚsuperÚ__init__)Úselfr!   r"   Ú	__class__s      €úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/biogpt/modeling_biogpt.pyr'   z)BioGptLearnedPositionalEmbedding.__init__8   s$   ø€ ð ˆŒÜ‰Ñ˜¨$¯+©+Ñ5°}ÕEó    NÚattention_maskÚpast_key_values_lengthÚposition_idsc                 ó°   •— |€8t        j                  |d¬«      }||z  dz
  j                  «       }|dd…|d…f   }t        ‰| �  || j
                  z   «      S )z3`input_ids_shape` is expected to be [bsz x seqlen].Nr   ©Údim)ÚtorchÚcumsumÚlongr&   Úforwardr%   )r(   r,   r-   r.   r)   s       €r*   r5   z(BioGptLearnedPositionalEmbedding.forward>   s^   ø€ ð ÐÜ Ÿ<™<¨¸AÔ>ˆLØ(¨>Ñ9¸AÑ=×CÑCÓEˆLà'ªÐ+AÑ+BÐ(BÑCˆLä‰w‰˜|¨d¯k©kÑ9Ó:Ð:r+   )r   N)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr'   r2   Ú
LongTensorr5   Ú__classcell__©r)   s   @r*   r    r    3   s]   ø„ ñðF sð F¸3õ Fð '(Ø04ñ	;à×(Ñ(ð;ð !$ð;ð ×&Ñ&¨Ñ-÷	;ñ ;r+   r    c            
       ó`   ‡ — e Zd ZdZd
dededededz  fˆ fd„Zdej                  fˆ fd	„Z	ˆ xZ
S )ÚBioGptScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    r!   r"   Úpadding_idxÚembed_scaleNc                 ó6   •— t         ‰| �  |||«       || _        y ©N)r&   r'   rA   )r(   r!   r"   r@   rA   r)   s        €r*   r'   z"BioGptScaledWordEmbedding.__init__T   s   ø€ Ü‰Ñ˜¨¸ÔDØ&ˆÕr+   Ú	input_idsc                 ó<   •— t         ‰| �  |«      | j                  z  S rC   )r&   r5   rA   )r(   rD   r)   s     €r*   r5   z!BioGptScaledWordEmbedding.forwardX   s   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<Ñ<Ð<r+   )ç      ð?)r6   r7   r8   r9   r:   Úfloatr'   r2   ÚTensorr5   r<   r=   s   @r*   r?   r?   O   sE   ø„ ññ' sð '¸3ð 'ÈSð 'Ð_dÐgkÑ_kõ 'ð= §¡÷ =ñ =r+   r?   ÚmoduleÚqueryÚkeyÚvaluer,   ÚscalingÚdropoutÚkwargsc                 óœ  — |€|j                  d«      dz  }t        j                  ||j                  dd«      «      |z  }|�||z   }t        j
                  j                  |d¬«      }t        j
                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Néÿÿÿÿç      à¿r$   r   r0   ©ÚpÚtrainingr   )
Úsizer2   ÚmatmulÚ	transposeÚnnÚ
functionalÚsoftmaxrN   rU   Ú
contiguous)
rI   rJ   rK   rL   r,   rM   rN   rO   Úattn_weightsÚattn_outputs
             r*   Úeager_attention_forwardr_   \   s»   € ð €Ø—*‘*˜R“. DÑ(ˆô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r+   c                   ó  ‡ — e Zd ZdZ	 	 	 	 	 	 ddedededededed	edz  d
edz  fˆ fd„Z	 	 	 dde	j                  de	j                  dz  dedz  de	j                  dz  dee   dee	j                  e	j                  dz  f   fd„Zˆ xZS )ÚBioGptAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNÚ	embed_dimÚ	num_headsrN   Ú
is_decoderÚbiasÚ	is_causalÚconfigÚ	layer_idxc	                 óŽ  •— t         ‰	| �  «        || _        || _        || _        ||z  | _        || _        | j
                  |z  | j                  k7  rt        d| j                  › d|› d�«      ‚| j
                  dz  | _        || _	        || _
        || _        |€9| j                  r-t        j                  d| j                  j                  › d�«       t!        j"                  |||¬«      | _        t!        j"                  |||¬«      | _        t!        j"                  |||¬«      | _        t!        j"                  |||¬«      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).rR   zInstantiating a decoder z¸ without passing `layer_idx` is not recommended and will lead to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.©re   )r&   r'   rb   rc   rN   Úhead_dimrg   Ú
ValueErrorrM   rd   rf   rh   ÚloggerÚwarning_oncer)   r6   rY   ÚLinearÚk_projÚv_projÚq_projÚout_proj)
r(   rb   rc   rN   rd   re   rf   rg   rh   r)   s
            €r*   r'   zBioGptAttention.__init__{   s$  ø€ ô 	‰ÑÔØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØˆŒà�M‰M˜IÑ%¨$¯.©.Ò8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÑ*ˆŒØ$ˆŒØ"ˆŒØ"ˆŒØÐ §¢Ü×ÑØ*¨4¯>©>×+BÑ+BÐ*Cð D,ð ,ôô —i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜŸ	™	 )¨Y¸TÔBˆ�r+   Úhidden_statesÚkey_value_statesÚpast_key_valuesr,   rO   Úreturnc                 óØ  — |du}|j                   dd }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	d}
|�St        |t        «      rA|j                  j                  | j                  «      }
|r|j                  }n|j                  }n|}|r|n|}|rK|�I|
rGj                  | j                     j                  }|j                  | j                     j                  }nÑ| j                  |«      }| j!                  |«      }g |j                   dd ¢d‘| j                  ‘­}|j                  |«      j	                  dd«      }|j                  |«      j	                  dd«      }|�Kj#                  ||| j                  «      \  }}|r)t        |t        «      rd|j                  | j                  <   t%        j&                  | j(                  j*                  t,        «      } || |	|||f| j.                  sdn| j0                  | j2                  dœ|¤Ž\  }} |j4                  g |¢d‘­Ž j7                  «       }| j9                  |«      }||fS )	z#Input shape: Batch x Time x ChannelNrQ   r   r$   FTç        )rN   rM   )Úshaperk   rr   ÚviewrX   Ú
isinstancer   Ú
is_updatedÚgetrh   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesrp   rq   Úupdater   Úget_interfacerg   Ú_attn_implementationr_   rU   rN   rM   Úreshaper\   rs   )r(   rt   ru   rv   r,   rO   Úis_cross_attentionÚinput_shapeÚhidden_shapeÚquery_statesr}   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shapeÚattention_interfacer^   r]   s                      r*   r5   zBioGptAttention.forward¢   sd  € ð .°TÐ9Ðð $×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆð —{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàˆ
ØÐ&Ü˜/Ô+>Ô?Ø,×7Ñ7×;Ñ;¸D¿N¹NÓK�
Ù%à+:×+PÑ+PÑ(à+:×+OÑ+OÑ(à'6Ð$á-?Ñ)À]ˆÙ /Ð"=Á*à-×4Ñ4°T·^±^ÑD×IÑIˆJØ/×6Ñ6°t·~±~ÑF×MÑM‰LàŸ™ ^Ó4ˆJØŸ;™; ~Ó6ˆLØF˜×-Ñ-¨c¨rÐ2ÐF°BÐF¸¿¹ÑFˆHØ#Ÿ™¨Ó2×<Ñ<¸QÀÓBˆJØ'×,Ñ,¨XÓ6×@Ñ@ÀÀAÓFˆLàÐ*Ø+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜Lá%¬*°_ÔFYÔ*ZØAE�O×.Ñ.¨t¯~©~Ñ>ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$·,±,Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m KÓ0ˆà˜LÐ(Ð(r+   )ry   FTFNN)NNN)r6   r7   r8   r9   r:   rG   Úboolr   r'   r2   rH   r	   r   r   Útupler5   r<   r=   s   @r*   ra   ra   x   s  ø„ ÙGð Ø ØØØ&*Ø $ñ%Càð%Cð ð%Cð ð	%Cð
 ð%Cð ð%Cð ð%Cð ˜tÑ#ð%Cð ˜‘:õ%CðT 15Ø(,Ø.2ñH)à—|‘|ðH)ð  Ÿ,™,¨Ñ-ðH)ð  ™ð	H)ð
 Ÿ™ tÑ+ðH)ð Ð-Ñ.ðH)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷H)r+   ra   c                   óÌ   ‡ — e Zd Zddededz  fˆ fd„Z	 	 	 	 ddej                  dej                  dz  dedz  de	dz  d	ej                  dz  d
ee   dej                  fd„Zˆ xZS )ÚBioGptDecoderLayerNrg   rh   c           	      ón  •— t         ‰| �  «        |j                  | _        t	        | j                  |j
                  |j                  dd||¬«      | _        |j                  | _	        t        |j                     | _        |j                  | _        t        j                  | j                  «      | _        t        j"                  | j                  |j$                  «      | _        t        j"                  |j$                  | j                  «      | _        t        j                  | j                  «      | _        y )NT)rb   rc   rN   rd   rf   rg   rh   )r&   r'   Úhidden_sizerb   ra   Únum_attention_headsÚattention_probs_dropout_probÚ	self_attnÚhidden_dropout_probrN   r   Ú
hidden_actÚactivation_fnÚactivation_dropoutrY   Ú	LayerNormÚself_attn_layer_normro   Úintermediate_sizeÚfc1Úfc2Úfinal_layer_norm)r(   rg   rh   r)   s      €r*   r'   zBioGptDecoderLayer.__init__î   sÞ   ø€ Ü‰ÑÔØ×+Ñ+ˆŒä(Ø—n‘nØ×0Ñ0Ø×7Ñ7ØØØØô
ˆŒð ×1Ñ1ˆŒÜ# F×$5Ñ$5Ñ6ˆÔØ"(×";Ñ";ˆÔä$&§L¡L°·±Ó$@ˆÔ!ä—9‘9˜TŸ^™^¨V×-EÑ-EÓFˆŒÜ—9‘9˜V×5Ñ5°t·~±~ÓFˆŒÜ "§¡¨T¯^©^Ó <ˆÕr+   rt   r,   rv   Ú	use_cacher.   rO   rw   c                 óD  — |}| j                  |«      } | j                  d||||dœ|¤Ž\  }}t        j                  j	                  || j                  | j
                  ¬«      }||z   }|}| j                  |«      }| j                  |«      }| j                  |«      }t        j                  j	                  || j                  | j
                  ¬«      }| j                  |«      }t        j                  j	                  || j                  | j
                  ¬«      }||z   }|S )a‘  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            past_key_values (`Cache`): cached past key and value projection states
        )rt   rv   r,   r.   rS   © )r    rš   rY   rZ   rN   rU   r¤   r¢   r�   rž   r£   )	r(   rt   r,   rv   r¥   r.   rO   ÚresidualÚ_s	            r*   r5   zBioGptDecoderLayer.forward  s  € ð  !ˆà×1Ñ1°-Ó@ˆð *˜4Ÿ>™>ð 
Ø'Ø+Ø)Ø%ñ	
ð
 ñ
Ñˆ�qô Ÿ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ  =Ñ0ˆð !ˆØ×-Ñ-¨mÓ<ˆØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆÜŸ™×-Ñ-¨m¸t×?VÑ?VÐae×anÑanÐ-ÓoˆØŸ™ Ó/ˆÜŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ  =Ñ0ˆàÐr+   rC   )NNTN)r6   r7   r8   r   r:   r'   r2   rH   r	   r’   r;   r   r   r5   r<   r=   s   @r*   r•   r•   í   s›   ø„ ñ=˜|ð =¸¸d¹
õ =ð4 /3Ø(,Ø!%Ø04ñ)à—|‘|ð)ð Ÿ™ tÑ+ð)ð  ™ð	)ð
 ˜$‘;ð)ð ×&Ñ&¨Ñ-ð)ð Ð+Ñ,ð)ð 
�‰÷)r+   r•   c                   ó:   — e Zd ZU eed<   dZdZdZdZdZ	dZ
eedœZy)ÚBioGptPreTrainedModelrg   ÚbiogptT)rt   Ú
attentionsN)r6   r7   r8   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphr•   ra   Ú_can_record_outputsr§   r+   r*   r«   r«   1  s9   … àÓØ ÐØ&*Ð#ØÐØ€NØÐØ!Ðà+Ø%ñÑr+   r«   c                   óø   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dedz  d	ej                  dz  d
ee   deez  fd„«       «       «       Zˆ xZS )ÚBioGptModelrg   c           	      óÎ  •— t         ‰| �  |«       || _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        |j                  rt        j                  |j                  «      nd}t        |j                  | j                  | j                  |¬«      | _        t!        |j"                  | j                  «      | _        t'        j(                  t+        |j,                  «      D �cg c]  }t/        ||¬«      ‘Œ c}«      | _        t'        j2                  | j                  «      | _        d| _        | j9                  «        y c c}w )NrF   )rA   )rh   F)r&   r'   rg   Ú	layerdropr›   rN   r—   rb   Úpad_token_idr@   Úscale_embeddingÚmathÚsqrtr?   Ú
vocab_sizeÚembed_tokensr    Úmax_position_embeddingsÚembed_positionsrY   Ú
ModuleListÚrangeÚnum_hidden_layersr•   r�   rŸ   Ú
layer_normÚgradient_checkpointingÚ	post_init)r(   rg   rA   Úir)   s       €r*   r'   zBioGptModel.__init__B  s  ø€ Ü‰Ñ˜Ô ØˆŒØ×)Ñ)ˆŒØ×1Ñ1ˆŒØ×+Ñ+ˆŒØ!×.Ñ.ˆÔØ7=×7MÒ7M”d—i‘i × 2Ñ 2Ô3ÐSVˆä5Ø×Ñ˜tŸ~™~¨t×/?Ñ/?È[ô
ˆÔô  @À×@^Ñ@^Ð`d×`nÑ`nÓoˆÔä—m‘mÔV[Ð\b×\tÑ\tÓVuÖ$vÐQRÔ%7¸È!Ö%LÒ$vÓwˆŒÜŸ,™, t§~¡~Ó6ˆŒà&+ˆÔ#à�‰Õùò %ws   ÄE"NrD   r,   Úinputs_embedsrv   r¥   r.   rO   rw   c           	      ó`  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|j	                  «       d d \  }}	|�|j                  «       nd}
|€'|
|	z   }t        j                  |||j                  ¬«      }|}t        | j                  |||¬«      }|€5t        j                  |	|j                  ¬«      |
z   }|j                  d«      }| j                  ||
|¬«      }||z   }t        j                  j                  || j                  | j                   ¬«      }t#        | j$                  «      D ]D  \  }}| j                   r%t        j&                  g «      }|| j(                  k  rŒ7 ||f||||d	œ|¤Ž}ŒF | j+                  |«      }t-        ||¬
«      S )NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time)rg   rQ   r   ©Údevice)rg   rÉ   r,   rv   )r.   rS   )r,   rv   r¥   r.   )Úlast_hidden_staterv   )rl   r¿   r
   rg   rV   Úget_seq_lengthr2   ÚonesrÌ   r   ÚarangeÚ	unsqueezerÁ   rY   rZ   rN   rU   Ú	enumerater�   Úrandr¹   rÅ   r   )r(   rD   r,   rÉ   rv   r¥   r.   rO   Ú
batch_sizeÚ
seq_lengthr-   Úmask_seq_lengthÚself_attn_cacheÚcausal_maskÚ	positionsrt   ÚidxÚdecoder_layerÚdropout_probabilitys                      r*   r5   zBioGptModel.forwardW  sÑ  € ð ˜Ð -°tÐ";Ò<ÜÐsÓtÐtàÐ Ø ×-Ñ-¨iÓ8ˆMñ ˜Ð0Ü*°$·+±+Ô>ˆOà!.×!3Ñ!3Ó!5°c°rÐ!:Ñˆ
�JØETÐE` ×!?Ñ!?Ô!AÐfgÐàÐ!à4°zÑAˆOÜ"ŸZ™Z¨
°OÈM×L`ÑL`ÔaˆNà)ˆä(Ø—;‘;Ø'Ø)Ø+ô	
ˆð ÐÜ Ÿ<™<¨
¸=×;OÑ;OÔPÐSiÑiˆLØ'×1Ñ1°!Ó4ˆLà×(Ñ(¨Ð9OÐ^jÐ(Ókˆ	Ø%¨	Ñ1ˆÜŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-Ódˆä"+¨D¯K©KÓ"8ò 	ÑˆC�Ø�}Š}Ü&+§j¡j°£nÐ#Ø&¨¯©Ò7Øá)Øðà*Ø /Ø#Ø)ñð ñ‰Mð	ð Ÿ™¨Ó6ˆä8Ø+Ø+ô
ð 	
r+   )NNNNNN)r6   r7   r8   r   r'   r   r   r   r2   r;   ÚFloatTensorr	   r’   r   r   r“   r   r5   r<   r=   s   @r*   r·   r·   @  sß   ø„ ð˜|õ ð*  ØØð .2Ø37Ø26Ø(,Ø!%Ø04ñB
à×#Ñ# dÑ*ðB
ð ×)Ñ)¨DÑ0ðB
ð ×(Ñ(¨4Ñ/ð	B
ð
  ™ðB
ð ˜$‘;ðB
ð ×&Ñ&¨Ñ-ðB
ð Ð+Ñ,ðB
ð 
Ð:Ñ	:òB
ó ó ó  ôB
r+   r·   zR
    BioGPT Model with a `language modeling` head on top for CLM fine-tuning.
    )Úcustom_introc                   ó<  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  d	e	j                  dz  d
edz  de	j                  dz  dedz  de	j                  dz  dee	j                  z  dee   deez  fd„«       «       Zˆ xZS )ÚBioGptForCausalLMzoutput_projection.weightzbiogpt.embed_tokens.weightc                 óÆ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        | j                  «        y ©NFrj   )
r&   r'   r·   r¬   rY   ro   r—   r¾   Úoutput_projectionrÇ   ©r(   rg   r)   s     €r*   r'   zBioGptForCausalLM.__init__§  sJ   ø€ Ü‰Ñ˜Ô ä! &Ó)ˆŒÜ!#§¡¨6×+=Ñ+=¸v×?PÑ?PÐW\Ô!]ˆÔð 	�‰Õr+   c                 ó   — | j                   S rC   ©rã   ©r(   s    r*   Úget_output_embeddingsz'BioGptForCausalLM.get_output_embeddings°  s   € Ø×%Ñ%Ð%r+   c                 ó   — || _         y rC   ræ   )r(   Únew_embeddingss     r*   Úset_output_embeddingsz'BioGptForCausalLM.set_output_embeddings³  s
   € Ø!/ˆÕr+   NrD   r,   rÉ   rv   Úlabelsr¥   r.   Úlogits_to_keeprO   rw   c	           	      ó€  —  | j                   |f|||||dœ|	¤Ž}
|
d   }t        |t        «      rt        | d«      n|}| j	                  |dd…|dd…f   «      }d}|�* | j
                  d||| j                  j                  dœ|	¤Ž}t        |||
j                  |
j                  |
j                  |
j                  ¬«      S )a³  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        )r,   rÉ   rv   r¥   r.   r   N)Úlogitsrì   r¾   )Úlossrï   rv   rt   r­   Úcross_attentionsr§   )r¬   r|   r:   Úslicerã   Úloss_functionrg   r¾   r   rv   rt   r­   rñ   )r(   rD   r,   rÉ   rv   rì   r¥   r.   rí   rO   Úoutputsrt   Úslice_indicesrï   rð   s                  r*   r5   zBioGptForCausalLM.forward¶  sâ   € ð( �$—+‘+Øð
à)Ø'Ø+ØØ%ñ
ð ñ
ˆð   ™
ˆÜ8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ×'Ñ'¨²a¸ÊÐ6IÑ(JÓKˆàˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
r+   ©NNNNNNNr   )r6   r7   r8   Ú_tied_weights_keysr'   rè   rë   r   r   r2   r;   rÝ   r	   r’   r:   rH   r   r   r“   r   r5   r<   r=   s   @r*   rà   rà   Ÿ  s  ø„ ð 5Ð6RÐSÐôò&ò0ð Øð .2Ø37Ø26Ø(,Ø*.Ø!%Ø04Ø-.ñ+
à×#Ñ# dÑ*ð+
ð ×)Ñ)¨DÑ0ð+
ð ×(Ñ(¨4Ñ/ð	+
ð
  ™ð+
ð × Ñ  4Ñ'ð+
ð ˜$‘;ð+
ð ×&Ñ&¨Ñ-ð+
ð ˜eŸl™lÑ*ð+
ð Ð+Ñ,ð+
ð 
Ð2Ñ	2ò+
ó ó ô+
r+   rà   c                   ó  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de	dz  dej                  dz  dej                  dz  d	e
dz  d
ej                  dz  deez  fd„«       «       Zˆ xZS )ÚBioGptForTokenClassificationc                 óz  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        |d«      r|j                  �|j                  }n|j                  }t        j                  |«      | _
        t        j                  |j                  |j                  «      | _        | j                  «        y )NÚclassifier_dropout)r&   r'   Ú
num_labelsr·   r¬   Úhasattrrû   r›   rY   ÚDropoutrN   ro   r—   Ú
classifierrÇ   )r(   rg   rû   r)   s      €r*   r'   z%BioGptForTokenClassification.__init__è  s�   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &Ó)ˆŒÜ�6Ð/Ô0°V×5NÑ5NÐ5ZØ!'×!:Ñ!:Ñà!'×!;Ñ!;ÐÜ—z‘zÐ"4Ó5ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒà�‰Õr+   NrD   Útoken_type_idsr,   rv   rÉ   rì   r¥   r.   rw   c	           	      ód  —  | j                   |f|||||dœ|	¤Ž}
|
d   }| j                  |«      }| j                  |«      }d}|�Êt        «       }|�Œ|j	                  d«      dk(  }|j	                  d| j
                  «      }t        j                  ||j	                  d«      t        j                  |j                  «      j                  |«      «      } |||«      }n2 ||j	                  d| j
                  «      |j	                  d«      «      }t        |||
j                  |
j                  ¬«      S )á�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        ©rv   r,   rÉ   r¥   r.   r   NrQ   r   )rð   rï   rt   r­   )r¬   rN   rÿ   r   r{   rü   r2   ÚwhereÚtensorÚignore_indexÚtype_asr   rt   r­   )r(   rD   r   r,   rv   rÉ   rì   r¥   r.   rO   Útransformer_outputsrt   rï   rð   Úloss_fctÚactive_lossÚactive_logitsÚactive_labelss                     r*   r5   z$BioGptForTokenClassification.forwardö  s.  € ð( *˜dŸk™kØð
à+Ø)Ø'ØØ%ñ
ð ñ
Ðð ,¨AÑ.ˆØŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÐÜ'Ó)ˆHØÐ)Ø,×1Ñ1°"Ó5¸Ñ:�Ø &§¡¨B°·±Ó @�Ü %§¡Ø §¡¨R£´%·,±,¸x×?TÑ?TÓ2U×2]Ñ2]Ð^dÓ2eó!�ñ   ¨}Ó=‘á §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR�ä$ØØØ-×;Ñ;Ø*×5Ñ5ô	
ð 	
r+   )NNNNNNNN)r6   r7   r8   r'   r   r   r2   r;   rÝ   r	   r’   r“   r   r5   r<   r=   s   @r*   rù   rù   æ  sæ   ø„ ôð Øð .2Ø26Ø37Ø(,Ø26Ø*.Ø!%Ø04ñ2
à×#Ñ# dÑ*ð2
ð ×(Ñ(¨4Ñ/ð2
ð ×)Ñ)¨DÑ0ð	2
ð
  ™ð2
ð ×(Ñ(¨4Ñ/ð2
ð × Ñ  4Ñ'ð2
ð ˜$‘;ð2
ð ×&Ñ&¨Ñ-ð2
ð 
Ð&Ñ	&ò2
ó ó ô2
r+   rù   aÛ  
    The BioGpt Model transformer with a sequence classification head on top (linear layer).

    [`BioGptForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it is required to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   ó0  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  de
dz  dej                  dz  dej                  dz  d	edz  d
ej                  dz  deej                  z  deez  fd„«       «       Zd„ Zd„ Zˆ xZS )ÚBioGptForSequenceClassificationrg   c                 óè   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  | j                  d¬«      | _        | j                  «        y râ   )
r&   r'   rü   r·   r¬   rY   ro   r—   ÚscorerÇ   rä   s     €r*   r'   z(BioGptForSequenceClassification.__init__<  sS   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ! &Ó)ˆŒÜ—Y‘Y˜v×1Ñ1°4·?±?ÈÔOˆŒ
ð 	�‰Õr+   NrD   r,   rv   rÉ   rì   r¥   r.   rí   rw   c	           	      óÄ  —  | j                   |f|||||dœ|	¤Ž}
|
d   }t        |t        «      rt        | d«      n|}| j	                  |dd…|dd…f   «      }|�|j
                  dd \  }}n|j
                  dd \  }}| j                  j                  €|dk7  rt        d«      ‚| j                  j                  €d}nÃ|�“|| j                  j                  k7  j                  |j                  t        j                  «      }t        j                  |j
                  d   |j                  t        j                  ¬«      }||z  j                  d«      }n.d}t        j!                  | j"                  j$                  › d	�«       |t        j                  ||j                  ¬
«      |f   }d}|��‡| j                  j&                  €�| j(                  dk(  rd| j                  _        nl| j(                  dkD  rL|j*                  t        j,                  k(  s|j*                  t        j                  k(  rd| j                  _        nd| j                  _        | j                  j&                  dk(  rIt/        «       }| j(                  dk(  r& ||j1                  «       |j1                  «       «      }nŒ |||«      }n‚| j                  j&                  dk(  r=t3        «       } ||j5                  d| j(                  «      |j5                  d«      «      }n,| j                  j&                  dk(  rt7        «       } |||«      }t9        |||
j:                  |
j<                  |
j>                  ¬«      S )r  r  r   Nr$   r   z=Cannot handle batch sizes > 1 if no padding token is defined.rQ   )rÌ   ÚdtypezŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rË   Ú
regressionÚsingle_label_classificationÚmulti_label_classification)rð   rï   rv   rt   r­   ) r¬   r|   r:   rò   r  rz   rg   rº   rl   ÚtorÌ   r2   Úint32rÐ   Úargmaxrm   rn   r)   r6   Úproblem_typerü   r  r4   r   Úsqueezer   r{   r   r   rv   rt   r­   )r(   rD   r,   rv   rÉ   rì   r¥   r.   rí   rO   r  rt   rõ   rï   rÔ   Úsequence_lengthÚlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsrð   r	  s                         r*   r5   z'BioGptForSequenceClassification.forwardE  sû  € ð( *˜dŸk™kØð
à+Ø)Ø'ØØ%ñ
ð ñ
Ðð ,¨AÑ.ˆÜ8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜Mª!¨]ºAÐ*=Ñ>Ó?ˆàÐ Ø*3¯/©/¸"¸1Ð*=Ñ'ˆJ™à*7×*=Ñ*=¸b¸qÐ*AÑ'ˆJ˜à�;‰;×#Ñ#Ð+°
¸a²ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ð+Ø!#ÑØÐ"à%¨¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸL™L¨¯©¸Ñ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Ñ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|™|¨J¸v¿}¹}ÔMÐOaÐaÑbˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# M×$9Ñ$9Ó$;¸V¿^¹^Ó=MÓN‘Dá# M°6Ó:‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù × 2Ñ 2°2°t·±Ó GÈÏÉÐUWËÓY‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨vÓ6�ä/ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r+   c                 ó.   — | j                   j                  S rC   ©r¬   r¿   rç   s    r*   Úget_input_embeddingsz4BioGptForSequenceClassification.get_input_embeddingsœ  s   € Ø�{‰{×'Ñ'Ð'r+   c                 ó&   — || j                   _        y rC   r!  )r(   rL   s     r*   Úset_input_embeddingsz4BioGptForSequenceClassification.set_input_embeddingsŸ  s   € Ø#(ˆ�‰Õ r+   rö   )r6   r7   r8   r   r'   r   r   r2   r;   rÝ   r	   r’   r:   rH   r“   r   r5   r"  r$  r<   r=   s   @r*   r  r  -  s  ø„ ð˜|õ ð Øð .2Ø37Ø(,Ø26Ø*.Ø!%Ø04Ø-.ñS
à×#Ñ# dÑ*ðS
ð ×)Ñ)¨DÑ0ðS
ð  ™ð	S
ð
 ×(Ñ(¨4Ñ/ðS
ð × Ñ  4Ñ'ðS
ð ˜$‘;ðS
ð ×&Ñ&¨Ñ-ðS
ð ˜eŸl™lÑ*ðS
ð 
Ð1Ñ	1òS
ó ó ðS
òj(ö)r+   r  )rà   rù   r  r·   r«   )Nry   )>r¼   Úcollections.abcr   r2   Útorch.nnrY   r   r   r   Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_biogptr   Ú
get_loggerr6   rm   Ú	Embeddingr    r?   ÚModulerH   rG   r_   ra   r•   r«   r·   rà   rù   r  Ú__all__r§   r+   r*   ú<module>r8     sØ  ðó* Ý $ã Ý ß AÑ Aå !ß CÑ CÝ )Ý /Ý BÝ 9÷ó ÷ GÝ &ß RÓ RÝ 7Ý 5Ý .ð 
ˆ×	Ñ	˜HÓ	%€ô; r§|¡|ô ;ô8
= §¡ô 
=ð& !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(ó%ô8r)�b—i‘iô r)ôjAÐ3ô AðH ô˜Oó ó ðð ô[
Ð'ó [
ó ð[
ñ| ðôô
?
Ð-¨ó ?
óð
?
ðD ôC
Ð#8ó C
ó ðC
ñL ðôôe)Ð&;ó e)óðe)òP�r+   