Ë
    ÜÓ^jK#  ã                   óº  — d Z ddlmZ ddlZddlmZ ddlmZ ddlmZm	Z	 ddl
mZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z&  ejN                  e(«      Z) G d„ dejT                  «      Z+ G d„ de«      Z, G d„ de«      Z- G d„ de"«      Z. G d„ de«      Z/ G d„ de «      Z0 G d „ d!e!«      Z1g d"¢Z2y)#zPyTorch Starcoder2 model.é    )ÚCallableN)Únné   )ÚACT2FN)ÚCacheÚDynamicCache)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚMistralAttentionÚMistralDecoderLayerÚMistralForCausalLMÚ MistralForSequenceClassificationÚMistralForTokenClassificationÚMistralModelÚapply_rotary_pos_embÚeager_attention_forwardé   )ÚStarcoder2Configc                   óh   ‡ — e Zd Zdefˆ fd„Zdeej                     dz  dej                  fd„Zˆ xZ	S )ÚStarcoder2MLPÚconfigc                 óP  •— t         ‰| �  «        |j                  }t        j                  ||j
                  |j                  ¬«      | _        t        j                  |j
                  ||j                  ¬«      | _        t        |j                     | _        |j                  | _        y )N©Úbias)ÚsuperÚ__init__Úhidden_sizer   ÚLinearÚintermediate_sizeÚuse_biasÚc_fcÚc_projr   Ú
hidden_actÚactÚresidual_dropout)Úselfr    Ú	embed_dimÚ	__class__s      €ú|/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/starcoder2/modular_starcoder2.pyr%   zStarcoder2MLP.__init__5   su   ø€ Ü‰ÑÔØ×&Ñ&ˆ	Ü—I‘I˜i¨×)AÑ)AÈÏÉÔXˆŒ	Ü—i‘i × 8Ñ 8¸)È&Ï/É/ÔZˆŒÜ˜&×+Ñ+Ñ,ˆŒØ &× 7Ñ 7ˆÕó    Úhidden_statesNÚreturnc                 óØ   — | j                  |«      }| j                  |«      }| j                  |«      }t        j                  j                  || j                  | j                  ¬«      }|S )N©ÚpÚtraining)r*   r-   r+   r   Ú
functionalÚdropoutr.   r9   )r/   r4   s     r2   ÚforwardzStarcoder2MLP.forward=   sZ   € ØŸ	™	 -Ó0ˆØŸ™ Ó/ˆØŸ™ MÓ2ˆÜŸ™×-Ñ-¨m¸t×?TÑ?TÐ_c×_lÑ_lÐ-ÓmˆØÐr3   )
Ú__name__Ú
__module__Ú__qualname__r   r%   ÚtupleÚtorchÚFloatTensorr<   Ú__classcell__©r1   s   @r2   r   r   4   s9   ø„ ð8Ð/õ 8ð U¨5×+<Ñ+<Ñ%=ÀÑ%Dð È×IZÑIZ÷ r3   r   c                   ó  ‡ — e Zd Zddededz  fˆ fd„Z	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	e
e   d
eej                  ej                  dz  eej                     dz  f   fd„Zˆ xZS )ÚStarcoder2AttentionNr    Ú	layer_idxc                 óŠ  •— t         ‰| �  ||¬«       |j                  | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _	        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                  ¬«      | _        y )N©r    rG   r"   )r$   r%   r.   r   r'   r&   Únum_attention_headsÚhead_dimr)   Úq_projÚnum_key_value_headsÚk_projÚv_projÚo_proj©r/   r    rG   r1   s      €r2   r%   zStarcoder2Attention.__init__F   sì   ø€ Ü‰Ñ °)ÐÔ<Ø &× 7Ñ 7ˆÔÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^Ðek×etÑetÔuˆ�r3   r4   Úposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsr5   c           
      ó¢  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  sdn| j                   | j"                  t%        | j                  dd «      dœ|¤Ž\  }} |j&                  g |¢d‘­Ž j)                  «       }| j+                  |«      }t,        j.                  j1                  || j2                  | j                  ¬«      }||fS )Néÿÿÿÿr   r   g        Úsliding_window)r;   ÚscalingrX   r7   )ÚshaperK   rL   ÚviewÚ	transposerN   rO   r   ÚupdaterG   r   Úget_interfacer    Ú_attn_implementationr   r9   Úattention_dropoutrY   ÚgetattrÚreshapeÚ
contiguousrP   r   r:   r;   r.   )r/   r4   rR   rS   rT   rU   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   r2   r<   zStarcoder2Attention.forwardN   sÎ  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LÜ" 4§;¡;Ð0@À$ÓGñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆÜ—m‘m×+Ñ+Ø˜4×0Ñ0¸4¿=¹=ð ,ó 
ˆð ˜LÐ(Ð(r3   )N)r=   r>   r?   r   Úintr%   rA   ÚTensorr@   r   r   r   r<   rC   rD   s   @r2   rF   rF   E   s²   ø„ ñvÐ/ð v¸CÀ$¹Jõ vð )-ñ+)à—|‘|ð+)ð # 5§<¡<°·±Ð#=Ñ>ð+)ð Ÿ™ tÑ+ð	+)ð
  ™ð+)ð Ð-Ñ.ð+)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷+)r3   rF   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚStarcoder2DecoderLayerr    rG   c                 ó*  •— t         ‰| �  ||«       t        ||¬«      | _        t	        |«      | _        t        j                  |j                  |j                  ¬«      | _
        t        j                  |j                  |j                  ¬«      | _        y )NrI   ©Úeps)r$   r%   rF   Ú	self_attnr   Úmlpr   Ú	LayerNormr&   Únorm_epsilonÚinput_layernormÚpost_attention_layernormrQ   s      €r2   r%   zStarcoder2DecoderLayer.__init__}   sj   ø€ Ü‰Ñ˜ Ô+Ü,°FÀiÔPˆŒÜ  Ó(ˆŒÜ!Ÿ|™|¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ(*¯©°V×5GÑ5GÈV×M`ÑM`Ô(aˆÕ%r3   )r=   r>   r?   r   rn   r%   rC   rD   s   @r2   rq   rq   |   s    ø„ ðbÐ/ð b¸C÷ bñ br3   rq   c                   óî   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de
dz  dej                  dz  d	edz  d
ee   deez  fd„«       «       Zˆ xZS )ÚStarcoder2Modelr    c           	      ó:  •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        j                  |j                  |j                  ¬«      | _        |j                  | _        y c c}w )Nrs   )r$   r%   r   Ú
ModuleListÚrangeÚnum_hidden_layersrq   Úlayersrw   r&   rx   ÚnormÚembedding_dropoutrQ   s      €r2   r%   zStarcoder2Model.__init__†   su   ø€ Ü‰Ñ˜Ô Ü—m‘mÜHMÈf×NfÑNfÓHgÖh¸9Ô# F¨IÕ6Òhó
ˆŒô —L‘L ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ø!'×!9Ñ!9ˆÕùò is   ·BNÚ	input_idsrS   Úposition_idsrT   Úinputs_embedsÚ	use_cacherU   r5   c           
      óø  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }| j                  j                  €t        nt        }	 |	| j                  ||||¬«      }
|}t        j                  j                  || j                   | j"                  ¬«      }| j%                  ||¬«      }| j&                  d | j                  j(                   D ]  } ||f|
||||d	œ|¤Ž}Œ | j+                  |«      }t-        ||r|¬
«      S d ¬
«      S )Nz:You must specify exactly one of input_ids or inputs_embeds)r    r   r   )Údevice)r    r†   rS   rT   r…   r7   )r…   )rS   r…   rT   r‡   rR   )Úlast_hidden_staterT   )Ú
ValueErrorÚembed_tokensr   r    Úget_seq_lengthrA   ÚarangerZ   r‰   Ú	unsqueezerX   r	   r
   r   r:   r;   rƒ   r9   Ú
rotary_embr�   r€   r‚   r   )r/   r„   rS   r…   rT   r†   r‡   rU   Úpast_seen_tokensÚmask_functionÚcausal_maskr4   rR   Údecoder_layers                 r2   r<   zStarcoder2Model.forwardŽ   sŸ  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÐ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆÜŸ™×-Ñ-Ø˜T×3Ñ3¸d¿m¹mð .ó 
ˆð #Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 		ˆMÙ)Øðà*Ø)Ø /Ø#Ø$7ñð ñ‰Mð		ð Ÿ	™	 -Ó0ˆä&Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
r3   )NNNNNN)r=   r>   r?   r   r%   r   r   rA   Ú
LongTensorro   r   rB   Úboolr   r   r@   r   r<   rC   rD   s   @r2   r|   r|   …   sÌ   ø„ ð:Ð/õ :ð  Øð .2Ø.2Ø04Ø(,Ø26Ø!%ñ7
à×#Ñ# dÑ*ð7
ð Ÿ™ tÑ+ð7
ð ×&Ñ&¨Ñ-ð	7
ð
  ™ð7
ð ×(Ñ(¨4Ñ/ð7
ð ˜$‘;ð7
ð Ð+Ñ,ð7
ð 
Ð(Ñ	(ò7
ó ó  ô7
r3   r|   c                   ó   — e Zd Zy)ÚStarcoder2ForCausalLMN©r=   r>   r?   © r3   r2   r˜   r˜   Ê   ó   „ Ør3   r˜   c                   ó   — e Zd Zy)Ú#Starcoder2ForSequenceClassificationNr™   rš   r3   r2   r�   r�   Î   r›   r3   r�   c                   ó   — e Zd Zy)Ú Starcoder2ForTokenClassificationNr™   rš   r3   r2   rŸ   rŸ   Ò   r›   r3   rŸ   )r˜   r|   ÚStarcoder2PreTrainedModelr�   rŸ   )3Ú__doc__Úcollections.abcr   rA   r   Úactivationsr   Úcache_utilsr   r   Úmasking_utilsr	   r
   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   Úutils.output_capturingr   Úmistral.modeling_mistralr   r   r   r   r   r   r   r   Úconfiguration_starcoder2r   Ú
get_loggerr=   ÚloggerÚModuler   rF   rq   r|   r˜   r�   rŸ   Ú__all__rš   r3   r2   ú<module>r³      sÃ   ðñ&  å $ã Ý å !ß .ß RÝ BÝ 7Ý 5Ý &ß 0Ý 7Ý 5÷	÷ 	ó 	õ 7ð 
ˆ×	Ñ	˜HÓ	%€ô�B—I‘Iô ô"4)Ð*ô 4)ônbÐ0ô bôB
�lô B
ôJ	Ð.ô 	ô	Ð*Jô 	ô	Ð'Dô 	ò�r3   