Ë
    ÛÓ^jÚO  ã                   óv  — d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z
mZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZ ddlmZ  ej8                  e«      Zdededej@                  fd„Z!dej@                  dej@                  fd„Z"dej@                  dej@                  dej@                  dej@                  fd„Z# G d„ dejH                  «      Z% G d„ dejH                  «      Z& G d„ de«      Z'e G d „ d!e«      «       Z(e G d"„ d#e(«      «       Z) ed$¬%«       G d&„ d'e(e«      «       Z*g d(¢Z+y))zPyTorch CodeGen model.é    N)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚPreTrainedModel)Úauto_docstringÚloggingé   )ÚCodeGenConfigÚnum_posÚdimÚreturnc                 óŒ  — ddt        j                  d|dt         j                  ¬«      |z  z  z  }t        j                  dt        j                  | t         j                  ¬«      j	                  «       |«      j	                  «       }t        j
                  t        j                  |«      t        j                  |«      fd¬«      S )	Ng      ð?i'  r   é   )Údtypezi , j -> i jr   ©r   )ÚtorchÚarangeÚint64ÚeinsumÚfloatÚcatÚsinÚcos)r   r   Úinv_freqÚsinusoid_inps       úw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/codegen/modeling_codegen.pyÚcreate_sinusoidal_positionsr%   (   s…   € Ø�e¤§¡¨Q°°Q¼e¿k¹kÔ JÈSÑ PÑQÑR€HÜ—<‘< ´·±¸WÌEÏKÉKÔ0X×0^Ñ0^Ó0`ÐbjÓk×qÑqÓs€LÜ�9‰9”e—i‘i Ó-¬u¯y©y¸Ó/FÐGÈQÔOÐOó    Úxc                 ó    — | d d …d d …d d …d d d…f   }| d d …d d …d d …dd d…f   }t        j                  | |fd¬«      } | j                  d«      S )Nr   r   éÿÿÿÿr   éþÿÿÿ)r   ÚstackÚflatten)r'   Úx1Úx2s      r$   Úrotate_every_twor/   /   sS   € Ø	
Š1Ša’‘C�a�Cˆ<‰€BØ	
Š1Ša’�A�D�q�Dˆ=Ñ	€BÜ�‰�b�S˜"�I 2Ô&€AØ�9‰9�R‹=Ðr&   Útensorr    r!   c                 óº   — t        j                  |d d …d d …d d d …f   dd«      }t        j                  |d d …d d …d d d …f   dd«      }| |z  t        | «      |z  z   S )Nr   r   )r   Úrepeat_interleaver/   )r0   r    r!   s      r$   Úapply_rotary_pos_embr3   7   s^   € Ü
×
!Ñ
! #¢aª¨D²! mÑ"4°a¸Ó
;€CÜ
×
!Ñ
! #¢aª¨D²! mÑ"4°a¸Ó
;€CØ�S‰LÔ-¨fÓ5¸Ñ;Ñ<Ð<r&   c                   ób  ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Z	 dd„Z	 	 	 	 	 ddej                  dz  de	dz  dej                  dz  d	ej                  dz  d
edz  dedz  deej                  eej                     f   eej                  eej                     eej                  df   f   z  dz  fd„Zˆ xZS )ÚCodeGenAttentionNc                 ó.  •— t         ‰| �  «        |j                  | _        t	        j
                  |j                  «      | _        t	        j
                  |j                  «      | _	        || _
        |€-t        j                  d| j                  j                  › d�«       |j                  | _        |j"                  | _        | j                   | j"                  z  | _        | j$                  | j"                  z  | j                   k7  r&t'        d| j                   › d| j"                  › d�«      ‚t)        j*                  | j$                  «      | _        t	        j.                  | j                   | j                   dz  d¬«      | _        t	        j.                  | j                   | j                   d¬«      | _        |j4                  | _        | j4                  xs | j                   | _        | j9                  d	t;        | j                  | j6                  «      d¬
«       y )NzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.zEembed_dim must be divisible by num_attention_heads (got `embed_dim`: z and `num_attention_heads`: z).r   F)ÚbiasÚembed_positions)Ú
persistent)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_positionsr   ÚDropoutÚ
attn_pdropÚattn_dropoutÚresid_pdropÚresid_dropoutÚ	layer_idxÚloggerÚwarning_onceÚ	__class__Ú__name__Úhidden_sizeÚ	embed_dimÚnum_attention_headsÚhead_dimÚ
ValueErrorÚmathÚsqrtÚ
scale_attnÚLinearÚqkv_projÚout_projÚ
rotary_dimÚpos_embd_dimÚregister_bufferr%   )ÚselfÚconfigrC   rF   s      €r$   r;   zCodeGenAttention.__init__>   s©  ø€ Ü‰ÑÔà#×;Ñ;ˆÔÜŸJ™J v×'8Ñ'8Ó9ˆÔÜŸZ™Z¨×(:Ñ(:Ó;ˆÔØ"ˆŒØÐÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôð  ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ ØŸ™¨$×*BÑ*BÑBˆŒØ�=‰=˜4×3Ñ3Ñ3°t·~±~ÒEÜØWÐX\×XfÑXfÐWgð h+Ø+/×+CÑ+CÐ*DÀBðHóð ô Ÿ)™) D§M¡MÓ2ˆŒÜŸ	™	 $§.¡.°$·.±.À1Ñ2DÈ5ÔQˆŒäŸ	™	 $§.¡.°$·.±.ÀuÔMˆŒØ ×+Ñ+ˆŒØ ŸO™OÒ=¨t¯~©~ˆÔØ×ÑØÔ:¸4×;MÑ;MÈt×O`ÑO`ÓaÐnsð 	õ 	
r&   c                 ó´   — |j                  |j                  d d ||z  |fz   «      }|j                  |j                  d d dz   |j                  dd  z   «      }|S )Nr)   r*   )r)   )ÚreshapeÚshape)rV   r'   Ún_headÚdim_headÚmp_numÚreshapeds         r$   Ú_split_headszCodeGenAttention._split_heads^   s]   € Ø—9‘9˜QŸW™W S b˜\¨V°vÑ-=¸xÐ,HÑHÓIˆØ×#Ñ# A§G¡G¨C¨R L°5Ñ$8¸8¿>¹>È"È#Ð;NÑ$NÓOˆØˆr&   c                 óˆ  — t        |j                  «      dk(  r$|j                  ddddd«      j                  «       }n\t        |j                  «      dk(  r#|j                  dddd«      j                  «       }n!t	        dt        |j                  «      › �«      ‚|j                  «       dd	 ||z  fz   }|j                  |«      S )
zM
        Merges attn_head_size dim and num_attn_heads dim into n_ctx
        é   r   r   r   r   é   z3Input tensor rank should be one of [4, 5], but is: Nr*   )ÚlenrZ   ÚpermuteÚ
contiguousrL   ÚsizeÚview)rV   r0   rJ   Úattn_head_sizeÚ	new_shapes        r$   Ú_merge_headszCodeGenAttention._merge_headsc   s²   € ô ˆv�|‰|Ó Ò!Ø—^‘^ A q¨!¨Q°Ó2×=Ñ=Ó?‰FÜ�—‘Ó !Ò#Ø—^‘^ A q¨!¨QÓ/×:Ñ:Ó<‰FäÐRÔSVÐW]×WcÑWcÓSdÐReÐfÓgÐgØ—K‘K“M # 2Ð&Ð*=ÀÑ*NÐ)PÑPˆ	Ø�{‰{˜9Ó%Ð%r&   c                 óº  — |j                  t        j                  «      }|j                  t        j                  «      }t        j                  ||j	                  dd«      «      }|�||z   }|| j
                  z  } t        j                  d¬«      |«      }|j                  |j                  «      }| j                  |«      }t        j                  ||«      }||fS )Nr)   r*   r   )
Útor   Úfloat32ÚmatmulÚ	transposerO   r   ÚSoftmaxr   r@   )rV   ÚqueryÚkeyÚvalueÚattention_maskÚattn_weightsÚattn_outputs          r$   Ú_attnzCodeGenAttention._attnp   s³   € ð —‘œŸ™Ó'ˆØ�f‰f”U—]‘]Ó#ˆä—|‘| E¨3¯=©=¸¸RÓ+@ÓAˆàÐ%Ø'¨.Ñ8ˆLà# d§o¡oÑ5ˆØ)”r—z‘z bÔ)¨,Ó7ˆØ#—‘ u§{¡{Ó3ˆØ×(Ñ(¨Ó6ˆä—l‘l <°Ó7ˆà˜LÐ(Ð(r&   Úhidden_statesÚ
layer_pastrt   Úposition_idsÚ	use_cacheÚoutput_attentionsr   .c                 ó6  — | j                  |«      }d}|j                  |j                  d d |dfz   «      }	| j                  | j                  z  |z  }
t        j                  |	|
d¬«      \  }}}| j                  || j                  | j                  |¬«      }| j                  || j                  | j                  |¬«      }| j                  || j                  | j                  |¬«      }|j                  dddd«      }| j                  }|j                  |j                  k7  r"|j                  |j                  «      }|| _	        ||   }t        j                  ||j                  d   dz  d¬«      \  }}| j                  �¹|d d …d d …d d …d | j                  …f   }|d d …d d …d d …| j                  d …f   }|d d …d d …d d …d | j                  …f   }|d d …d d …d d …| j                  d …f   }t        |||«      }t        |||«      }t        j                  ||gd¬«      }t        j                  ||gd¬«      }nt        |||«      }t        |||«      }|j                  dddd«      }|j                  dddd«      }|�9|j                  |j                  |j                   «      || j"                  «      \  }}| j%                  ||||«      \  }}| j'                  || j                  | j                  «      }| j)                  |«      }| j+                  |«      }||fS )	Nrb   r)   r   )r]   r   r   r   r   )rQ   rY   rZ   rK   rJ   r   Úsplitr_   rd   r8   Údevicerl   rS   r3   r   Úupdater   rC   rw   rj   rR   rB   )rV   rx   ry   rt   rz   r{   r|   Úqkvr]   Ú	qkv_splitÚ	local_dimrq   rs   rr   r8   Úsincosr    r!   Úk_rotÚk_passÚq_rotÚq_passrv   ru   s                           r$   ÚforwardzCodeGenAttention.forward‰   sê  € ð �m‰m˜MÓ*ˆàˆØ—K‘K §	¡	¨#¨2 °&¸"°Ñ =Ó>ˆ	à—M‘M D×$<Ñ$<Ñ<ÀÑFˆ	Ü!ŸK™K¨	°9À"ÔEÑˆˆu�cØ×!Ñ! %¨×)AÑ)AÀ4Ç=Á=ÐY_Ð!Ó`ˆØ×Ñ  T×%=Ñ%=¸t¿}¹}ÐU[ÐÓ\ˆà×!Ñ! %¨×)AÑ)AÀ4Ç=Á=ÐY_Ð!Ó`ˆØ—‘˜a  A qÓ)ˆà×.Ñ.ˆØ×!Ñ! \×%8Ñ%8Ò8Ø-×0Ñ0°×1DÑ1DÓEˆOØ#2ˆDÔ à  Ñ.ˆÜ—;‘;˜v v§|¡|°BÑ'7¸1Ñ'<À"ÔE‰ˆˆSà�?‰?Ð&Øšš1šaÐ!2 4§?¡?Ð!2Ð2Ñ3ˆEØššAšq $§/¡/Ñ"3Ð3Ñ4ˆFàš!šQ¢Ð#4 T§_¡_Ð#4Ð4Ñ5ˆEØš1ša¢ D§O¡OÑ$5Ð5Ñ6ˆFä(¨°°SÓ9ˆEÜ(¨°°SÓ9ˆEä—)‘)˜U F˜O°Ô4ˆCÜ—I‘I˜u f˜o°2Ô6‰Eä& s¨C°Ó5ˆCÜ(¨°°SÓ9ˆEà�k‰k˜!˜Q  1Ó%ˆØ—‘˜a  A qÓ)ˆð Ð!Ø#×*Ñ*¨3¯6©6°-×2EÑ2EÓ+FÈÈtÏ~É~Ó^‰JˆC�ð %)§J¡J¨u°c¸5À.Ó$QÑ!ˆ�\à×'Ñ'¨°T×5MÑ5MÈtÏ}É}Ó]ˆØ—m‘m KÓ0ˆØ×(Ñ(¨Ó5ˆØ˜LÐ(Ð(r&   ©N©NNNFF)rG   Ú
__module__Ú__qualname__r;   r_   rj   rw   r   ÚFloatTensorr   Ú
LongTensorÚboolÚtupleÚTensorr‰   Ú__classcell__©rF   s   @r$   r5   r5   =   sû   ø„ õ
ò@ò
&ð$ ó)ð8 $(Ø37Ø04Ø!&Ø).ñ@)à×(Ñ(¨4Ñ/ð@)ð ˜D‘Lð@)ð ×)Ñ)¨DÑ0ð	@)ð
 ×&Ñ&¨Ñ-ð@)ð ˜$‘;ð@)ð   $™;ð@)ð 	ˆe�l‰l˜E %§,¡,Ñ/Ð/Ñ0Ø
�—‘˜e E§L¡LÑ1°5¸¿¹ÀsÐ9JÑ3KÐKÑ
Lñ	Mà
ñ	÷@)r&   r5   c                   ó\   ‡ — e Zd Zˆ fd„Zdej
                  dz  dej
                  fd„Zˆ xZS )Ú
CodeGenMLPc                 ó  •— t         ‰| �  «        |j                  }t        j                  ||«      | _        t        j                  ||«      | _        t        |j                     | _	        t        j                  |j                  «      | _        y rŠ   )r:   r;   Ún_embdr   rP   Úfc_inÚfc_outr   Úactivation_functionÚactr>   rA   Údropout)rV   Úintermediate_sizerW   rI   rF   s       €r$   r;   zCodeGenMLP.__init__Î   se   ø€ Ü‰ÑÔØ—M‘Mˆ	ä—Y‘Y˜yÐ*;Ó<ˆŒ
Ü—i‘iÐ 1°9Ó=ˆŒä˜&×4Ñ4Ñ5ˆŒÜ—z‘z &×"4Ñ"4Ó5ˆ�r&   rx   Nr   c                 óŽ   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S rŠ   )r™   rœ   rš   r�   )rV   rx   s     r$   r‰   zCodeGenMLP.forwardØ   s@   € ØŸ
™
 =Ó1ˆØŸ™ Ó/ˆØŸ™ MÓ2ˆØŸ™ ]Ó3ˆØÐr&   )rG   rŒ   r�   r;   r   rŽ   r‰   r“   r”   s   @r$   r–   r–   Í   s,   ø„ ô6ð U×%6Ñ%6¸Ñ%=ð À%×BSÑBS÷ r&   r–   c                   ó  ‡ — e Zd Zdˆ fd„	Z	 	 	 	 	 ddej
                  dz  dedz  dej
                  dz  dej                  dz  dedz  dedz  d	e	ej                     e	ej                  e	ej
                  d
f   f   z  dz  fd„Zˆ xZS )ÚCodeGenBlockNc                 ó  •— t         ‰| �  «        |j                  �|j                  nd|j                  z  }t	        j
                  |j                  |j                  ¬«      | _        t        ||«      | _	        t        ||«      | _        y )Nrb   ©Úeps)r:   r;   Ún_innerr˜   r   Ú	LayerNormÚlayer_norm_epsilonÚln_1r5   Úattnr–   Úmlp)rV   rW   rC   Ú	inner_dimrF   s       €r$   r;   zCodeGenBlock.__init__ã   sc   ø€ Ü‰ÑÔØ&,§n¡nÐ&@�F—N’NÀaÈ&Ï-É-ÑFWˆ	Ü—L‘L §¡°F×4MÑ4MÔNˆŒ	Ü$ V¨YÓ7ˆŒ	Ü˜i¨Ó0ˆ�r&   rx   ry   rt   rz   r{   r|   r   .c                 ó–   — |}| j                  |«      }| j                  ||||||¬«      \  }	}
| j                  |«      }|	|z   |z   }||
fS )N)rx   ry   rt   rz   r{   r|   )r¨   r©   rª   )rV   rx   ry   rt   rz   r{   r|   ÚkwargsÚresidualÚattn_outputsru   Úfeed_forward_hidden_statess               r$   r‰   zCodeGenBlock.forwardê   sm   € ð !ˆØŸ	™	 -Ó0ˆØ%)§Y¡YØ'Ø!Ø)Ø%ØØ/ð &/ó &
Ñ"ˆ�lð &*§X¡X¨mÓ%<Ð"Ø$Ð'AÑAÀHÑLˆà˜lÐ*Ð*r&   rŠ   r‹   )rG   rŒ   r�   r;   r   rŽ   r   r�   r�   r‘   r’   r‰   r“   r”   s   @r$   r¡   r¡   á   sÁ   ø„ õ1ð $(Ø37Ø04Ø!&Ø).ñ+à×(Ñ(¨4Ñ/ð+ð ˜D‘Lð+ð ×)Ñ)¨DÑ0ð	+ð
 ×&Ñ&¨Ñ-ð+ð ˜$‘;ð+ð   $™;ð+ð 
ˆu�|‰|Ñ	˜u U§\¡\°5¸×9JÑ9JÈCÐ9OÑ3PÐ%PÑQÑ	QÐTXÑ	X÷+r&   r¡   c                   óB   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	ˆ fd„Z
ˆ xZS )ÚCodeGenPreTrainedModelrW   ÚtransformerTr¡   Úpast_key_valuesc                 óÂ   •— t         ‰| �  |«       t        |t        «      r?t	        j
                  |j                  t        |j                  |j                  «      «       y y rŠ   )
r:   Ú_init_weightsÚ
isinstancer5   ÚinitÚcopy_r8   r%   r=   rT   )rV   ÚmodulerF   s     €r$   r¶   z$CodeGenPreTrainedModel._init_weights  sI   ø€ Ü‰Ñ˜fÔ%Ü�fÔ.Ô/Ü�J‰J�v×-Ñ-Ô/JÈ6×K_ÑK_Ðag×atÑatÓ/uÕvð 0r&   )rG   rŒ   r�   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_can_compile_fullgraphr¶   r“   r”   s   @r$   r²   r²     s9   ø… àÓØ%ÐØ&*Ð#Ø'Ð(ÐØ#4Ð"5ÐØ!Ð÷wð wr&   r²   c                   ó$  ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 ddej                  dz  de	dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dedz  dedz  dedz  dedz  deez  fd„«       Zˆ xZS )ÚCodeGenModelc           	      ó–  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  «      | _
        t        j                  t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}«      | _        t        j                   | j                  |j"                  ¬«      | _        t'        |j(                  |j*                  |j,                  z  «      | _        d| _        | j1                  «        y c c}w )N)rC   r£   F)r:   r;   r˜   rI   Ú
vocab_sizer   Ú	EmbeddingÚwter>   Ú
embd_pdropÚdropÚ
ModuleListÚrangeÚn_layerr¡   Úhr¦   r§   Úln_fÚminrS   Ún_ctxrJ   Úgradient_checkpointingÚ	post_init)rV   rW   ÚirF   s      €r$   r;   zCodeGenModel.__init__  sá   ø€ Ü‰Ñ˜Ô àŸ™ˆŒØ ×+Ñ+ˆŒÜ—<‘< × 1Ñ 1°4·>±>ÓBˆŒÜ—J‘J˜v×0Ñ0Ó1ˆŒ	Ü—‘Ì5ÐQW×Q_ÑQ_ÓK`ÖaÀa¤¨V¸qÖ AÒaÓbˆŒÜ—L‘L §¡°V×5NÑ5NÔOˆŒ	Ü˜f×/Ñ/°·±À×A[ÑA[Ñ1[Ó\ˆŒà&+ˆÔ#ð 	�‰Õùò  bs   Â,Ec                 ó   — | j                   S rŠ   ©rÆ   )rV   s    r$   Úget_input_embeddingsz!CodeGenModel.get_input_embeddings%  s   € Ø�x‰xˆr&   c                 ó   — || _         y rŠ   rÔ   )rV   Únew_embeddingss     r$   Úset_input_embeddingsz!CodeGenModel.set_input_embeddings(  s	   € Ø!ˆ�r&   NÚ	input_idsr´   rt   Útoken_type_idsrz   Úinputs_embedsr{   r|   Úoutput_hidden_statesÚreturn_dictr   c           	      óœ  — |�|n| j                   j                  }|	�|	n| j                   j                  }	|�|n| j                   j                  }|
�|
n| j                   j                  }
|du |duz  rt        d«      ‚| j                  r%| j                  r|rt        j                  d«       d}|€| j                  |«      }|r|€t        | j                   ¬«      }|j                  d   }|€V|�|j                  «       nd}t        j                  |j                  d   |j                   ¬«      |z   }|j#                  d«      }t%        | j                   ||||¬	«      }|}|�(|j'                  d
|«      }| j                  |«      }||z   }| j)                  |«      }d
||j+                  d
«      f}|rdnd}|	rdnd}t-        | j.                  «      D ],  \  }}|	r||fz   } |||||||¬«      }|d   }|sŒ$||d   fz   }Œ. | j1                  |«      }|j'                  |«      }|	r||fz   }|
st3        d„ ||||fD «       «      S t5        ||||¬«      S )a�  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)rW   r   r   )r   )rW   rÛ   rt   r´   rz   r)   © )ry   rt   rz   r{   r|   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrŠ   rß   )Ú.0Úvs     r$   ú	<genexpr>z'CodeGenModel.forward.<locals>.<genexpr>Š  s   è ø€ ò ØÐghÑgt”ñùs   ‚Š)Úlast_hidden_stater´   rx   Ú
attentions)rW   r|   rÜ   r{   rÝ   rL   rÐ   ÚtrainingrD   rE   rÆ   r   rZ   Úget_seq_lengthr   r   r   Ú	unsqueezer
   rg   rÈ   rf   Ú	enumeraterÌ   rÍ   r‘   r   )rV   rÙ   r´   rt   rÚ   rz   rÛ   r{   r|   rÜ   rÝ   r­   Ú
seq_lengthÚpast_seen_tokensÚcausal_maskrx   Útoken_type_embedsÚoutput_shapeÚall_self_attentionsÚall_hidden_statesrÒ   ÚblockÚoutputss                          r$   r‰   zCodeGenModel.forward+  s«  € ð* 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×BYÑBYˆà˜Ð -°tÐ";Ò<ÜÐYÓZÐZà×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øpôð "�	àÐ Ø ŸH™H YÓ/ˆMá˜Ð0Ü*°$·+±+Ô>ˆOà"×(Ñ(¨Ñ+ˆ
ØÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆàÐ%Ø+×0Ñ0°°ZÓ@ˆNØ $§¡¨Ó 8ÐØ)Ð,=Ñ=ˆMàŸ	™	 -Ó0ˆØ˜J¨×(:Ñ(:¸2Ó(>Ð?ˆá$5™b¸4ÐÙ"6™B¸DÐÜ! $§&¡&Ó)ò 	J‰HˆAˆuÙ#Ø$5¸Ð8HÑ$HÐ!áØØ*Ø*Ø)Ø#Ø"3ôˆGð $ A™JˆMÚ Ø&9¸WÀQ¹Z¸MÑ&IÑ#ð	Jð" Ÿ	™	 -Ó0ˆà%×*Ñ*¨<Ó8ˆáØ 1°]Ð4DÑ DÐáÜñ Ø)¨?Ð<MÐObÐcôó ð ô 'Ø+Ø+Ø+Ø*ô	
ð 	
r&   )
NNNNNNNNNN)rG   rŒ   r�   r;   rÕ   rØ   r   r   r�   r   rŽ   r�   r‘   r   r‰   r“   r”   s   @r$   rÂ   rÂ     s  ø„ ôò ò"ð ð .2Ø(,Ø37Ø26Ø04Ø26Ø!%Ø)-Ø,0Ø#'ñg
à×#Ñ# dÑ*ðg
ð  ™ðg
ð ×)Ñ)¨DÑ0ð	g
ð
 ×(Ñ(¨4Ñ/ðg
ð ×&Ñ&¨Ñ-ðg
ð ×(Ñ(¨4Ñ/ðg
ð ˜$‘;ðg
ð   $™;ðg
ð # T™kðg
ð ˜D‘[ðg
ð 
Ð(Ñ	(òg
ó ôg
r&   rÂ   zM
    The CodeGen Model transformer with a language modeling head on top.
    )Úcustom_introc                   ó`  ‡ — e Zd ZddiZˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dedz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  de
dz  de
dz  de
dz  de
dz  deej                  z  deez  fd„«       Zˆ xZS )ÚCodeGenForCausalLMzlm_head.weightztransformer.wte.weightc                 óÂ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  «      | _        | j                  «        y rŠ   )
r:   r;   rÂ   r³   r   rP   r˜   rÄ   Úlm_headrÑ   )rV   rW   rF   s     €r$   r;   zCodeGenForCausalLM.__init__ž  sE   ø€ Ü‰Ñ˜Ô Ü'¨Ó/ˆÔÜ—y‘y §¡°×0AÑ0AÓBˆŒð 	�‰Õr&   NrÙ   r´   rt   rÚ   rz   rÛ   Úlabelsr{   r|   rÜ   rÝ   Úlogits_to_keepr   c                 óÌ  — |�|n| j                   j                  }| j                  ||||||||	|
|¬«
      }|d   }t        |t        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }d}|�* | j                  d||| j                   j                  dœ|¤Ž}|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                  ¬«      S )aG  
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N)	r´   rt   rÚ   rz   rÛ   r{   r|   rÜ   rÝ   r   )Úlogitsrø   rÄ   r   )Úlossrû   r´   rx   rå   rß   )rW   rÝ   r³   r·   ÚintÚslicer÷   Úloss_functionrÄ   r   r´   rx   rå   )rV   rÙ   r´   rt   rÚ   rz   rÛ   rø   r{   r|   rÜ   rÝ   rù   r­   Útransformer_outputsrx   Úslice_indicesrû   rü   Úoutputs                       r$   r‰   zCodeGenForCausalLM.forward¦  s  € ð6 &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø)Ø%Ø'ØØ/Ø!5Ø#ð /ó 
Ðð ,¨AÑ.ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDáØ�YÐ!4°Q°RÐ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä%ØØØ/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r&   )NNNNNNNNNNNr   )rG   rŒ   r�   Ú_tied_weights_keysr;   r   r   r�   r   rŽ   r�   rý   r’   r‘   r   r‰   r“   r”   s   @r$   rõ   rõ   –  s5  ø„ ð +Ð,DÐEÐôð ð .2Ø(,Ø37Ø26Ø04Ø26Ø*.Ø!%Ø)-Ø,0Ø#'Ø-.ñ<
à×#Ñ# dÑ*ð<
ð  ™ð<
ð ×)Ñ)¨DÑ0ð	<
ð
 ×(Ñ(¨4Ñ/ð<
ð ×&Ñ&¨Ñ-ð<
ð ×(Ñ(¨4Ñ/ð<
ð × Ñ  4Ñ'ð<
ð ˜$‘;ð<
ð   $™;ð<
ð # T™kð<
ð ˜D‘[ð<
ð ˜eŸl™lÑ*ð<
ð 
Ð'Ñ	'ò<
ó ô<
r&   rõ   )rõ   rÂ   r²   ),Ú__doc__rM   r   r   Ú r   r¸   Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úmasking_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_codegenr   Ú
get_loggerrG   rD   rý   r’   r%   r/   r3   ÚModuler5   r–   r¡   r²   rÂ   rõ   Ú__all__rß   r&   r$   ú<module>r     s]  ðñ ã ã Ý å &Ý !ß .Ý )Ý /Ý 9ß OÝ -÷õ 1ð 
ˆ×	Ñ	˜HÓ	%€ðP¨ð P°3ð P¸5¿<¹<ó Pð˜Ÿ™ð ¨¯©ó ð= §¡ð =°E·L±Lð =ÀuÇ|Á|ð =ÐX]×XdÑXdó =ôL)�r—y‘yô L)ô`�—‘ô ô( +Ð-ô  +ðF ôw˜_ó wó ðwð ô
Ð)ó 
ó ð
ñD ðôô
H
Ð/°ó H
óð
H
òV K�r&   