Ë
    ÛÓ^jüT  ã                   ó‚  — d Z ddlmZ ddlmZ ddlZddlmZ ddlmZ	 ddl
mZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z,  G d„ de&«      Z- G d„ dej\                  «      Z/ G d„ dej\                  «      Z0 G d„ dej\                  «      Z1 G d „ d!ej\                  «      Z2 G d"„ d#ej\                  «      Z3 G d$„ d%ej\                  «      Z4 G d&„ d'e«      Z5 G d(„ d)e«      Z6e G d*„ d+e6«      «       Z7 G d,„ d-e6e«      Z8g d.¢Z9y)/z"Modular components for DBRX model.é    )ÚCallable)ÚAnyN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)Úload_balancing_loss_funcé   )Ú
DbrxConfigc                   ó   — e Zd Zy)ÚDbrxRotaryEmbeddingN)Ú__name__Ú
__module__Ú__qualname__© ó    úp/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/dbrx/modular_dbrx.pyr    r    -   s   „ Ør%   r    c                   óÖ   ‡ — e Zd ZdZ	 ddedz  fˆ fd„Z	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
ej                  ej                  f   f
d
„Zˆ xZS )ÚDbrxAttentionzYModular DBRX attention component that can be reused across different model architectures.NÚ	layer_idxc                 óð  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        |j                  | _	        || _
        |j                  }|j                  | _        |j                  | _        |j                  | _        | j                  | j                   z  | _        | j                  dz  | _        |j&                  | _        d| _        t+        j,                  | j                  | j                  d| j                   z  | j                  z  z   d¬«      | _        t+        j,                  | j                  | j                  d¬«      | _        y )Ng      à¿Tr   F©Úbias)ÚsuperÚ__init__ÚconfigÚd_modelÚhidden_sizeÚn_headsÚ	num_headsÚhead_dimÚmax_seq_lenÚmax_position_embeddingsr)   Úattn_configÚ
attn_pdropÚattention_dropoutÚclip_qkvÚ
kv_n_headsÚnum_key_value_headsÚnum_key_value_groupsÚscalingÚ
rope_thetaÚ	is_causalr   ÚLinearÚWqkvÚout_proj)Úselfr/   r)   Úkwargsr7   Ú	__class__s        €r&   r.   zDbrxAttention.__init__4   s*  ø€ ô 	‰ÑÔØˆŒØ!Ÿ>™>ˆÔØŸ™ˆŒØ×(Ñ(¨D¯N©NÑ:ˆŒØ'-×'9Ñ'9ˆÔ$Ø"ˆŒà×(Ñ(ˆØ!,×!7Ñ!7ˆÔØ#×,Ñ,ˆŒØ#.×#9Ñ#9ˆÔ Ø$(§N¡N°d×6NÑ6NÑ$NˆÔ!Ø—}‘} dÑ*ˆŒØ%×0Ñ0ˆŒØˆŒä—I‘IØ×Ñ˜d×.Ñ.°°T×5MÑ5MÑ1MÐPT×P]ÑP]Ñ1]Ñ]Ðdiô
ˆŒ	ô Ÿ	™	 $×"2Ñ"2°D×4DÑ4DÈ5ÔQˆ�r%   Úhidden_statesÚattention_maskÚposition_embeddingsÚpast_key_valuesÚreturnc                 óæ  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      }| j                  �| j                   nd }	|j	                  |	| j                  ¬«      }|j                  | j                  | j                  | j                  z  | j                  | j                  z  gd¬«      \  }
}}|
j                  |«      j                  dd«      }
|j                  |«      j                  dd«      }|j                  |«      j                  dd«      }|\  }}t        |
|||«      \  }
}|� |j                  ||| j                  «      \  }}t        j                  | j                  j                   t"        «      } || |
|||f| j$                  sdn| j&                  | j(                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }| j/                  |«      }||fS )Néÿÿÿÿ)ÚminÚmaxr   ©Údimr   ç        )Údropoutr>   )Úshaper4   rB   r:   ÚclampÚsplitr1   r<   ÚviewÚ	transposer   Úupdater)   r   Úget_interfacer/   Ú_attn_implementationr   Útrainingr9   r>   ÚreshapeÚ
contiguousrC   )rD   rG   rH   rI   rJ   rE   Úinput_shapeÚhidden_shapeÚ
qkv_statesÚmin_valÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                     r&   ÚforwardzDbrxAttention.forwardP   s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—Y‘Y˜}Ó-ˆ
Ø$(§M¡MÐ$=�4—=‘=‘.À4ˆØ×%Ñ%¨'°t·}±}Ð%ÓEˆ
à1;×1AÑ1Aà× Ñ Ø×(Ñ(¨4¯=©=Ñ8Ø×(Ñ(¨4¯=©=Ñ8ðð
 ð 2Bó 2
Ñ.ˆ�j ,ð $×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m KÓ0ˆØ˜LÐ(Ð(r%   ©N©NNN)r!   r"   r#   Ú__doc__Úintr.   ÚtorchÚTensorÚ
LongTensorr	   Útuplerk   Ú__classcell__©rF   s   @r&   r(   r(   1   sŽ   ø„ Ùcð
 !%ñRð ˜‘:õRð> /3Ø7;Ø(,ñ3)à—|‘|ð3)ð Ÿ™ tÑ+ð3)ð #×-Ñ-°Ñ4ð	3)ð
  ™ð3)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷3)r%   r(   c            
       óž   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  dej
                  dej
                  f
d„Zˆ xZS )ÚDbrxExpertGLUc                 ó¶  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        t        j                  t        j                  | j                  | j                  z  | j                  «      «      | _	        t        j                  t        j                  | j                  | j                  z  | j                  «      «      | _
        t        j                  t        j                  | j                  | j                  z  | j                  «      «      | _        |j                  j                  dd«      }t        |   | _        y )NÚnameÚsilu)r-   r.   r1   Úffn_hidden_sizeÚmoe_num_expertsr   Ú	Parameterrp   ÚemptyÚw1Úv1Úw2Ú
ffn_act_fnÚgetr   Úactivation_fn)rD   r/   Úact_fn_namerF   s      €r&   r.   zDbrxExpertGLU.__init__‡   sö   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ%×5Ñ5ˆÔØ%×5Ñ5ˆÔä—,‘,œuŸ{™{¨4×+?Ñ+?À$×BVÑBVÑ+VÐX\×XhÑXhÓiÓjˆŒÜ—,‘,œuŸ{™{¨4×+?Ñ+?À$×BVÑBVÑ+VÐX\×XhÑXhÓiÓjˆŒÜ—,‘,œuŸ{™{¨4×+?Ñ+?À$×BVÑBVÑ+VÐX\×XhÑXhÓiÓjˆŒà×'Ñ'×+Ñ+¨F°FÓ;ˆÜ# KÑ0ˆÕr%   ÚxÚ	expert_w1Ú	expert_v1Ú	expert_w2rK   c                 ó´   — |j                  |«      }|j                  |«      }| j                  |«      }||z  }|j                  |j                  «       «      }|S rl   )Úmatmulr„   Út)	rD   r†   r‡   rˆ   r‰   Ú	gate_projÚup_projÚintermediate_statesÚ	down_projs	            r&   rk   zDbrxExpertGLU.forward”   sW   € ð —H‘H˜YÓ'ˆ	Ø—(‘(˜9Ó%ˆØ×&Ñ& yÓ1ˆ	Ø'¨'Ñ1ÐØ'×.Ñ.¨y¯{©{«}Ó=ˆ	ØÐr%   ©r!   r"   r#   r.   rp   rq   rk   rt   ru   s   @r&   rw   rw   †   sK   ø„ ô1ðØ—‘ðØ*/¯,©,ðØCHÇ<Á<ðØ\a×\hÑ\hðà	�‰÷r%   rw   c                   ó†   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  dej
                  fd„Zˆ xZS )ÚDbrxExpertsc                 ó¨   •— t         ‰| �  «        t        |«      | _        |j                  | _        |j
                  | _        |j                  | _        y rl   )r-   r.   rw   Úmlpr1   r{   r|   Únum_experts©rD   r/   rF   s     €r&   r.   zDbrxExperts.__init__    sD   ø€ Ü‰ÑÔÜ  Ó(ˆŒØ!×-Ñ-ˆÔØ%×5Ñ5ˆÔØ!×1Ñ1ˆÕr%   rG   Útop_k_indexÚtop_k_weightsrK   c                 óž  — |j                   d   }|j                  d| j                  «      }t        j                  ||j
                  |j                  ¬«      }t        j                  «       5  t        j                  j                  j                  || j                  ¬«      }|j                  ddd«      }t        j                  |j                  d¬«      d«      j                  «       }d d d «       d| j                  | j                   f}D �]  }	|	d   }	t        j                  «       5  t        j"                  |	   «      \  }
}d d d «       | j$                  j&                  j)                  |«      |	   }| j$                  j*                  j)                  |«      |	   }| j$                  j,                  j)                  |«      |	   }| j%                  |   |||«      }|j)                  d| j                  «      ||
d f   z  }|j/                  d||«       �Œ
 |j)                  |d| j                  «      }|S # 1 sw Y   �ŒOxY w# 1 sw Y   ŒÿxY w)	Nr   rM   )ÚdtypeÚdevice)Únum_classesr   r   )rM   éþÿÿÿrP   )rT   r]   r{   rp   Ú
zeros_liker›   rœ   Úno_gradr   Ú
functionalÚone_hotr–   ÚpermuteÚgreaterÚsumÚnonzeror1   Úwherer•   r€   rW   r   r�   Ú
index_add_)rD   rG   r˜   r™   Ú
batch_sizeÚnext_statesÚexpert_maskÚ
expert_hitÚsplit_expert_shapeÚ
expert_idxÚidxÚ	token_idxr€   r   r�   Ústatess                   r&   rk   zDbrxExperts.forward§   s  € ð #×(Ñ(¨Ñ+ˆ
Ø%×-Ñ-¨b°$×2FÑ2FÓGˆä×&Ñ& }¸M×<OÑ<OÐXe×XlÑXlÔmˆÜ�]‰]‹_ñ 	SÜŸ(™(×-Ñ-×5Ñ5°kÈt×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸ™ {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷	Sð
 ! $×"6Ñ"6¸×8HÑ8HÐIÐØ$ó 		9ˆJØ# A™ˆJÜ—‘“ñ FÜ!&§¡¨[¸Ñ-DÓ!E‘��Y÷Fà—‘—‘×!Ñ!Ð"4Ó5°jÑAˆBØ—‘—‘×!Ñ!Ð"4Ó5°jÑAˆBØ—‘—‘×!Ñ!Ð"4Ó5°jÑAˆBØ—X‘X˜m¨IÑ6¸¸BÀÓCˆFØ—[‘[  T×%9Ñ%9Ó:¸]È9ÐVYÐ[_ÐK_Ñ=`Ñ`ˆFØ×"Ñ" 1 i°Ö8ð		9ð "×&Ñ& z°2°t×7KÑ7KÓLˆØÐ÷%	Sñ 	Sú÷Fð Fús   Á,A=H6Ä)IÈ6I ÉI	r‘   ru   s   @r&   r“   r“   Ÿ   sC   ø„ ô2ðà—|‘|ðð —\‘\ðð —|‘|ð	ð
 
�‰÷r%   r“   c                   óŠ   ‡ — e Zd Zˆ fd„Zdej
                  deej
                  ej
                  ej                  f   fd„Zˆ xZ	S )Ú
DbrxRouterc                 óÈ   •— t         ‰| �  «        |j                  | _        |j                  | _        t        j                  | j                  |j                  d¬«      | _        y ©NFr+   )	r-   r.   r{   r1   Úmoe_jitter_epsr   rA   r|   Úlayerr—   s     €r&   r.   zDbrxRouter.__init__Ç   sJ   ø€ Ü‰ÑÔØ!×1Ñ1ˆÔØ$×3Ñ3ˆÔÜ—Y‘Y˜t×/Ñ/°×1GÑ1GÈeÔTˆ�
r%   rG   rK   c                 ó  — | j                   rN| j                  �B|t        j                  |«      j	                  d| j                  z
  d| j                  z   «      z  }|j                  d|j                  d   «      }| j                  |«      }|S )Ng      ð?rM   )r\   r¶   rp   Ú
empty_likeÚuniform_rW   rT   r·   )rD   rG   Úrouter_logitss      r&   rk   zDbrxRouter.forwardÍ   s„   € Ø�=Š=˜T×0Ñ0Ð<ØœU×-Ñ-¨mÓ<×EÑEØ�d×)Ñ)Ñ)¨3°×1DÑ1DÑ+Dóñ ˆMð &×*Ñ*¨2¨}×/BÑ/BÀ2Ñ/FÓGˆØŸ
™
 =Ó1ˆØÐr%   )
r!   r"   r#   r.   rp   rq   rs   rr   rk   rt   ru   s   @r&   r³   r³   Æ   s;   ø„ ôUð U§\¡\ð °e¸E¿L¹LÈ%Ï,É,ÐX]×XhÑXhÐ<hÑ6i÷ r%   r³   c                   ó~   ‡ — e Zd ZdZˆ fd„Zd„ Zdej                  deej                  ej                  f   fd„Z	ˆ xZ
S )ÚDbrxFFNz0Modular DBRX MLP/FFN component with MoE support.c                 óö   •— t         ‰| �  «        t        |j                  «      | _        t        |j                  «      | _        |j                  j                  | _        |j                  j                  | _	        y rl   )
r-   r.   r³   Ú
ffn_configÚrouterr“   ÚexpertsÚmoe_normalize_expert_weightsÚ	moe_top_kÚtop_k)rD   r/   rE   rF   s      €r&   r.   zDbrxFFN.__init__Ú   sY   ø€ Ü‰ÑÔÜ  ×!2Ñ!2Ó3ˆŒÜ" 6×#4Ñ#4Ó5ˆŒà,2×,=Ñ,=×,ZÑ,ZˆÔ)Ø×&Ñ&×0Ñ0ˆ�
r%   c                 ó$  — t         j                  j                  j                  |d|j                  ¬«      }t        j
                  || j                  d¬«      \  }}| j                  �&|t        j                  || j                  dd¬«      z  }||fS )Nr   )rQ   r›   rM   rP   T)ÚprQ   Úkeepdim)	rp   r   r¡   Úsoftmaxr›   ÚtopkrÄ   rÂ   Únorm)rD   r»   Úrouter_top_valueÚrouter_indicess       r&   Úroute_tokens_to_expertszDbrxFFN.route_tokens_to_expertsâ   s†   € ÜŸ™×+Ñ+×3Ñ3°MÀqÐP]×PcÑPcÐ3ÓdˆÜ+0¯:©:°mÀTÇZÁZÐUWÔ+XÑ(Ð˜.Ø×,Ñ,Ð8Ø/´%·*±*Ø  D×$EÑ$EÈ2ÐW[ô3ñ  Ðð   Ð/Ð/r%   rG   rK   c                 óv   — | j                  |«      }| j                  |«      \  }}| j                  |||«      }|S rl   )rÀ   rÍ   rÁ   )rD   rG   r»   r™   r˜   Úoutputs         r&   rk   zDbrxFFN.forwardë   s<   € ØŸ™ MÓ2ˆØ%)×%AÑ%AÀ-Ó%PÑ"ˆ�{Ø—‘˜m¨[¸-ÓHˆØˆr%   )r!   r"   r#   rn   r.   rÍ   rp   rq   rs   rk   rt   ru   s   @r&   r½   r½   ×   s9   ø„ Ù:ô1ò0ð U§\¡\ð °e¸E¿L¹LÈ%Ï,É,Ð<VÑ6W÷ r%   r½   c                   óÐ   ‡ — e Zd Zddededz  fˆ fd„Z	 	 ddej                  dej                  dej                  dz  de	dz  d	e
d
eej                  ej                  f   fd„Zˆ xZS )ÚDbrxNormAttentionNormNr/   r)   c                 ó  •— t         ‰| �  «        || _        |j                  | _        t	        j
                  |j                  d¬«      | _        t        ||¬«      | _	        t	        j
                  |j                  d¬«      | _
        y )NFr+   ©r/   r)   )r-   r.   r)   Úresid_pdropr   Ú	LayerNormr0   Únorm_1r(   ÚattnÚnorm_2©rD   r/   r)   rF   s      €r&   r.   zDbrxNormAttentionNorm.__init__ó   sc   ø€ Ü‰ÑÔØ"ˆŒØ!×-Ñ-ˆÔÜ—l‘l 6§>¡>¸Ô>ˆŒÜ!ØØô
ˆŒ	ô —l‘l 6§>¡>¸Ô>ˆ�r%   rG   rI   rH   rJ   rE   rK   c                 ód  — |}| j                  |«      j                  |j                  «      } | j                  d||||dœ|¤Ž\  }}t        j
                  j                  || j                  | j                  ¬«      }||z   }|}| j                  |«      j                  |j                  «      }||fS ©N)rG   rH   rI   rJ   )rÆ   r\   r$   )
rÖ   Útor›   r×   r   r¡   rS   rÔ   r\   rØ   )rD   rG   rI   rH   rJ   rE   Úresidual_statesÚ_s           r&   rk   zDbrxNormAttentionNorm.forwardþ   s»   € ð (ˆØŸ™ MÓ2×5Ñ5°m×6IÑ6IÓJˆà$˜4Ÿ9™9ð 
Ø'Ø)Ø 3Ø+ñ	
ð
 ñ
Ñˆ�qô Ÿ™×-Ñ-¨m¸t×?OÑ?OÐZ^×ZgÑZgÐ-ÓhˆØ%¨Ñ7ˆà'ˆØŸ™ MÓ2×5Ñ5°m×6IÑ6IÓJˆà Ð-Ð-r%   rl   )NN)r!   r"   r#   r   ro   r.   rp   rq   rr   r	   r   rs   rk   rt   ru   s   @r&   rÑ   rÑ   ò   sŠ   ø„ ñ	?˜zð 	?°c¸D±jõ 	?ð /3Ø(,ñ.à—|‘|ð.ð #×-Ñ-ð.ð Ÿ™ tÑ+ð	.ð
  ™ð.ð ð.ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷.r%   rÑ   c                   óš   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
f
d
„Zˆ xZS )Ú	DbrxBlockr/   r)   c                 óº   •— t         ‰| �  «        |j                  | _        |j                  | _        || _        t        ||¬«      | _        t        |¬«      | _	        y )NrÓ   ©r/   )
r-   r.   r0   r1   rÔ   r)   rÑ   Únorm_attn_normr½   ÚffnrÙ   s      €r&   r.   zDbrxBlock.__init__  sP   ø€ Ü‰ÑÔØ!Ÿ>™>ˆÔØ!×-Ñ-ˆÔØ"ˆŒÜ3ØØô
ˆÔô  &Ô)ˆ�r%   NrG   rH   rI   rJ   rE   c                 óÒ   —  | j                   d||||dœ|¤Ž\  }}| j                  |«      }t        j                  j	                  || j
                  | j                  ¬«      }||z   }|S rÛ   )rã   rä   r   r¡   rS   rÔ   r\   )rD   rG   rH   rI   rJ   rE   Úresid_statess          r&   rk   zDbrxBlock.forward&  s~   € ð ': d×&9Ñ&9ð '
Ø'Ø)Ø 3Ø+ñ	'
ð
 ñ'
Ñ#ˆ�mð Ÿ™ Ó/ˆÜŸ™×-Ñ-¨m¸t×?OÑ?OÐZ^×ZgÑZgÐ-ÓhˆØ$ }Ñ4ˆØÐr%   rm   )r!   r"   r#   r   ro   r.   rp   rq   rr   r	   r   rk   rt   ru   s   @r&   rà   rà     sr   ø„ ð	*˜zð 	*°cõ 	*ð /3Ø7;Ø(,ñà—|‘|ðð Ÿ™ tÑ+ðð #×-Ñ-°Ñ4ð	ð
  ™ðð ÷r%   rà   c                   óž   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZ ej$                  «       dej(                  fˆ fd	„«       Zˆ xZS )
ÚDbrxPreTrainedModelr/   ÚtransformerTrà   rJ   F)rG   Ú
attentionsÚmodulec                 ó>  •— t         ‰| �  |«       | j                  j                  }t	        |t
        «      rgt        j                  |j                  d|¬«       t        j                  |j                  d|¬«       t        j                  |j                  d|¬«       y y )NrR   )ÚmeanÚstd)r-   Ú_init_weightsr/   Úinitializer_rangeÚ
isinstancerw   ÚinitÚnormal_r   r€   r�   )rD   rë   rî   rF   s      €r&   rï   z!DbrxPreTrainedModel._init_weightsL  sj   ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœmÔ,Ü�L‰L˜Ÿ™¨°#Õ6Ü�L‰L˜Ÿ™¨°#Õ6Ü�L‰L˜Ÿ™¨°#Ö6ð -r%   )r!   r"   r#   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flex_attnÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphrà   r(   Ú_can_record_outputsrp   r    r   ÚModulerï   rt   ru   s   @r&   rè   rè   <  sx   ø… ØÓØ%ÐØ&*Ð#Ø$˜ÐØ#4Ð"5ÐØÐØ"&ÐØÐØ€NØ"Ðà"Ø#ñÐð
 €U‡]�]ƒ_ð7 B§I¡Iô 7ó ô7r%   rè   c                   ó6  ‡ — e Zd ZdZdefˆ fd„Zdej                  fd„Zdej                  fd„Z	e
ee	 	 	 	 	 	 dd	ej                  dz  d
ej                  dz  dej                  dz  dedz  dej"                  dz  dedz  dee   defd„«       «       «       Zˆ xZS )Ú	DbrxModela©  Transformer decoder consisting of *config.num_hidden_layers*. Each layer is a [`DbrxBlock`] layer.

    Args:
        config ([`DbrxConfig`]): Model configuration class with all parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
    r/   c           	      ó,  •— t         ‰| �  |«       |j                  | _        |j                  | _        |j
                  | _        t        |«      | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        j"                  |j                  d¬«      | _        d| _        | j)                  «        y c c}w rµ   )r-   r.   Úpad_token_idÚpadding_idxÚ
vocab_sizeÚ	emb_pdropr    Ú
rotary_embr   Ú	Embeddingr0   ÚwteÚ
ModuleListÚrangeÚn_layersrà   ÚblocksrÕ   Únorm_fÚgradient_checkpointingÚ	post_initrÙ   s      €r&   r.   zDbrxModel.__init__`  sÇ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒØ×)Ñ)ˆŒÜ-¨fÓ5ˆŒÜ—<‘< × 1Ñ 1°6·>±>À4×CSÑCSÓTˆŒÜ—m‘mÔSXÐY_×YhÑYhÓSiÖ$jÀi¤Y¨v°yÕ%AÒ$jÓkˆŒÜ—l‘l 6§>¡>¸Ô>ˆŒØ&+ˆÔ#ð 	�‰Õùò %ks   Â4DrK   c                 ó   — | j                   S rl   ©r	  ©rD   s    r&   Úget_input_embeddingszDbrxModel.get_input_embeddingsn  s   € Ø�x‰xˆr%   Úvaluec                 ó   — || _         y rl   r  ©rD   r  s     r&   Úset_input_embeddingszDbrxModel.set_input_embeddingsq  s	   € Øˆ�r%   NÚ	input_idsrH   Úposition_idsrJ   Úinputs_embedsÚ	use_cacherE   c           
      ó:  — |d u |d uz  rt        d«      ‚|r|€t        | j                  ¬«      }|€| j                  |«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        | j                  ||||¬«      }	|}
| j                  |
|«      }| j                  d | j                  j                   D ]  } ||
f||	|||dœ|¤Ž}
Œ | j                  |
«      }
t        |
|¬«      S )	Nz:You must specify exactly one of input_ids or inputs_embedsrâ   r   r   )rœ   )r/   r  rH   rJ   r  )rI   rH   r  rJ   r  )Úlast_hidden_staterJ   )Ú
ValueErrorr
   r/   r	  Úget_seq_lengthrp   ÚarangerT   rœ   Ú	unsqueezer   r  r  Únum_hidden_layersr  r   )rD   r  rH   r  rJ   r  r  rE   Úpast_seen_tokensÚcausal_maskrG   rI   Údecoder_layers                r&   rk   zDbrxModel.forwardt  sJ  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZá˜Ð0Ü*°$·+±+Ô>ˆOàÐ Ø ŸH™H YÓ/ˆMàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆð #Ÿo™o¨m¸\ÓJÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 		ˆMÙ)Øðà$7Ø*Ø)Ø /Ø#ñð ñ‰Mð		ð Ÿ™ MÓ2ˆä%Ø+Ø+ô
ð 	
r%   )NNNNNN)r!   r"   r#   rn   r   r.   r   r  r  r  r   r   r   rp   rr   rq   r	   ÚFloatTensorÚboolr   r   r   rk   rt   ru   s   @r&   r  r  V  sò   ø„ ñð˜zõ ð b§l¡ló ð¨"¯,©,ó ð  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ5
à×#Ñ# dÑ*ð5
ð Ÿ™ tÑ+ð5
ð ×&Ñ&¨Ñ-ð	5
ð
  ™ð5
ð ×(Ñ(¨4Ñ/ð5
ð ˜$‘;ð5
ð Ð+Ñ,ð5
ð 
 ò5
ó ó ó  ô5
r%   r  c                   óì  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zd	ej                  fd
„Z
dej                  fd„Zd	ej                  fd„Zdej                  fd„Zdefd„Zd	efd„Zee	 	 	 	 	 	 	 	 	 ddej*                  dz  dej,                  dz  dej*                  dz  dedz  dej0                  dz  dej*                  dz  dedz  dedz  deej,                  z  dee   d	efd„«       «       Zˆ xZS ) ÚDbrxForCausalLMzlm_head.weightztransformer.wte.weightÚlm_headÚcolwise_gather_outputrG   Úlogitsr/   c                 óŠ  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        |j                  j                  | _        |j                  j                  | _        |j                  j                  | _        | j!                  «        y rµ   )r-   r.   r  ré   r  r   rA   r1   r+  r¿   Úmoe_loss_weightÚrouter_aux_loss_coefr|   r–   rÃ   Únum_experts_per_tokr  r—   s     €r&   r.   zDbrxForCausalLM.__init__´  s�   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆÔØ ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$5Ñ$5×$EÑ$EˆÔ!Ø!×,Ñ,×<Ñ<ˆÔØ#)×#4Ñ#4×#>Ñ#>ˆÔ Ø�‰Õr%   rK   c                 ó6   — | j                   j                  «       S rl   )ré   r  r  s    r&   r  z$DbrxForCausalLM.get_input_embeddings¾  s   € Ø×Ñ×4Ñ4Ó6Ð6r%   r  c                 ó:   — | j                   j                  |«       y rl   )ré   r  r  s     r&   r  z$DbrxForCausalLM.set_input_embeddingsÁ  s   € Ø×Ñ×-Ñ-¨eÕ4r%   c                 ó   — | j                   S rl   ©r+  r  s    r&   Úget_output_embeddingsz%DbrxForCausalLM.get_output_embeddingsÄ  s   € Ø�|‰|Ðr%   Únew_embeddingsc                 ó   — || _         y rl   r5  )rD   r7  s     r&   Úset_output_embeddingsz%DbrxForCausalLM.set_output_embeddingsÇ  s	   € Ø%ˆ�r%   Údecoderc                 ó   — || _         y rl   ©ré   )rD   r:  s     r&   Úset_decoderzDbrxForCausalLM.set_decoderÊ  s
   € Ø"ˆÕr%   c                 ó   — | j                   S rl   r<  r  s    r&   Úget_decoderzDbrxForCausalLM.get_decoderÍ  s   € Ø×ÑÐr%   Nr  rH   r  rJ   r  Úlabelsr  Úoutput_router_logitsÚlogits_to_keeprE   c
                 ój  — |�|n| j                   j                  } | j                  d|||||||dœ|
¤Ž}|j                  }t	        |	t
        «      rt        |	 d«      n|	}| j                  |dd…|dd…f   «      }d}|� | j                  ||| j                  fi |
¤Ž}d}|rYt        |j                  | j                  | j                  |«      }|�+|| j                  |j                  |j                   «      z  z  }t#        ||||j$                  |j&                  |j(                  |j                  ¬«      S )a‹  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >> from transformers import AutoTokenizer, DbrxForCausalLM

        >> model = DbrxForCausalLM.from_pretrained("transformers-community/dbrx-instruct")
        >> tokenizer = AutoTokenizer.from_pretrained("transformers-community/dbrx-instruct")

        >> prompt = "Hey, are you conscious? Can you talk to me?"
        >> inputs = tokenizer(prompt, return_tensors="pt")

        >> # Generate
        >> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```
        N)r  rH   r  rJ   r  r  rA  )ÚlossÚaux_lossr-  rJ   rG   rê   r»   r$   )r/   rA  ré   r  rñ   ro   Úslicer+  Úloss_functionr  r   r»   r–   r1  r0  rÜ   rœ   r   rJ   rG   rê   )rD   r  rH   r  rJ   r  r@  r  rA  rB  rE   ÚoutputsrG   Úslice_indicesr-  rD  rE  s                    r&   rk   zDbrxForCausalLM.forwardÐ  sW  € ðN %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð
 +;¨$×*:Ñ*:ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ñ	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDàˆÙÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ð!Ø˜×1Ñ1°H·K±KÀÇÁÓ4LÑLÑL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r%   )	NNNNNNNNr   ) r!   r"   r#   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr   r.   r   r  r  r  rA   r6  r9  r  r=  r?  r   r   rp   rr   rq   r	   r'  r(  ro   r   r   r   rk   rt   ru   s   @r&   r*  r*  ¯  s”  ø„ Ø*Ð,DÐEÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hð˜zõ ð7 b§l¡ló 7ð5¨"¯,©,ó 5ð r§y¡yó ð&°B·I±Ió &ð# 9ó #ð ˜Yó  ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ñP
à×#Ñ# dÑ*ðP
ð Ÿ™ tÑ+ðP
ð ×&Ñ&¨Ñ-ð	P
ð
  ™ðP
ð ×(Ñ(¨4Ñ/ðP
ð × Ñ  4Ñ'ðP
ð ˜$‘;ðP
ð # T™kðP
ð ˜eŸl™lÑ*ðP
ð Ð+Ñ,ðP
ð 
#òP
ó ó ôP
r%   r*  )r*  r  rè   ):rn   Úcollections.abcr   Útypingr   rp   r   Ú r   rò   Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úllama.modeling_llamar   r   r   Úmixtral.modeling_mixtralr   Úconfiguration_dbrxr   r    rÿ   r(   rw   r“   r³   r½   rÑ   rà   rè   r  r*  Ú__all__r$   r%   r&   ú<module>r_     s  ðñ )å $Ý ã Ý å &Ý !ß .Ý )Ý /õ÷ Rß FÝ &ß IÑ IÝ 7Ý 5÷ñ õ
 @Ý *ô	Ð.ô 	ôR)�B—I‘Iô R)ôj�B—I‘Iô ô2$�"—)‘)ô $ôN�—‘ô ô"ˆb�i‰iô ô6%.˜BŸI™Iô %.ôPÐ*ô ôD7˜/ô 7ð4 ôU
Ð#ó U
ó ðU
ôps
Ð)¨?ô s
òl B�r%   