Ë
    ÛÓ^j†r  ã                   óâ  — d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZmZ ddlmZ ddlmZmZ ddlmZmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0  G d„ dejb                  «      Z2 ed«       G d„ dejb                  «      «       Z3 G d„ dejb                  «      Z4 G d„ dejb                  «      Z5e G d„ d ejb                  «      «       Z6 G d!„ d"ejb                  «      Z7d#„ Z8 ed$«      d>d%„«       Z9d&ejt                  d'e;d(ejt                  fd)„Z<	 d?d*ejb                  d+ejt                  d,ejt                  d-ejt                  d.ejt                  dz  d/e=d0e=d1e$e&   fd2„Z> ee9«       G d3„ d4ejb                  «      «       Z? G d5„ d6e«      Z@ G d7„ d8e"«      ZAe' G d9„ d:eA«      «       ZBe' G d;„ d<eAe«      «       ZCg d=¢ZDy)@é    )ÚCallable)ÚOptionalN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚAfmoeConfigc                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚAfmoeRotaryEmbeddingÚinv_freqNÚconfigc                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultr%   F)Ú
persistentÚoriginal_inv_freq)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr&   Úrope_parametersr(   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)Úselfr&   ÚdeviceÚrope_init_fnr%   Ú	__class__s        €ús/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/afmoe/modeling_afmoe.pyr-   zAfmoeRotaryEmbedding.__init__4   s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUó    r7   ztorch.deviceÚseq_lenÚreturnztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   é   ©Údtype)r7   rC   )	r1   ÚgetattrÚhidden_sizeÚnum_attention_headsÚtorchÚarangeÚint64ÚtoÚfloat)r&   r7   r<   ÚbaseÚdimÚattention_factorr%   s          r:   r2   z4AfmoeRotaryEmbedding.compute_default_rope_parametersD   s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r;   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   éÿÿÿÿr!   ÚmpsÚcpuF)Údevice_typeÚenabledrA   ©rM   rB   )r%   rK   ÚexpandÚshaperJ   r7   Ú
isinstanceÚtypeÚstrr   Ú	transposerG   ÚcatÚcosr3   ÚsinrC   )
r6   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrS   ÚfreqsÚembr]   r^   s
             r:   ÚforwardzAfmoeRotaryEmbedding.forwardb   sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$©N)NNN)Ú__name__Ú
__module__Ú__qualname__rG   ÚTensorÚ__annotations__r"   r-   Ústaticmethodr   ÚintÚtuplerK   r2   Úno_gradr   re   Ú__classcell__©r9   s   @r:   r$   r$   1   s˜   ø… Ø�l‰lÓñV˜{õ Vð  à%)Ø+/Ø"ñ*Ø˜dÑ"ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r;   r$   ÚRMSNormc                   óP   ‡ — e Zd Zddeddfˆ fd„Zdej                  fd„Zd„ Zˆ xZ	S )ÚAfmoeRMSNormÚepsr=   Nc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z;
        AfmoeRMSNorm is equivalent to T5LayerNorm
        N)r,   r-   r   Ú	ParameterrG   ÚonesÚweightÚvariance_epsilon)r6   rE   ru   r9   s      €r:   r-   zAfmoeRMSNorm.__init__t   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr;   c                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |z  j                  |«      S )NrA   rP   T)Úkeepdim)	rC   rJ   rG   Úfloat32ÚpowÚmeanÚrsqrtrz   ry   )r6   Úhidden_statesÚinput_dtypeÚvariances       r:   re   zAfmoeRMSNorm.forward|   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ—‘˜mÑ+×/Ñ/°Ó<Ð<r;   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)rn   ry   rW   rz   )r6   s    r:   Ú
extra_reprzAfmoeRMSNorm.extra_reprƒ   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr;   )g�íµ ÷Æ°>)
rg   rh   ri   rK   r-   rG   rj   re   r…   rp   rq   s   @r:   rt   rt   r   s,   ø„ ñ$¨ð $¸$õ $ð=¨¯©ó =öJr;   rt   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚAfmoeMLPc                 óÒ  •— t         ‰| �  «        || _        |j                  | _        |€|j                  n|| _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        t        |j                     | _        y ©NF©Úbias)r,   r-   r&   rE   Úintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn)r6   r&   rŒ   r9   s      €r:   r-   zAfmoeMLP.__init__ˆ   s®   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ=NÐ=V ×!9Ò!9Ð\mˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Ñ/ˆ�r;   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S rf   )r�   r’   rŽ   r�   )r6   r_   r�   s      r:   re   zAfmoeMLP.forward’   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr;   rf   )rg   rh   ri   r-   re   rp   rq   s   @r:   r‡   r‡   ‡   s   ø„ õ0ör;   r‡   c                   óZ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚAfmoeTokenChoiceRouterz©
    Token-choice top-K router for MoE routing.

    This router assigns each token to the top-K experts based on sigmoid scores, matching the released checkpoints.
    c                 óø   •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        t        j                  |j                  |j
                  d¬«      | _
        y r‰   )r,   r-   r&   Únum_experts_per_tokÚtop_kÚnum_expertsÚroute_scaler   r�   rE   Úgate©r6   r&   r9   s     €r:   r-   zAfmoeTokenChoiceRouter.__init__ž   s^   ø€ Ü‰ÑÔØˆŒØ×/Ñ/ˆŒ
Ø!×-Ñ-ˆÔØ!×-Ñ-ˆÔÜ—I‘I˜f×0Ñ0°&×2DÑ2DÈ5ÔQˆ�	r;   r�   Úexpert_biasc                 ó   — |j                   \  }}}|j                  d|«      }| j                  |«      j                  t        j
                  «      }t	        j                  |«      }t	        j                  ||z   | j                  d¬«      \  }}|j                  d|¬«      }|j                  dd¬«      dz   }	||	z  }|| j                  z  }|||fS )NrP   r!   )ÚkrM   )rM   ÚindexT)rM   r|   g#B’¡œÇ;)rW   Úviewr›   rJ   rG   r}   ÚsigmoidÚtopkr˜   ÚgatherÚsumrš   )
r6   r�   r�   Ú_Ú
hidden_dimÚrouter_logitsÚscoresÚselected_expertsÚ
top_scoresÚdenominators
             r:   re   zAfmoeTokenChoiceRouter.forward¦   sÅ   € Ø(×.Ñ.Ñˆˆ1ˆjØ%×*Ñ*¨2¨zÓ:ˆàŸ	™	 -Ó0×3Ñ3´E·M±MÓBˆÜ—‘˜}Ó-ˆä#Ÿj™j¨°+Ñ)=ÀÇÁÐQRÔSÑˆÐØ—]‘] qÐ0@�]ÓAˆ
Ø —n‘n¨°T�nÓ:¸UÑBˆØ +Ñ-ˆ
Ø $×"2Ñ"2Ñ2ˆ
Ø˜jÐ*:Ð:Ð:r;   ©	rg   rh   ri   Ú__doc__r-   rG   rj   re   rp   rq   s   @r:   r•   r•   —   s)   ø„ ñôRð; U§\¡\ð ;ÀÇÁ÷ ;r;   r•   c                   óŠ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  dej                  dej                  fd„Zˆ xZS )ÚAfmoeExpertsz2Collection of expert weights stored as 3D tensors.c                 óò  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        t        j                  t        j                  | j                  d| j                  z  | j                  «      «      | _        t        j                  t        j                  | j                  | j                  | j                  «      «      | _        t        |j                     | _        y )NrA   )r,   r-   r™   rE   r§   Úmoe_intermediate_sizeÚintermediate_dimr   rw   rG   ÚemptyÚgate_up_projr�   r   r‘   r’   rœ   s     €r:   r-   zAfmoeExperts.__init__¹   s­   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ ×,Ñ,ˆŒØ &× <Ñ <ˆÔÜŸL™L¬¯©°T×5EÑ5EÀqÈ4×K`ÑK`ÑG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸ™¤e§k¡k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Ñ/ˆ�r;   r�   Útop_k_indexÚtop_k_weightsr=   c                 óf  — t        j                  |«      }t        j                  «       5  t         j                  j                  j                  || j                  ¬«      }|j                  ddd«      }t        j                  |j                  d¬«      d«      j                  «       }d d d «       D ]ð  }|d   }|| j                  k(  rŒt        j                  |   «      \  }}	||	   }
t        j                  j                  |
| j                  |   «      j                  dd¬«      \  }}| j                  |«      |z  }t        j                  j                  || j                   |   «      }|||	|d f   z  }|j#                  d|	|j%                  |j&                  «      «       Œò |S # 1 sw Y   �ŒxY w)N)Únum_classesrA   r!   r   )rP   éþÿÿÿrU   rP   )rG   Ú
zeros_likero   r   Ú
functionalÚone_hotr™   ÚpermuteÚgreaterr¥   ÚnonzeroÚwhereÚlinearrµ   Úchunkr’   r�   Ú
index_add_rJ   rC   )r6   r�   r¶   r·   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stater›   ÚupÚcurrent_hidden_statess                 r:   re   zAfmoeExperts.forwardÂ   s™  € ô $×.Ñ.¨}Ó=ÐÜ�]‰]‹_ñ 	SÜŸ(™(×-Ñ-×5Ñ5°kÈt×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸ™ {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷	Sð
 %ò 
	nˆJØ# A™ˆJØ˜T×-Ñ-Ò-ØÜ#(§;¡;¨{¸:Ñ/FÓ#GÑ ˆI�yØ)¨)Ñ4ˆMÜ—}‘}×+Ñ+¨M¸4×;LÑ;LÈZÑ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆD�"Ø$(§K¡K°Ó$5¸Ñ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9NÐPT×P^ÑP^Ð_iÑPjÓ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÑ<eÑ$eÐ!Ø×*Ñ*¨1¨iÐ9N×9QÑ9QÐRe×RkÑRkÓ9lÕmð
	nð #Ð"÷#	Sñ 	Sús   ªA=F&Æ&F0r­   rq   s   @r:   r°   r°   µ   sF   ø„ á<ô0ð#à—|‘|ð#ð —\‘\ð#ð —|‘|ð	#ð
 
�‰÷#r;   r°   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚAfmoeSparseMoeBlockzÎ
    Mixture of Experts (MoE) module for AFMoE.

    This module implements a sparse MoE layer with both shared experts (always active) and
    routed experts (activated based on token-choice routing).
    c                 ó2  •— t         ‰| �  «        || _        t        |«      | _        t        ||j                  |j                  z  «      | _        t        |«      | _
        t        j                  t        j                  |j                  «      d¬«      | _        y )NF)Úrequires_grad)r,   r-   r&   r•   Úrouterr‡   r²   Únum_shared_expertsÚshared_expertsr°   Úexpertsr   rw   rG   Úzerosr™   r�   rœ   s     €r:   r-   zAfmoeSparseMoeBlock.__init__å   sp   ø€ Ü‰ÑÔØˆŒÜ,¨VÓ4ˆŒÜ& v¨v×/KÑ/KÈf×NgÑNgÑ/gÓhˆÔÜ# FÓ+ˆŒÜŸ<™<¬¯©°F×4FÑ4FÓ(GÐW\Ô]ˆÕr;   c                 ó  — |j                   \  }}}|j                  d|«      }| j                  || j                  «      \  }}}| j	                  |«      j                  |||«      }	| j                  |||«      j                  |||«      }
|	|
z   S )NrP   )rW   r¡   rÒ   r�   rÔ   rÕ   )r6   r�   Ú
batch_sizer<   r§   Úhidden_states_flatr¦   r«   rª   Úshared_outputÚrouted_outputs              r:   re   zAfmoeSparseMoeBlock.forwardí   sž   € Ø*7×*=Ñ*=Ñ'ˆ
�G˜ZØ*×/Ñ/°°JÓ?Ðð +/¯+©+°mÀT×EUÑEUÓ*VÑ'ˆˆ:Ð'ð ×+Ñ+Ð,>Ó?×DÑDÀZÐQXÐZdÓeˆØŸ™Ð%7Ð9IÈ:ÓV×[Ñ[Ø˜ ó
ˆð ˜}Ñ,Ð,r;   )rg   rh   ri   r®   r-   re   rp   rq   s   @r:   rÏ   rÏ   Ý   s   ø„ ñô^ö-r;   rÏ   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..NrP   rA   rU   )rW   rG   r\   )r_   Úx1Úx2s      r:   Úrotate_halfrß   ü   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r;   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerß   )ÚqrŸ   r]   r^   Úunsqueeze_dimÚq_embedÚk_embeds          r:   Úapply_rotary_pos_embrç     sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr;   r�   Ún_repr=   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r!   N)rW   rV   Úreshape)r�   rè   ÚbatchÚnum_key_value_headsÚslenr@   s         r:   Ú	repeat_kvrî     so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr;   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )NrA   r   rP   )rM   rC   )ÚpÚtrainingr!   )rî   Únum_key_value_groupsrG   Úmatmulr[   r   r¼   Úsoftmaxr}   rJ   rC   rõ   rù   Ú
contiguous)rï   rð   rñ   rò   ró   rô   rõ   rö   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r:   Úeager_attention_forwardr  )  sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r;   c                   óî   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej                  de	ej                  ej                  f   dej                  dz  d	e
dz  d
ee   de	ej                  ej                  f   fd„Zˆ xZS )ÚAfmoeAttentionaJ  
    Multi-headed attention module with optional sliding window and gating.

    This attention mechanism supports both full attention and sliding window attention,
    and includes Q/K normalization and gating of the output. It inherits from [`LlamaAttention`] to minimize the amount
    of custom logic we need to maintain.
    r&   Ú	layer_idxc                 óä  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                  ¬«      | _        |j(                  |   dk(  | _        | j*                  r|j,                  nd | _        t/        | j                  |j0                  ¬«      | _        t/        | j                  |j0                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  d¬«      | _        y )Nr@   g      à¿TrŠ   Úsliding_attention©ru   F)r,   r-   r&   r  rD   rE   rF   r@   rì   rú   rô   Úattention_dropoutÚ	is_causalr   r�   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚlayer_typesÚis_local_attentionÚsliding_windowrt   Úrms_norm_epsÚq_normÚk_normrŽ   ©r6   r&   r  r9   s      €r:   r-   zAfmoeAttention.__init__L  sÞ  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒð
 #)×"4Ñ"4°YÑ"?ÐCVÑ"VˆÔØ7;×7NÒ7N˜f×3Ò3ÐTXˆÔä" 4§=¡=°f×6IÑ6IÔJˆŒÜ" 4§=¡=°f×6IÑ6IÔJˆŒÜŸ™ 6×#5Ñ#5°v×7QÑ7QÐTX×TaÑTaÑ7aÐhmÔnˆ�r;   Nr�   Úposition_embeddingsró   Úpast_key_valuerö   r=   c                 óÒ  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      }| j	                  |«      j                  |«      }	| j                  |«      j                  |«      }
| j                  |«      }| j                  |«      j                  dd«      }| j                  |	«      j                  dd«      }	|
j                  dd«      }
| j                  r|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                   j"                  t$        «      } || ||	|
f|| j&                  sdn| j(                  | j*                  | j,                  dœ|¤Ž\  }} |j                  g |¢d‘­Ž j/                  «       }|t1        j2                  |«      z  }| j5                  |«      }||fS )NrP   r!   rA   ç        )ró   rõ   rô   r  )rW   r@   r  r¡   r  r  rŽ   r  r[   r  r  rç   Úupdater  r   Úget_interfacer&   Ú_attn_implementationr  rù   r	  rô   r  rý   rG   r¢   r  )r6   r�   r  ró   r  rö   Úinput_shapeÚhidden_shapeÚquery_statesrþ   rÿ   Úgate_statesr]   r^   Úattention_interfaceÚoutputr   r  s                     r:   re   zAfmoeAttention.forwardk  sà  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—[‘[ Ó/×4Ñ4°\ÓBˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓDˆØ—n‘n ]Ó3ˆà—{‘{ <Ó0×:Ñ:¸1¸aÓ@ˆØ—[‘[ Ó,×6Ñ6°q¸!Ó<ˆ
Ø#×-Ñ-¨a°Ó3ˆà×"Ò"Ø*‰HˆC�Ü';¸LÈ*ÐVYÐ[^Ó'_Ñ$ˆL˜*àÐ%Ø'5×'<Ñ'<¸ZÈÐW[×WeÑWeÓ'fÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ  3ØØØØð	
 
ð
 *Ø#Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
 
ð ñ
 
Ñˆ�ð �—‘Ð.˜kÐ.¨2Ò.×9Ñ9Ó;ˆØœ%Ÿ-™-¨Ó4Ñ4ˆØ—k‘k &Ó)ˆØ˜LÐ(Ð(r;   rf   )rg   rh   ri   r®   r"   rm   r-   rG   rj   rn   r	   r   r   re   rp   rq   s   @r:   r  r  B  sž   ø„ ñðo˜{ð o°sõ oðH (,ñ.)à—|‘|ð.)ð # 5§<¡<°·±Ð#=Ñ>ð.)ð Ÿ™ tÑ+ð	.)ð
  ™ð.)ð Ð+Ñ,ð.)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷.)r;   r  c                   ó  ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  d	e
dz  d
edz  deej                  ej                  f   dz  dee   dej                  fd„Zˆ xZS )ÚAfmoeDecoderLayerzÌ
    AFMoE decoder layer with dual normalization.

    This layer applies self-attention followed by either a dense MLP or MoE block,
    with dual normalization (pre and post) around each component.
    r&   r  c                 ó(  •— t         ‰| �  «        |j                  | _        || _        t	        ||¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _	        t        |j                  |j                  ¬«      | _
        t        |j                  |j                  ¬«      | _        ||j                  k\  | _        | j                  rt        |«      | _        y t!        |«      | _        y )N)r&   r  r  )r,   r-   rE   r  r  Ú	self_attnrt   r  Úinput_layernormÚpost_attention_layernormÚpre_mlp_layernormÚpost_mlp_layernormÚnum_dense_layersÚmoe_enabledrÏ   Úmlpr‡   r  s      €r:   r-   zAfmoeDecoderLayer.__init__¤  s×   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ"ˆŒä'¨vÀÔKˆŒô  ,¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ(4°V×5GÑ5GÈV×M`ÑM`Ô(aˆÔ%ô ".¨f×.@Ñ.@Àf×FYÑFYÔ!ZˆÔÜ".¨v×/AÑ/AÀv×GZÑGZÔ"[ˆÔð %¨×(?Ñ(?Ñ?ˆÔØ×ÒÜ*¨6Ó2ˆD�Hä Ó'ˆD�Hr;   Nr�   ró   r`   r  Ú	use_cacher  rö   r=   c           
      ó  — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	| j                  |«      }||z   }|}| j                  |«      }| j	                  |«      }| j                  |«      }||z   }|S )N)r�   ró   r`   r  r/  r  © )r(  r'  r)  r*  r.  r+  )
r6   r�   ró   r`   r  r/  r  rö   Úresidualr¦   s
             r:   re   zAfmoeDecoderLayer.forwardº  s²   € ð !ˆð ×,Ñ,¨]Ó;ˆØ)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø)ØØ 3ñ
ð ñ
Ñˆ�qð ×5Ñ5°mÓDˆØ  =Ñ0ˆð !ˆØ×.Ñ.¨}Ó=ˆØŸ™ Ó/ˆØ×/Ñ/°Ó>ˆà  =Ñ0ˆØÐr;   )NNNNN)rg   rh   ri   r®   r"   rm   r-   rG   rj   Ú
LongTensorr	   Úboolrn   r   r   ÚFloatTensorre   rp   rq   s   @r:   r%  r%  œ  sÂ   ø„ ñð(˜{ð (°sõ (ð2 /3Ø04Ø'+Ø!%ØHLñ!à—|‘|ð!ð Ÿ™ tÑ+ð!ð ×&Ñ&¨Ñ-ð	!ð
  ™ð!ð ˜$‘;ð!ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð!ð Ð+Ñ,ð!ð 
×	Ñ	÷!r;   r%  c                   óz   ‡ — e Zd ZU dZeed<   dZdgZdgZ e	e
d¬«      eedœZg d	¢Zd
Zd
Zd
Zd
Zd
Zd
Zˆ fd„Zˆ xZS )ÚAfmoePreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    r&   Úmodelr%  Úpast_key_valuesr   )r    )r¨   r�   Ú
attentions)r(  r)  r*  r+  r  r  Únormr�   Tc                 óÎ  •— t         ‰| �  |«       | j                  j                  }t	        |t
        «      rEt        j                  |j                  d|¬«       t        j                  |j                  d|¬«       yt	        |t        «      r*t        j                  |j                  j                  «       yt	        |t        «      r t        j                  |j                  «       yy)zInitialize the weightsr  )r   ÚstdN)r,   Ú_init_weightsr&   Úinitializer_rangerX   r°   ÚinitÚnormal_rµ   r�   r•   Úzeros_r›   ry   rÏ   r�   )r6   rï   r=  r9   s      €r:   r>  z"AfmoePreTrainedModel._init_weightsþ  s˜   ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœlÔ+Ü�L‰L˜×,Ñ,°3¸CÕ@Ü�L‰L˜×)Ñ)°¸Ö=Ü˜Ô 6Ô7Ü�K‰K˜Ÿ™×*Ñ*Õ+Ü˜Ô 3Ô4Ü�K‰K˜×*Ñ*Õ+ð 5r;   )rg   rh   ri   r®   r"   rk   Úbase_model_prefixÚ_no_split_modulesÚ_skip_keys_device_placementr   r•   r%  r  Ú_can_record_outputsÚ_keep_in_fp32_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚsupports_gradient_checkpointingr>  rp   rq   s   @r:   r7  r7  Þ  sx   ø… ñð
 ÓØÐØ,Ð-ÐØ#4Ð"5Ðá'Ð(>ÀaÔHØ*Ø$ñÐò
	Ðð €NØÐØÐØ!ÐØ"&ÐØ&*Ð#÷
,ð 
,r;   r7  c                   óü   ‡ — e Zd ZdZdefˆ fd„Zeee	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  d	edz  d
edz  dee   deez  fd„«       «       «       Zˆ xZS )Ú
AfmoeModelz›
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`AfmoeDecoderLayer`]

    Args:
        config: AfmoeConfig
    r&   c           	      ó  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t#        |¬«      | _        d| _        | j)                  «        y c c}w )Nr  ©r&   F)r,   r-   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrE   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr%  Úlayersrt   r  r;  r$   Ú
rotary_embÚgradient_checkpointingÚ	post_initr  s      €r:   r-   zAfmoeModel.__init__  sÄ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜCHÈ×IaÑIaÓCbÖc°iÔ˜v yÕ1Òcó
ˆŒô ! ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ü.°fÔ=ˆŒØ&+ˆÔ#à�‰Õùò ds   ÂDNÚ	input_idsró   Úinputs_embedsr`   r9  r/  rö   r=   c           
      ó  — |d u |d uz  rt        d«      ‚|r|€t        | j                  ¬«      }|€| j                  |«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s(| j                  |||dœ}
t        di |
¤Žt        di |
¤Ždœ}	|}| j                  j                  r|| j                  j                  dz  z  }| j!                  ||«      }t#        | j$                  «      D ].  \  }} ||f|	| j                  j&                  |      ||||d	œ|¤Ž}Œ0 | j)                  |«      }t+        ||r|¬
«      S d ¬
«      S )Nz:You must specify exactly one of input_ids or inputs_embedsrQ  r   r!   )r7   )r&   r_  ró   r9  )Úfull_attentionr  g      à?)ró   r`   r  r/  r  )Úlast_hidden_stater9  r1  )Ú
ValueErrorr
   r&   rV  Úget_seq_lengthrG   rH   rW   r7   râ   rX   Údictr   r   Úmup_enabledrE   r[  Ú	enumeraterZ  r  r;  r   )r6   r^  ró   r_  r`   r9  r/  rö   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr�   r  ÚiÚdecoder_layers                  r:   re   zAfmoeModel.forward#  s¹  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZá˜Ð0Ü*°$·+±+Ô>ˆOàÐ Ø ×-Ñ-¨iÓ8ˆMàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-ÄÔFàŸ+™+Ø!.Ø"0Ø#2ñ	ˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%Uñ#Ðð
 &ˆð �;‰;×"Ò"Ø)¨T¯[©[×-DÑ-DÀcÑ-IÑJˆMà"Ÿo™o¨m¸\ÓJÐä )¨$¯+©+Ó 6ò 		ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ)Ø.Ø#Ø$7ñð ñ‰Mð		ð Ÿ	™	 -Ó0ˆÜ%Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
r;   )NNNNNN)rg   rh   ri   r®   r"   r-   r   r   r    rG   r3  rj   r5  r	   r4  r   r   rn   r   re   rp   rq   s   @r:   rO  rO    sØ   ø„ ñð˜{õ ð ØØð .2Ø.2Ø26Ø04Ø(,Ø!%ñ<
à×#Ñ# dÑ*ð<
ð Ÿ™ tÑ+ð<
ð ×(Ñ(¨4Ñ/ð	<
ð
 ×&Ñ&¨Ñ-ð<
ð  ™ð<
ð ˜$‘;ð<
ð Ð+Ñ,ð<
ð 
Ð'Ñ	'ò<
ó ó  ó ô<
r;   rO  c                   óN  ‡ — e Zd ZddiZddiZddgdgfiZˆ fd„Zee	 	 	 	 	 	 	 	 	 dd	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  dedz  dedz  dee	j                  z  dee   defd„«       «       Zˆ xZS )ÚAfmoeForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr�   Úlogitsc                 óè   •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        | j                  «        y r‰   )
r,   r-   rO  r8  rT  r   r�   rE   ro  r]  rœ   s     €r:   r-   zAfmoeForCausalLM.__init__k  sS   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ�‰Õr;   Nr^  ró   r`   r9  r_  Úlabelsr/  Úoutput_router_logitsÚlogits_to_keeprö   r=   c
                 ó®  — |�|n| j                   j                  } | j                  d|||||||dœ|
¤Ž}|j                  }t	        |	t
        «      rt        |	 d«      n|	}| j                  |dd…|dd…f   «      }d}|� | j                  ||| j                  fi |
¤Ž}t        |||j                  |j                  |j                  |j                  ¬«      S )aÏ  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, AfmoeForCausalLM

        >>> model = AfmoeForCausalLM.from_pretrained("meta-afmoe/Afmoe-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-afmoe/Afmoe-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r^  ró   r`   r9  r_  r/  rt  )Úlossrq  r9  r�   r:  r¨   r1  )r&   rt  r8  rb  rX   rm   Úslicero  Úloss_functionrT  r   r9  r�   r:  r¨   )r6   r^  ró   r`   r9  r_  rs  r/  rt  ru  rö   Úoutputsr�   Úslice_indicesrq  rw  s                   r:   re   zAfmoeForCausalLM.forwardr  sø   € ðB %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð +5¨$¯*©*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ñ	+
ð ñ	+
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDä(ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r;   )	NNNNNNNNr   )rg   rh   ri   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr-   r   r   rG   r3  rj   r	   r5  r4  rm   r   r   r   re   rp   rq   s   @r:   rn  rn  e  s(  ø„ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hôð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ñ<
à×#Ñ# dÑ*ð<
ð Ÿ™ tÑ+ð<
ð ×&Ñ&¨Ñ-ð	<
ð
  ™ð<
ð ×(Ñ(¨4Ñ/ð<
ð × Ñ  4Ñ'ð<
ð ˜$‘;ð<
ð # T™kð<
ð ˜eŸl™lÑ*ð<
ð Ð+Ñ,ð<
ð 
#ò<
ó ó ô<
r;   rn  )rn  rO  r7  )r!   )r  )EÚcollections.abcr   Útypingr   rG   r   Ú r   r@  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   r   r   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr   r    Úconfiguration_afmoer"   ÚModuler$   rt   r‡   r•   r°   rÏ   rß   rç   rj   rm   rî   rK   r  r  r%  r7  rO  rn  Ú__all__r1  r;   r:   ú<module>r’     s  ðõ* %Ý ã Ý å &Ý !ß .Ý )÷ó ÷ SÝ 9ß Qß Kß FÝ &ß IÑ Iß Gß EÝ ,ô><˜2Ÿ9™9ô ><ñB ˜YÓ'ôJ�2—9‘9ó Jó (ðJô(ˆr�y‰yô ô ;˜RŸY™Yô ;ð< ô$#�2—9‘9ó $#ó ð$#ôN-˜"Ÿ)™)ô -ò>(ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ñ2 Ð)Ó*ôV)�R—Y‘Yó V)ó +ðV)ôr?Ð2ô ?ôD*,˜?ô *,ðZ ôV
Ð%ó V
ó ðV
ðr ôJ
Ð+¨_ó J
ó ðJ
òZ E�r;   