Ë
    ÛÓ^jhu  ã                   ó`  — d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1  ed«       G d„ dejd                  «      «       Z3 G d„ dejd                  «      Z4 G d„ dejd                  «      Z5e G d„ dejd                  «      «       Z6 G d„ d ejd                  «      Z7d!„ Z8 ed"«      d?d#„«       Z9d$ejt                  d%e;d&ejt                  fd'„Z<	 d@d(ejd                  d)ejt                  d*ejt                  d+ejt                  d,ejt                  dz  d-e=d.e=d/e&e(   fd0„Z> ee9«       G d1„ d2ejd                  «      «       Z? G d3„ d4e«      Z@e) G d5„ d6e$«      «       ZAe) G d7„ d8eA«      «       ZB	 	 	 dAd9ejt                  eCejt                     z  dz  d:e;dz  d,ejt                  dz  d&ejt                  e;z  fd;„ZDe) G d<„ d=eAe«      «       ZEg d>¢ZFy)Bé    )ÚCallable)ÚOptionalN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstring)Úcan_return_tupleÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚGraniteMoeConfigÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚGraniteMoeRMSNormÚepsÚreturnNc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z@
        GraniteMoeRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer$   Ú	__class__s      €ú}/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/granitemoe/modeling_granitemoe.pyr(   zGraniteMoeRMSNorm.__init__5   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕó    Úhidden_statesc                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor*   Úfloat32ÚpowÚmeanÚrsqrtr-   r,   )r.   r3   Úinput_dtypeÚvariances       r1   ÚforwardzGraniteMoeRMSNorm.forward=   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r2   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler,   Úshaper-   )r.   s    r1   Ú
extra_reprzGraniteMoeRMSNorm.extra_reprD   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr2   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Úfloatr(   r*   ÚTensorr@   rD   Ú__classcell__©r0   s   @r1   r#   r#   3   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJr2   r#   c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚGraniteMoeRotaryEmbeddingÚinv_freqNÚconfigc                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultrN   F)Ú
persistentÚoriginal_inv_freq)r'   r(   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrO   Úrope_parametersrQ   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r.   rO   ÚdeviceÚrope_init_fnrN   r0   s        €r1   r(   z"GraniteMoeRotaryEmbedding.__init__K   s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr2   r]   ztorch.deviceÚseq_lenr%   ztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   r5   ©r8   )r]   r8   )	rX   Úgetattrr/   Únum_attention_headsr*   ÚarangeÚint64r9   rH   )rO   r]   r_   ÚbaseÚdimÚattention_factorrN   s          r1   rY   z9GraniteMoeRotaryEmbedding.compute_default_rope_parameters[   s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r2   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   r6   r   ÚmpsÚcpuF)Údevice_typeÚenabledr5   ©ri   rc   )rN   rH   ÚexpandrC   r9   r]   Ú
isinstanceÚtypeÚstrr   Ú	transposer*   ÚcatÚcosrZ   Úsinr8   )
r.   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrn   ÚfreqsÚembrw   rx   s
             r1   r@   z!GraniteMoeRotaryEmbedding.forwardy   sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$©N©NNN)rE   rF   rG   r*   rI   Ú__annotations__r    r(   Ústaticmethodr   ÚintrB   rH   rY   Úno_gradr   r@   rJ   rK   s   @r1   rM   rM   H   s™   ø… Ø�l‰lÓñVÐ/õ Vð  à*.Ø+/Ø"ñ*Ø  4Ñ'ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r2   rM   c                   ó”   ‡ — e Zd ZdZdefˆ fd„Zdej                  deej                  ej                  ej                  f   fd„Z	ˆ xZ
S )ÚGraniteMoeTopKRoutera¡  Top-k gating that returns the routing decisions without grouping tokens by expert.

    Returns ``(top_k_index, top_k_weights, router_logits)``; the grouping/scattering used to live
    here (via ``expert_size.tolist()``, which broke fullgraph compile) and now happens inside the
    experts forward via ``use_experts_implementation`` so the default ``grouped_mm`` / ``batched_mm``
    paths can compile cleanly.
    rO   c                 óê   •— t         ‰| �  «        |j                  | _        |j                  | _        t        j                  t        j                  | j                  |j                  «      «      | _        y r   )r'   r(   Únum_local_expertsÚnum_expertsÚnum_experts_per_tokÚtop_kr   r)   r*   Úemptyr/   r,   ©r.   rO   r0   s     €r1   r(   zGraniteMoeTopKRouter.__init__’   sP   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ×/Ñ/ˆŒ
Ü—l‘l¤5§;¡;¨t×/?Ñ/?À×ASÑASÓ#TÓUˆ�r2   r3   r%   c                 óô   — t        j                  || j                  «      j                  «       }|j	                  | j
                  d¬«      \  }}t        j                  |d¬«      j                  |«      }|||fS )Nr6   rp   )	ÚFÚlinearr,   rH   Útopkr‹   r*   ÚsoftmaxÚtype_as)r.   r3   Úrouter_logitsÚtop_k_logitsÚtop_k_indexÚtop_k_weightss         r1   r@   zGraniteMoeTopKRouter.forward˜   se   € ÜŸ™ °·±Ó<×BÑBÓDˆØ$1×$6Ñ$6°t·z±zÀrÐ$6Ó$JÑ!ˆ�kÜŸ™ l¸Ô;×CÑCÀMÓRˆØ˜M¨=Ð8Ð8r2   )rE   rF   rG   Ú__doc__r    r(   r*   rI   rB   r@   rJ   rK   s   @r1   r†   r†   ‰   sI   ø„ ñðVÐ/õ Vð9 U§\¡\ð 9°e¸E¿L¹LÈ%Ï,É,ÐX]×XdÑXdÐ<dÑ6e÷ 9r2   r†   c                   ó�   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  dej                  dej                  fd„Zˆ xZ	S )	ÚGraniteMoeExpertsz2Collection of expert weights stored as 3D tensors.rO   c                 óò  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        t        j                  t        j                  | j                  d| j                  z  | j
                  «      «      | _        t        j                  t        j                  | j                  | j
                  | j                  «      «      | _        t        |j                     | _        y )Nr5   )r'   r(   rˆ   r‰   r/   Ú
hidden_dimÚintermediate_sizeÚintermediate_dimr   r)   r*   rŒ   Úgate_up_projÚ	down_projr   Ú
hidden_actÚact_fnr�   s     €r1   r(   zGraniteMoeExperts.__init__£   s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× 8Ñ 8ˆÔÜŸL™L¬¯©°T×5EÑ5EÀqÈ4×K`ÑK`ÑG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸ™¤e§k¡k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Ñ/ˆ�r2   r3   r–   r—   r%   c                 óf  — t        j                  |«      }t        j                  «       5  t         j                  j                  j                  || j                  ¬«      }|j                  ddd«      }t        j                  |j                  d¬«      d«      j                  «       }d d d «       D ]ð  }|d   }|| j                  k(  rŒt        j                  |   «      \  }}	||	   }
t        j                  j                  |
| j                  |   «      j                  dd¬«      \  }}| j                  |«      |z  }t        j                  j                  || j                   |   «      }|||	|d f   z  }|j#                  d|	|j%                  |j&                  «      «       Œò |S # 1 sw Y   �ŒxY w)N)Únum_classesr5   r   r   )r6   éþÿÿÿrp   r6   )r*   Ú
zeros_liker„   r   Ú
functionalÚone_hotr‰   ÚpermuteÚgreaterÚsumÚnonzeroÚwherer�   rŸ   Úchunkr¢   r    Ú
index_add_r9   r8   )r.   r3   r–   r—   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 r1   r@   zGraniteMoeExperts.forward¬   s™  € ô $×.Ñ.¨}Ó=ÐÜ�]‰]‹_ñ 	SÜŸ(™(×-Ñ-×5Ñ5°kÈt×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸ™ {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷	Sð
 %ò 
	nˆJØ# A™ˆJØ˜T×-Ñ-Ò-ØÜ#(§;¡;¨{¸:Ñ/FÓ#GÑ ˆI�yØ)¨)Ñ4ˆMÜ—}‘}×+Ñ+¨M¸4×;LÑ;LÈZÑ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆD�"Ø$(§K¡K°Ó$5¸Ñ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9NÐPT×P^ÑP^Ð_iÑPjÓ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÑ<eÑ$eÐ!Ø×*Ñ*¨1¨iÐ9N×9QÑ9QÐRe×RkÑRkÓ9lÕmð
	nð #Ð"÷#	Sñ 	Sús   ªA=F&Æ&F0©
rE   rF   rG   r˜   r    r(   r*   rI   r@   rJ   rK   s   @r1   rš   rš   Ÿ   sN   ø„ á<ð0Ð/õ 0ð#à—|‘|ð#ð —\‘\ð#ð —|‘|ð	#ð
 
�‰÷#r2   rš   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚGraniteMoeMoEzISparsely-gated mixture-of-experts block: router decides, experts compute.rO   c                 ó„   •— t         ‰| �  «        |j                  | _        t	        |«      | _        t        |«      | _        y r   )r'   r(   r/   Ú
input_sizer†   Úrouterrš   Úexpertsr�   s     €r1   r(   zGraniteMoeMoE.__init__Ê   s3   ø€ Ü‰ÑÔØ ×,Ñ,ˆŒÜ*¨6Ó2ˆŒÜ(¨Ó0ˆ�r2   Úlayer_inputr%   c                 óØ   — |j                  «       \  }}}|j                  d|«      }| j                  |«      \  }}}| j                  |||«      }	|	j	                  ||| j
                  «      S )Nr6   )ÚsizeÚreshaper¿   rÀ   Úviewr¾   )
r.   rÁ   ÚbszÚlengthÚemb_sizer3   r–   r—   Ú_Úlayer_outputs
             r1   r@   zGraniteMoeMoE.forwardÐ   si   € Ø +× 0Ñ 0Ó 2ÑˆˆV�XØ#×+Ñ+¨B°Ó9ˆØ(,¯©°MÓ(BÑ%ˆ�] AØ—|‘| M°;ÀÓNˆØ× Ñ   f¨d¯o©oÓ>Ð>r2   rº   rK   s   @r1   r¼   r¼   Ç   s.   ø„ ÙSð1Ð/õ 1ð? 5§<¡<ð ?°E·L±L÷ ?r2   r¼   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr6   r5   rp   )rC   r*   rv   )ry   Úx1Úx2s      r1   Úrotate_halfrÎ   Ø   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r2   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerÎ   )ÚqÚkrw   rx   Úunsqueeze_dimÚq_embedÚk_embeds          r1   Úapply_rotary_pos_embr×   ß   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr2   r3   Ún_repr%   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)rC   rq   rÄ   )r3   rØ   ÚbatchÚnum_key_value_headsÚslenrb   s         r1   Ú	repeat_kvrÝ   ù   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr2   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )Nr5   r   r6   )ri   r8   )ÚpÚtrainingr   )rÝ   Únum_key_value_groupsr*   Úmatmulru   r   r§   r’   r:   r9   r8   rä   rè   Ú
contiguous)rÞ   rß   rà   rá   râ   rã   rä   rå   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r1   Úeager_attention_forwardrð     sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r2   c                   óø   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
dz  d
ee   de	ej                  ej                  f   fd„Zˆ xZS )ÚGraniteMoeAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrO   Ú	layer_idxc                 ó^  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        |j                  | _        |j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                   ¬«      | _        y )Nrb   T©Úbias)r'   r(   rO   ró   rd   r/   re   rb   rÛ   ré   Úattention_multiplierrã   Úattention_dropoutÚ	is_causalr   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_proj©r.   rO   ró   r0   s      €r1   r(   zGraniteMoeAttention.__init__"  sJ  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø×2Ñ2ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆ�r2   Nr3   Úposition_embeddingsrâ   Úpast_key_valuesrå   r%   c                 ó
  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  sdn| j                   | j"                  dœ|¤Ž\  }} |j$                  g |¢d‘­Ž j'                  «       }| j)                  |«      }||fS )Nr6   r   r5   ç        )rä   rã   )rC   rb   rü   rÅ   ru   rý   rþ   r×   Úupdateró   r   Úget_interfacerO   Ú_attn_implementationrð   rè   rø   rã   rÄ   rë   rÿ   )r.   r3   r  râ   r  rå   Úinput_shapeÚhidden_shapeÚquery_statesrì   rí   rw   rx   Úattention_interfacerï   rî   s                   r1   r@   zGraniteMoeAttention.forward9  s‘  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r2   r€   )rE   rF   rG   r˜   r    rƒ   r(   r*   rI   rB   r	   r   r   r@   rJ   rK   s   @r1   rò   rò     s¦   ø„ áGð
Ð/ð 
¸Cõ 
ð4 IMØ.2Ø(,ñ&)à—|‘|ð&)ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð&)ð Ÿ™ tÑ+ð	&)ð
  ™ð&)ð Ð+Ñ,ð&)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷&)r2   rò   c                   óÌ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej                  dej                  dz  dedz  de	ej                  ej                  f   dz  d	ej                  f
d
„Z
ˆ xZS )ÚGraniteMoeDecoderLayerrO   ró   c                 óB  •— t         ‰| �  «        |j                  | _        t        ||¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _
        |j                  | _        y )N)rO   ró   ©r$   )r'   r(   r/   rò   Ú	self_attnr#   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr¼   Úblock_sparse_moeÚresidual_multiplierr   s      €r1   r(   zGraniteMoeDecoderLayer.__init__c  s|   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ,°FÀiÔPˆŒÜ0°×1CÑ1CÈ×I\ÑI\Ô]ˆÔÜ(9¸&×:LÑ:LÐRX×ReÑReÔ(fˆÔ%Ü -¨fÓ 5ˆÔØ#)×#=Ñ#=ˆÕ r2   Nr3   râ   r  r  r%   c                 óð   — |}| j                  |«      } | j                  d||||dœ|¤Ž\  }}||| j                  z  z   }|}| j                  |«      }| j	                  |«      }||| j                  z  z   }|S )N)r3   râ   r  r  © )r  r  r  r  r  )r.   r3   râ   r  r  rå   ÚresidualrÉ   s           r1   r@   zGraniteMoeDecoderLayer.forwardl  s    € ð !ˆØ×,Ñ,¨]Ó;ˆØ)˜4Ÿ>™>ð 
Ø'Ø)Ø+Ø 3ñ	
ð
 ñ
Ñˆ�qð ! =°4×3KÑ3KÑ#KÑKˆØ ˆØ×5Ñ5°mÓDˆØ×-Ñ-¨mÓ<ˆØ  =°4×3KÑ3KÑ#KÑKˆØÐr2   r€   )rE   rF   rG   r    rƒ   r(   r*   rI   r	   rB   r@   rJ   rK   s   @r1   r  r  b  sƒ   ø„ ð>Ð/ð >¸Cõ >ð /3Ø(,ØHLñà—|‘|ðð Ÿ™ tÑ+ðð  ™ð	ð
 # 5§<¡<°·±Ð#=Ñ>ÀÑEðð 
�‰÷r2   r  c                   ó„   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZ ej$                  «       ˆ fd„«       Zˆ xZS )ÚGraniteMoePreTrainedModelrO   ÚmodelTr  r  )r3   Ú
attentionsc                 ó¬  •— t         ‰| �  |«       t        |t        «      rmt	        j
                  |j                  d| j                  j                  ¬«       t	        j
                  |j                  d| j                  j                  ¬«       y t        |t        «      r7t	        j
                  |j                  d| j                  j                  ¬«       y y )Nr  )r<   Ústd)r'   Ú_init_weightsrr   rš   ÚinitÚnormal_rŸ   rO   Úinitializer_ranger    r†   r,   )r.   rÞ   r0   s     €r1   r  z'GraniteMoePreTrainedModel._init_weights–  s‰   ø€ ä‰Ñ˜fÔ%Ü�fÔ/Ô0Ü�L‰L˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�L‰L˜×)Ñ)°¸¿¹×9VÑ9VÖWÜ˜Ô 4Ô5Ü�L‰L˜Ÿ™¨S°d·k±k×6SÑ6SÖTð 6r2   )rE   rF   rG   r    r�   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr  rò   Ú_can_record_outputsr*   r„   r  rJ   rK   s   @r1   r  r  …  sp   ø… àÓØÐØ&*Ð#Ø1Ð2ÐØ#4Ð"5ÐØÐØ€NØÐØ!ÐØ"&Ðà/Ø)ñÐð
 €U‡]�]ƒ_óUó ôUr2   r  c                   óò   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )ÚGraniteMoeModelrO   c           	      ó.  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t#        |¬«      | _        d| _        |j(                  | _        | j+                  «        y c c}w )Nr  ©rO   F)r'   r(   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr/   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr  Úlayersr#   r  ÚnormrM   Ú
rotary_embÚgradient_checkpointingÚembedding_multiplierÚ	post_initr   s      €r1   r(   zGraniteMoeModel.__init__¢  sÔ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜHMÈf×NfÑNfÓHgÖh¸9Ô# F¨IÕ6Òhó
ˆŒô & f×&8Ñ&8¸f×>QÑ>QÔRˆŒ	Ü3¸6ÔBˆŒØ&+ˆÔ#Ø$*×$?Ñ$?ˆÔ!ð 	�‰Õùò is   ÂDNÚ	input_idsrâ   rz   r  Úinputs_embedsÚ	use_cacherå   r%   c           
      óX  — |d u |d uz  rt        d«      ‚|r|€t        | j                  ¬«      }|€| j                  |«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        | j                  ||||¬«      }	|| j                  z  }|}
| j                  |
|«      }| j                  d | j                  j                   D ]  } ||
f||	|||dœ|¤Ž}
Œ | j                  |
«      }
t!        |
|¬«      S )	Nz:You must specify exactly one of input_ids or inputs_embedsr0  r   r   )r]   )rO   r@  râ   r  rz   )r  râ   rz   r  rA  )Úlast_hidden_stater  )Ú
ValueErrorr
   rO   r5  Úget_seq_lengthr*   rf   rC   r]   rÑ   r   r=  r;  r9  r8  r:  r   )r.   r?  râ   rz   r  r@  rA  rå   Úpast_seen_tokensÚcausal_maskr3   r  Údecoder_layers                r1   r@   zGraniteMoeModel.forward³  s\  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZá˜Ð0Ü*°$·+±+Ô>ˆOàÐ Ø ×-Ñ-¨iÓ8ˆMàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &¨×(AÑ(AÑAˆØ%ˆð #Ÿo™o¨m¸\ÓJÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 		ˆMÙ)Øðà$7Ø*Ø)Ø /Ø#ñð ñ‰Mð		ð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r2   )NNNNNN)rE   rF   rG   r    r(   r   r   r   r*   Ú
LongTensorrI   r	   ÚFloatTensorÚboolr   r   r   r@   rJ   rK   s   @r1   r.  r.     sÎ   ø„ ðÐ/õ ð"  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ5
à×#Ñ# dÑ*ð5
ð Ÿ™ tÑ+ð5
ð ×&Ñ&¨Ñ-ð	5
ð
  ™ð5
ð ×(Ñ(¨4Ñ/ð5
ð ˜$‘;ð5
ð Ð+Ñ,ð5
ð 
 ò5
ó ó ó  ô5
r2   r.  Úgate_logitsr‰   c                 ó¾  — | �t        | t        «      syt        | t        «      rC| d   j                  }t        j                  | D �cg c]  }|j                  |«      ‘Œ c}d¬«      }t        j                  j                  j                  d¬«      }t        j                  ||d¬«      \  }}	t        j                  j                  j                  |	|«      }
|€>t        j                  |
j                  «       d¬«      }t        j                  |d¬«      }�n|j                  \  }}|j                  d   ||z  z  }|ddd…dd…ddf   j                  |||||f«      j                  d||«      j                  «      }t        j                   |
j                  «       |z  d¬«      t        j                   |d¬«      z  }|ddd…dd…df   j                  ||||f«      j                  d|«      j                  |«      }t        j                   ||z  d¬«      t        j                   |d¬«      z  }t        j                   ||j#                  d«      z  «      }||z  S c c}w )aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   rp   r6   )rr   rB   r]   r*   rv   r9   r   r§   r’   r‘   r¨   r<   rH   rC   rq   rÄ   r«   rÑ   )rL  r‰   r‹   râ   Úcompute_deviceÚ
layer_gateÚconcatenated_gate_logitsÚrouting_weightsrÉ   Úselected_expertsr±   Útokens_per_expertÚrouter_prob_per_expertÚ
batch_sizeÚsequence_lengthr8  Úexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_losss                      r1   Úload_balancing_loss_funcrZ  î  s9  € ð: Ð¤*¨[¼%Ô"@Øä�+œuÔ%Ø$ Q™×.Ñ.ˆÜ#(§9¡9Ð^iÖ-jÐPZ¨j¯m©m¸NÕ.KÒ-jÐpqÔ#rÐ ä—h‘h×)Ñ)×1Ñ1Ð2JÐPRÐ1ÓS€OäŸ*™* _°eÀÔDÑ€AÐä—(‘(×%Ñ%×-Ñ-Ð.>ÀÓL€KàÐä!ŸJ™J {×'8Ñ'8Ó':ÀÔBÐô "'§¡¨OÀÔ!CÒà&4×&:Ñ&:Ñ#ˆ
�OØ4×:Ñ:¸1Ñ=À*ÈÑB^Ñ_Ðð ˜4¢¢A t¨TÐ1Ñ2ß‰VÐ&¨
°OÀUÈKÐXÓYß‰W�R˜ Ó,ß‰R�Óð	 	ô "ŸI™I k×&7Ñ&7Ó&9Ð<QÑ&QÐWXÔYÔ\a×\eÑ\eØ! qô]
ñ 
Ðð ˜4¢¢A tÐ+Ñ,ß‰VÐ&¨
°OÀ[ÐQÓRß‰W�R˜Ó%ß‰R�Óð	 	)ô "'§¡¨?Ð=]Ñ+]ÐcdÔ!eÔhm×hqÑhqØ,°!ôi
ñ "
Ðô —9‘9Ð.Ð1G×1QÑ1QÐRSÓ1TÑTÓU€LØ˜+Ñ%Ð%ùò[ .ks   ÁIc                   óD  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zee		 	 	 	 	 	 	 	 dd
e
j                  d	z  de
j                  d	z  de
j                  d	z  ded	z  de
j                  d	z  de
j                  d	z  ded	z  dee
j                  z  deez  fd„«       «       Zˆ xZS )ÚGraniteMoeForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr3   ÚlogitsrO   c                 óp  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        |j                  | _	        |j                  | _        |j                  | _        |j                  | _        | j                  «        y )NFrõ   )r'   r(   r.  r  r3  r   rú   r/   r]  Úrouter_aux_loss_coefrˆ   r‰   rŠ   Úlogits_scalingr>  r�   s     €r1   r(   zGraniteMoeForCausalLM.__init__F  s�   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×3Ñ3ˆÔØ#)×#=Ñ#=ˆÔ Ø$×3Ñ3ˆÔð 	�‰Õr2   Nr?  râ   rz   r  r@  ÚlabelsÚoutput_router_logitsÚlogits_to_keepr%   c	           	      ó®  — |�|n| j                   j                  } | j                  d|||||dœ|	¤Ž}
|
j                  }t	        |t
        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }|| j                   j                  z  }d}|�* | j                  ||fd| j                   j                  i|	¤Ž}d}|rYt        |
j                  | j                  | j                  |«      }|�+|| j                  |j!                  |j"                  «      z  z  }t%        ||||
j&                  |
j(                  |
j*                  |
j                  ¬«      S )al  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, GraniteMoeForCausalLM

        >>> model = GraniteMoeForCausalLM.from_pretrained("ibm/PowerMoE-3b")
        >>> tokenizer = AutoTokenizer.from_pretrained("ibm/PowerMoE-3b")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r?  râ   rz   r  r@  r3  )ÚlossÚaux_lossr_  r  r3   r  r”   r  )rO   rd  r  rC  rr   rƒ   Úslicer]  rb  Úloss_functionr3  rZ  r”   r‰   rŠ   ra  r9   r]   r   r  r3   r  )r.   r?  râ   rz   r  r@  rc  rd  re  rå   Úoutputsr3   Úslice_indicesr_  rg  rh  s                   r1   r@   zGraniteMoeForCausalLM.forwardS  s|  € ðJ %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð �$—*‘*ð 
ØØ)Ø%Ø+Ø'ñ
ð ñ
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ˜$Ÿ+™+×4Ñ4Ñ4ˆàˆØÐà%�4×%Ñ%ØØñð  Ÿ;™;×1Ñ1ðð ñ	ˆDð ˆÙÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ð!Ø˜×1Ñ1°H·K±KÀÇÁÓ4LÑLÑL�Ü(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r2   )NNNNNNNr   )rE   rF   rG   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr    r(   r   r   r*   rI  rI   r	   rJ  rK  rƒ   rB   r   r@   rJ   rK   s   @r1   r\  r\  @  s   ø„ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HðÐ/õ ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø,0Ø-.ñQ
à×#Ñ# dÑ*ðQ
ð Ÿ™ tÑ+ðQ
ð ×&Ñ&¨Ñ-ð	Q
ð
  ™ðQ
ð ×(Ñ(¨4Ñ/ðQ
ð × Ñ  4Ñ'ðQ
ð # T™kðQ
ð ˜eŸl™lÑ*ðQ
ð 
Ð*Ñ	*òQ
ó ó ôQ
r2   r\  )r\  r.  r  )r   )r  )Nr5   N)GÚcollections.abcr   Útypingr   r*   Útorch.nn.functionalr   r§   r�   Ú r   r   Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   r   r   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   r   r   Úutils.output_capturingr   Úconfiguration_granitemoer    ÚModuler#   rM   r†   rš   r¼   rÎ   r×   rI   rƒ   rÝ   rH   rð   rò   r  r  r.  rB   rZ  r\  Ú__all__r  r2   r1   ú<module>r„     sŠ  ðõ, %Ý ã ß Ð Ý å &Ý !ß .Ý )÷ó õ 0Ý 9ß Qß Kß FÝ &ß 7ß YÑ YÝ 5Ý 6ñ ˜YÓ'ôJ˜Ÿ	™	ó Jó (ðJô(>< §	¡	ô ><ôB9˜2Ÿ9™9ô 9ð, ô$#˜Ÿ	™	ó $#ó ð$#ôN?�B—I‘Iô ?ò"(ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ñ2 Ð)Ó*ô@)˜"Ÿ)™)ó @)ó +ð@)ôF Ð7ô  ðF ôU ó Uó ðUð4 ôJ
Ð/ó J
ó ðJ
ð^ #Ø
Ø*.ñ	O&Ø—‘  e§l¡lÑ 3Ñ3°dÑ:ðO&à�t‘ðO&ð —L‘L 4Ñ'ð	O&ð
 ‡\�\�CÑóO&ðd ôe
Ð5°ó e
ó ðe
òP T�r2   