Ë
    ÛÓ^j€  ã                   ó¨  — d dl mZ d dlmZmZ d dlZd dlmc mZ	 d dlmZ ddl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z* ddl+m,Z,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2  G d„ ded¬«      Z3 G d„ dejh                  «      Z5 ed«       G d„ dejh                  «      «       Z6 G d„ d ejh                  «      Z7e G d!„ d"ejh                  «      «       Z8 G d#„ d$ejh                  «      Z9d%„ Z: ed&«      dEd'„«       Z;d(ejx                  d)e=d*ejx                  fd+„Z>	 dFd,ejh                  d-ejx                  d.ejx                  d/ejx                  d0ejx                  dz  d1e?d2e?d3e'e)   fd4„Z@ ee;«       G d5„ d6ejh                  «      «       ZA G d7„ d8e«      ZBe* G d9„ d:e%«      «       ZC G d;„ d<ejh                  «      ZDe* G d=„ d>eC«      «       ZE	 	 	 dGd?ejx                  eFejx                     z  dz  d@e=dz  d0ejx                  dz  d*ejx                  e=z  fdA„ZGe* G dB„ dCeCe«      «       ZHg dD¢ZIy)Hé    )ÚCallable)ÚOptionalÚ	TypedDictN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstring)Úcan_return_tupleÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚGraniteMoeSharedConfigc                   ó€   — e Zd ZU dZej
                  ed<   ej
                  ed<   eed<   eed<   ej                  ed<   y)ÚGraniteFlashAttentionKwargsaT  
    Keyword arguments for advanced Flash Attention, causal-conv1d, and mamba_ssm kernel usage.
    Use cases include padding-free training and fewer `torch.compile` graph breaks.

    cu_seq_lens_q (`torch.LongTensor`):
        Gets cumulative sequence length for query state.
    cu_seq_lens_k (`torch.LongTensor`):
        Gets cumulative sequence length for key state.
    max_length_q (`int`):
        Maximum sequence length for query state.
    max_length_k (`int`):
        Maximum sequence length for key state.
    seq_idx (`torch.IntTensor):
        Index of each packed sequence.
    Úcu_seq_lens_qÚcu_seq_lens_kÚmax_length_qÚmax_length_kÚseq_idxN)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchÚ
LongTensorÚ__annotations__ÚintÚ	IntTensor© ó    ú‰/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/granitemoeshared/modeling_granitemoeshared.pyr#   r#   2   s7   … ñð  ×#Ñ#Ó#Ø×#Ñ#Ó#ØÓØÓØ�_‰_Ôr3   r#   F)Útotalc                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚGraniteMoeSharedMLPz~
    MLP layer for shared experts

    Args:
        config:
            Configuration object with model hyperparameters.
    Úconfigc                 ó`  •— t         ‰| �  «        |j                  | _        |j                  | _        t
        |j                     | _        t        j                  | j                  | j                  dz  d¬«      | _
        t        j                  | j                  | j                  d¬«      | _        y )Né   F©Úbias)ÚsuperÚ__init__Úhidden_sizeÚ
input_sizeÚshared_intermediate_sizer	   Ú
hidden_actÚ
activationr   ÚLinearÚinput_linearÚoutput_linear©Úselfr8   Ú	__class__s     €r4   r>   zGraniteMoeSharedMLP.__init__S   s   ø€ Ü‰ÑÔà ×,Ñ,ˆŒØ!×:Ñ:ˆÔÜ  ×!2Ñ!2Ñ3ˆŒÜŸI™I d§o¡o°t×7GÑ7GÈ!Ñ7KÐRWÔXˆÔÜŸY™Y t×'7Ñ'7¸¿¹ÈuÔUˆÕr3   Úhidden_statesÚreturnc                 ó¤   — | j                  |«      }|j                  dd¬«      }| j                  |d   «      |d   z  }| j                  |«      }|S )Nr:   éÿÿÿÿ©Údimr   r    )rE   ÚchunkrC   rF   )rH   rJ   Úchunked_hidden_statess      r4   ÚforwardzGraniteMoeSharedMLP.forward\   s^   € Ø×)Ñ)¨-Ó8ˆØ -× 3Ñ 3°A¸2Ð 3Ó >ÐØŸ™Ð(=¸aÑ(@ÓAÐDYÐZ[ÑD\Ñ\ˆØ×*Ñ*¨=Ó9ˆØÐr3   ©
r)   r*   r+   r,   r!   r>   r-   ÚTensorrR   Ú__classcell__©rI   s   @r4   r7   r7   J   s2   ø„ ñðVÐ5õ Vð U§\¡\ð °e·l±l÷ r3   r7   ÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚGraniteMoeSharedRMSNormÚepsrK   Nc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)zF
        GraniteMoeSharedRMSNorm is equivalent to T5LayerNorm
        N)r=   r>   r   Ú	Parameterr-   ÚonesÚweightÚvariance_epsilon)rH   r?   rZ   rI   s      €r4   r>   z GraniteMoeSharedRMSNorm.__init__f   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr3   rJ   c                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Nr:   rM   T)Úkeepdim)	ÚdtypeÚtor-   Úfloat32ÚpowÚmeanÚrsqrtr_   r^   )rH   rJ   Úinput_dtypeÚvariances       r4   rR   zGraniteMoeSharedRMSNorm.forwardn   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r3   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler^   Úshaper_   )rH   s    r4   Ú
extra_reprz"GraniteMoeSharedRMSNorm.extra_repru   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr3   )g�íµ ÷Æ°>)
r)   r*   r+   Úfloatr>   r-   rT   rR   rm   rU   rV   s   @r4   rY   rY   d   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJr3   rY   c                   ó”   ‡ — e Zd ZdZdefˆ fd„Zdej                  deej                  ej                  ej                  f   fd„Z	ˆ xZ
S )ÚGraniteMoeSharedTopKRoutera¡  Top-k gating that returns the routing decisions without grouping tokens by expert.

    Returns ``(top_k_index, top_k_weights, router_logits)``; the grouping/scattering used to live
    here (via ``expert_size.tolist()``, which broke fullgraph compile) and now happens inside the
    experts forward via ``use_experts_implementation`` so the default ``grouped_mm`` / ``batched_mm``
    paths can compile cleanly.
    r8   c                 óê   •— t         ‰| �  «        |j                  | _        |j                  | _        t        j                  t        j                  | j                  |j                  «      «      | _        y ©N)r=   r>   Únum_local_expertsÚnum_expertsÚnum_experts_per_tokÚtop_kr   r\   r-   Úemptyr?   r^   rG   s     €r4   r>   z#GraniteMoeSharedTopKRouter.__init__‚   sP   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ×/Ñ/ˆŒ
Ü—l‘l¤5§;¡;¨t×/?Ñ/?À×ASÑASÓ#TÓUˆ�r3   rJ   rK   c                 óô   — t        j                  || j                  «      j                  «       }|j	                  | j
                  d¬«      \  }}t        j                  |d¬«      j                  |«      }|||fS )NrM   rN   )	ÚFÚlinearr^   rn   Útopkrv   r-   ÚsoftmaxÚtype_as)rH   rJ   Úrouter_logitsÚtop_k_logitsÚtop_k_indexÚtop_k_weightss         r4   rR   z"GraniteMoeSharedTopKRouter.forwardˆ   se   € ÜŸ™ °·±Ó<×BÑBÓDˆØ$1×$6Ñ$6°t·z±zÀrÐ$6Ó$JÑ!ˆ�kÜŸ™ l¸Ô;×CÑCÀMÓRˆØ˜M¨=Ð8Ð8r3   )r)   r*   r+   r,   r!   r>   r-   rT   rk   rR   rU   rV   s   @r4   rp   rp   y   sI   ø„ ñðVÐ5õ Vð9 U§\¡\ð 9°e¸E¿L¹LÈ%Ï,É,ÐX]×XdÑXdÐ<dÑ6e÷ 9r3   rp   c                   ó�   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  dej                  dej                  fd„Zˆ xZ	S )	ÚGraniteMoeSharedExpertsz2Collection of expert weights stored as 3D tensors.r8   c                 óò  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        t        j                  t        j                  | j                  d| j                  z  | j
                  «      «      | _        t        j                  t        j                  | j                  | j
                  | j                  «      «      | _        t        |j                     | _        y )Nr:   )r=   r>   rs   rt   r?   Ú
hidden_dimÚintermediate_sizeÚintermediate_dimr   r\   r-   rw   Úgate_up_projÚ	down_projr	   rB   Úact_fnrG   s     €r4   r>   z GraniteMoeSharedExperts.__init__“   s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× 8Ñ 8ˆÔÜŸL™L¬¯©°T×5EÑ5EÀqÈ4×K`ÑK`ÑG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸ™¤e§k¡k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Ñ/ˆ�r3   rJ   r€   r�   rK   c                 óf  — t        j                  |«      }t        j                  «       5  t         j                  j                  j                  || j                  ¬«      }|j                  ddd«      }t        j                  |j                  d¬«      d«      j                  «       }d d d «       D ]ð  }|d   }|| j                  k(  rŒt        j                  |   «      \  }}	||	   }
t        j                  j                  |
| j                  |   «      j                  dd¬«      \  }}| j                  |«      |z  }t        j                  j                  || j                   |   «      }|||	|d f   z  }|j#                  d|	|j%                  |j&                  «      «       Œò |S # 1 sw Y   �ŒxY w)N)Únum_classesr:   r    r   )rM   éþÿÿÿrN   rM   )r-   Ú
zeros_likeÚno_gradr   Ú
functionalÚone_hotrt   ÚpermuteÚgreaterÚsumÚnonzeroÚwhererz   rˆ   rP   rŠ   r‰   Ú
index_add_rc   rb   )rH   rJ   r€   r�   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 r4   rR   zGraniteMoeSharedExperts.forwardœ   s™  € ô $×.Ñ.¨}Ó=ÐÜ�]‰]‹_ñ 	SÜŸ(™(×-Ñ-×5Ñ5°kÈt×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸ™ {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷	Sð
 %ò 
	nˆJØ# A™ˆJØ˜T×-Ñ-Ò-ØÜ#(§;¡;¨{¸:Ñ/FÓ#GÑ ˆI�yØ)¨)Ñ4ˆMÜ—}‘}×+Ñ+¨M¸4×;LÑ;LÈZÑ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆD�"Ø$(§K¡K°Ó$5¸Ñ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9NÐPT×P^ÑP^Ð_iÑPjÓ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÑ<eÑ$eÐ!Ø×*Ñ*¨1¨iÐ9N×9QÑ9QÐRe×RkÑRkÓ9lÕmð
	nð #Ð"÷#	Sñ 	Sús   ªA=F&Æ&F0rS   rV   s   @r4   rƒ   rƒ   �   sN   ø„ á<ð0Ð5õ 0ð#à—|‘|ð#ð —\‘\ð#ð —|‘|ð	#ð
 
�‰÷#r3   rƒ   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚGraniteMoeSharedMoEzISparsely-gated mixture-of-experts block: router decides, experts compute.r8   c                 ó„   •— t         ‰| �  «        |j                  | _        t	        |«      | _        t        |«      | _        y rr   )r=   r>   r?   r@   rp   Úrouterrƒ   ÚexpertsrG   s     €r4   r>   zGraniteMoeSharedMoE.__init__º   s3   ø€ Ü‰ÑÔØ ×,Ñ,ˆŒÜ0°Ó8ˆŒÜ.¨vÓ6ˆ�r3   Úlayer_inputrK   c                 óØ   — |j                  «       \  }}}|j                  d|«      }| j                  |«      \  }}}| j                  |||«      }	|	j	                  ||| j
                  «      S )NrM   )ÚsizeÚreshaper¥   r¦   Úviewr@   )
rH   r§   ÚbszÚlengthÚemb_sizerJ   r€   r�   Ú_Úlayer_outputs
             r4   rR   zGraniteMoeSharedMoE.forwardÀ   si   € Ø +× 0Ñ 0Ó 2ÑˆˆV�XØ#×+Ñ+¨B°Ó9ˆØ(,¯©°MÓ(BÑ%ˆ�] AØ—|‘| M°;ÀÓNˆØ× Ñ   f¨d¯o©oÓ>Ð>r3   rS   rV   s   @r4   r£   r£   ·   s.   ø„ ÙSð7Ð5õ 7ð? 5§<¡<ð ?°E·L±L÷ ?r3   r£   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..NrM   r:   rN   )rl   r-   Úcat)ÚxÚx1Úx2s      r4   Úrotate_halfr¶   È   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r3   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer¶   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r4   Úapply_rotary_pos_embrÁ   Ï   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr3   rJ   Ún_reprK   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r    N)rl   Úexpandrª   )rJ   rÂ   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r4   Ú	repeat_kvrÉ   é   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr3   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )Nr:   r   rM   )rO   rb   )ÚpÚtrainingr    )rÉ   Únum_key_value_groupsr-   ÚmatmulÚ	transposer   r�   r|   rd   rc   rb   rÐ   rÔ   Ú
contiguous)rÊ   rË   rÌ   rÍ   rÎ   rÏ   rÐ   rÑ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r4   Úeager_attention_forwardrÝ   õ   sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r3   c                   óø   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
dz  d
ee   de	ej                  ej                  f   fd„Zˆ xZS )ÚGraniteMoeSharedAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr8   Ú	layer_idxc                 ó^  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        |j                  | _        |j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                   ¬«      | _        y )NrÈ   Tr;   )r=   r>   r8   rà   Úgetattrr?   Únum_attention_headsrÈ   rÆ   rÕ   Úattention_multiplierrÏ   Úattention_dropoutÚ	is_causalr   rD   Úattention_biasÚq_projÚk_projÚv_projÚo_proj©rH   r8   rà   rI   s      €r4   r>   z"GraniteMoeSharedAttention.__init__  sJ  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø×2Ñ2ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆ�r3   NrJ   Úposition_embeddingsrÎ   Úpast_key_valuesrÑ   rK   c                 ó
  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  sdn| j                   | j"                  dœ|¤Ž\  }} |j$                  g |¢d‘­Ž j'                  «       }| j)                  |«      }||fS )NrM   r    r:   ç        )rÐ   rÏ   )rl   rÈ   rè   r«   r×   ré   rê   rÁ   Úupdaterà   r   Úget_interfacer8   Ú_attn_implementationrÝ   rÔ   rå   rÏ   rª   rØ   rë   )rH   rJ   rí   rÎ   rî   rÑ   Úinput_shapeÚhidden_shapeÚquery_statesrÙ   rÚ   r¼   r½   Úattention_interfacerÜ   rÛ   s                   r4   rR   z!GraniteMoeSharedAttention.forward)  s‘  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r3   ©NNN)r)   r*   r+   r,   r!   r0   r>   r-   rT   rk   r
   r   r   rR   rU   rV   s   @r4   rß   rß     s¦   ø„ áGð
Ð5ð 
À#õ 
ð4 IMØ.2Ø(,ñ&)à—|‘|ð&)ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð&)ð Ÿ™ tÑ+ð	&)ð
  ™ð&)ð Ð+Ñ,ð&)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷&)r3   rß   c                   óP  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
e
dz  deej                  ej                  f   dz  dee   deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚGraniteMoeSharedDecoderLayerr8   rà   c                 ó�  •— t         ‰| �  «        |j                  | _        t        ||¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _
        |j                  | _        |j                  dk(  rd | _        y t        |«      | _        y )N)r8   rà   ©rZ   r   )r=   r>   r?   rß   Ú	self_attnrY   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr£   Úblock_sparse_moeÚresidual_multiplierrA   r7   Ú
shared_mlprì   s      €r4   r>   z%GraniteMoeSharedDecoderLayer.__init__S  s�   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ2¸&ÈIÔVˆŒÜ6°v×7IÑ7IÈv×ObÑObÔcˆÔÜ(?À×@RÑ@RÐX^×XkÑXkÔ(lˆÔ%Ü 3°FÓ ;ˆÔØ#)×#=Ñ#=ˆÔ Ø"(×"AÑ"AÀQÒ"F˜$ˆ�ÔL_Ð`fÓLgˆ�r3   NrJ   rÎ   Úposition_idsrî   Úoutput_attentionsÚ	use_cacherí   rÑ   rK   c                 ó<  — |}	| j                  |«      } | j                  d|||||||dœ|¤Ž\  }}
|	|| j                  z  z   }|}	| j                  |«      }| j	                  |«      }| j
                  €|}n|| j                  |«      z   }|	|| j                  z  z   }|S )N)rJ   rÎ   r  rî   r  r  rí   r2   )rÿ   rý   r  r   r  r  )rH   rJ   rÎ   r  rî   r  r  rí   rÑ   Úresidualr¯   Úmoe_hidden_statess               r4   rR   z$GraniteMoeSharedDecoderLayer.forward]  sÎ   € ð !ˆØ×,Ñ,¨]Ó;ˆð *˜4Ÿ>™>ð 	
Ø'Ø)Ø%Ø+Ø/ØØ 3ñ	
ð ñ	
Ñˆ�qð ! =°4×3KÑ3KÑ#KÑKˆà ˆØ×5Ñ5°mÓDˆØ ×1Ñ1°-Ó@Ðà�?‰?Ð"Ø-‰Mà-°·±ÀÓ0NÑNˆMØ  =°4×3KÑ3KÑ#KÑKˆØÐr3   )NNNFFN)r)   r*   r+   r!   r0   r>   r-   rT   r.   r
   Úboolrk   r   r#   ÚFloatTensorrR   rU   rV   s   @r4   rú   rú   R  sø   ø„ ðhÐ5ð hÀ#õ hð /3Ø04Ø(,Ø).Ø!&ØHLñ%à—|‘|ð%ð Ÿ™ tÑ+ð%ð ×&Ñ&¨Ñ-ð	%ð
  ™ð%ð   $™;ð%ð ˜$‘;ð%ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð%ð Ð4Ñ5ð%ð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷%r3   rú   c                   ó„   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZ ej$                  «       ˆ fd„«       Zˆ xZS )ÚGraniteMoeSharedPreTrainedModelr8   ÚmodelTrú   rî   )rJ   Ú
attentionsc                 ó¬  •— t         ‰| �  |«       t        |t        «      rmt	        j
                  |j                  d| j                  j                  ¬«       t	        j
                  |j                  d| j                  j                  ¬«       y t        |t        «      r7t	        j
                  |j                  d| j                  j                  ¬«       y y )Nrð   )rf   Ústd)r=   Ú_init_weightsÚ
isinstancerƒ   ÚinitÚnormal_rˆ   r8   Úinitializer_ranger‰   rp   r^   )rH   rÊ   rI   s     €r4   r  z-GraniteMoeSharedPreTrainedModel._init_weights–  s‰   ø€ ä‰Ñ˜fÔ%Ü�fÔ5Ô6Ü�L‰L˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�L‰L˜×)Ñ)°¸¿¹×9VÑ9VÖWÜ˜Ô :Ô;Ü�L‰L˜Ÿ™¨S°d·k±k×6SÑ6SÖTð <r3   )r)   r*   r+   r!   r/   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrú   rß   Ú_can_record_outputsr-   r�   r  rU   rV   s   @r4   r  r  …  sp   ø… à"Ó"ØÐØ&*Ð#Ø7Ð8ÐØ#4Ð"5ÐØÐØ€NØÐØ!ÐØ"&Ðà5Ø/ñÐð
 €U‡]�]ƒ_óUó ôUr3   r  c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚGraniteMoeSharedRotaryEmbeddingÚinv_freqNr8   c                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultr#  F)Ú
persistentÚoriginal_inv_freq)r=   r>   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr8   Úrope_parametersr%  Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)rH   r8   ÚdeviceÚrope_init_fnr#  rI   s        €r4   r>   z(GraniteMoeSharedRotaryEmbedding.__init__£  s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr3   r1  ztorch.deviceÚseq_lenrK   ztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarÈ   Ng      ð?r   r:   ©rb   )r1  rb   )	r,  râ   r?   rã   r-   ÚarangeÚint64rc   rn   )r8   r1  r3  ÚbaserO   Úattention_factorr#  s          r4   r-  z?GraniteMoeSharedRotaryEmbedding.compute_default_rope_parameters³  s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r3   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   rM   r    ÚmpsÚcpuF)Údevice_typeÚenabledr:   rN   r6  )r#  rn   rÄ   rl   rc   r1  r  ÚtypeÚstrr   r×   r-   r²   r¼   r.  r½   rb   )
rH   r³   r  Úinv_freq_expandedÚposition_ids_expandedr>  ÚfreqsÚembr¼   r½   s
             r4   rR   z'GraniteMoeSharedRotaryEmbedding.forwardÑ  sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$rr   rø   )r)   r*   r+   r-   rT   r/   r!   r>   Ústaticmethodr   r0   rk   rn   r-  r�   r   rR   rU   rV   s   @r4   r"  r"     s™   ø… Ø�l‰lÓñVÐ5õ Vð  à04Ø+/Ø"ñ*Ø&¨Ñ-ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r3   r"  c                   óò   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )ÚGraniteMoeSharedModelr8   c           	      ó.  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t#        |¬«      | _        d| _        |j(                  | _        | j+                  «        y c c}w )Nrü   ©r8   F)r=   r>   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr?   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrú   ÚlayersrY   rþ   Únormr"  Ú
rotary_embÚgradient_checkpointingÚembedding_multiplierÚ	post_initrì   s      €r4   r>   zGraniteMoeSharedModel.__init__ã  sÕ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜNSÐTZ×TlÑTlÓNmÖnÀÔ)¨&°)Õ<Ònó
ˆŒô ,¨F×,>Ñ,>ÀF×DWÑDWÔXˆŒ	Ü9ÀÔHˆŒØ&+ˆÔ#Ø$*×$?Ñ$?ˆÔ!ð 	�‰Õùò os   ÂDNÚ	input_idsrÎ   r  rî   Úinputs_embedsr  rÑ   rK   c           
      óX  — |d u |d uz  rt        d«      ‚|r|€t        | j                  ¬«      }|€| j                  |«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        | j                  ||||¬«      }	|| j                  z  }|}
| j                  |
|«      }| j                  d | j                  j                   D ]  } ||
f||	|||dœ|¤Ž}
Œ | j                  |
«      }
t!        |
|¬«      S )	Nz:You must specify exactly one of input_ids or inputs_embedsrJ  r   r    )r1  )r8   rZ  rÎ   rî   r  )rí   rÎ   r  rî   r  )Úlast_hidden_staterî   )Ú
ValueErrorr   r8   rO  Úget_seq_lengthr-   r7  rl   r1  r¹   r   rW  rU  rS  rR  rT  r   )rH   rY  rÎ   r  rî   rZ  r  rÑ   Úpast_seen_tokensÚcausal_maskrJ   rí   Údecoder_layers                r4   rR   zGraniteMoeSharedModel.forwardô  s\  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZá˜Ð0Ü*°$·+±+Ô>ˆOàÐ Ø ×-Ñ-¨iÓ8ˆMàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &¨×(AÑ(AÑAˆØ%ˆð #Ÿo™o¨m¸\ÓJÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 		ˆMÙ)Øðà$7Ø*Ø)Ø /Ø#ñð ñ‰Mð		ð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r3   )NNNNNN)r)   r*   r+   r!   r>   r   r   r   r-   r.   rT   r
   r  r
  r   r   r   rR   rU   rV   s   @r4   rH  rH  á  sÎ   ø„ ðÐ5õ ð"  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ5
à×#Ñ# dÑ*ð5
ð Ÿ™ tÑ+ð5
ð ×&Ñ&¨Ñ-ð	5
ð
  ™ð5
ð ×(Ñ(¨4Ñ/ð5
ð ˜$‘;ð5
ð Ð+Ñ,ð5
ð 
 ò5
ó ó ó  ô5
r3   rH  Úgate_logitsrt   c                 ó¾  — | �t        | t        «      syt        | t        «      rC| d   j                  }t        j                  | D �cg c]  }|j                  |«      ‘Œ c}d¬«      }t        j                  j                  j                  d¬«      }t        j                  ||d¬«      \  }}	t        j                  j                  j                  |	|«      }
|€>t        j                  |
j                  «       d¬«      }t        j                  |d¬«      }�n|j                  \  }}|j                  d   ||z  z  }|ddd…dd…ddf   j                  |||||f«      j                  d||«      j                  «      }t        j                   |
j                  «       |z  d¬«      t        j                   |d¬«      z  }|ddd…dd…df   j                  ||||f«      j                  d|«      j                  |«      }t        j                   ||z  d¬«      t        j                   |d¬«      z  }t        j                   ||j#                  d«      z  «      }||z  S c c}w )aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   rN   rM   )r  rk   r1  r-   r²   rc   r   r�   r|   r{   r‘   rf   rn   rl   rÄ   rª   r”   r¹   )rb  rt   rv   rÎ   Úcompute_deviceÚ
layer_gateÚconcatenated_gate_logitsÚrouting_weightsr¯   Úselected_expertsr™   Útokens_per_expertÚrouter_prob_per_expertÚ
batch_sizeÚsequence_lengthrR  Úexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_losss                      r4   Úload_balancing_loss_funcrp  /  s9  € ð: Ð¤*¨[¼%Ô"@Øä�+œuÔ%Ø$ Q™×.Ñ.ˆÜ#(§9¡9Ð^iÖ-jÐPZ¨j¯m©m¸NÕ.KÒ-jÐpqÔ#rÐ ä—h‘h×)Ñ)×1Ñ1Ð2JÐPRÐ1ÓS€OäŸ*™* _°eÀÔDÑ€AÐä—(‘(×%Ñ%×-Ñ-Ð.>ÀÓL€KàÐä!ŸJ™J {×'8Ñ'8Ó':ÀÔBÐô "'§¡¨OÀÔ!CÒà&4×&:Ñ&:Ñ#ˆ
�OØ4×:Ñ:¸1Ñ=À*ÈÑB^Ñ_Ðð ˜4¢¢A t¨TÐ1Ñ2ß‰VÐ&¨
°OÀUÈKÐXÓYß‰W�R˜ Ó,ß‰R�Óð	 	ô "ŸI™I k×&7Ñ&7Ó&9Ð<QÑ&QÐWXÔYÔ\a×\eÑ\eØ! qô]
ñ 
Ðð ˜4¢¢A tÐ+Ñ,ß‰VÐ&¨
°OÀ[ÐQÓRß‰W�R˜Ó%ß‰R�Óð	 	)ô "'§¡¨?Ð=]Ñ+]ÐcdÔ!eÔhm×hqÑhqØ,°!ôi
ñ "
Ðô —9‘9Ð.Ð1G×1QÑ1QÐRSÓ1TÑTÓU€LØ˜+Ñ%Ð%ùò[ .ks   ÁIc                   óD  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zee		 	 	 	 	 	 	 	 dd
e
j                  d	z  de
j                  d	z  de
j                  d	z  ded	z  de
j                  d	z  de
j                  d	z  ded	z  dee
j                  z  deez  fd„«       «       Zˆ xZS )ÚGraniteMoeSharedForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrJ   Úlogitsr8   c                 óp  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        |j                  | _	        |j                  | _        |j                  | _        |j                  | _        | j                  «        y )NFr;   )r=   r>   rH  r  rM  r   rD   r?   rs  Úrouter_aux_loss_coefrs   rt   ru   Úlogits_scalingrX  rG   s     €r4   r>   z$GraniteMoeSharedForCausalLM.__init__‡  s�   ø€ Ü‰Ñ˜Ô Ü*¨6Ó2ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×3Ñ3ˆÔØ#)×#=Ñ#=ˆÔ Ø$×3Ñ3ˆÔð 	�‰Õr3   NrY  rÎ   r  rî   rZ  ÚlabelsÚoutput_router_logitsÚlogits_to_keeprK   c	           	      ó®  — |�|n| j                   j                  } | j                  d|||||dœ|	¤Ž}
|
j                  }t	        |t
        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }|| j                   j                  z  }d}|�* | j                  ||fd| j                   j                  i|	¤Ž}d}|rYt        |
j                  | j                  | j                  |«      }|�+|| j                  |j!                  |j"                  «      z  z  }t%        ||||
j&                  |
j(                  |
j*                  |
j                  ¬«      S )ax  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, GraniteMoeSharedForCausalLM

        >>> model = GraniteMoeSharedForCausalLM.from_pretrained("ibm/PowerMoE-3b")
        >>> tokenizer = AutoTokenizer.from_pretrained("ibm/PowerMoE-3b")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)rY  rÎ   r  rî   rZ  rM  )ÚlossÚaux_lossru  rî   rJ   r  r~   r2   )r8   rz  r  r\  r  r0   Úslicers  rx  Úloss_functionrM  rp  r~   rt   ru   rw  rc   r1  r   rî   rJ   r  )rH   rY  rÎ   r  rî   rZ  ry  rz  r{  rÑ   ÚoutputsrJ   Úslice_indicesru  r}  r~  s                   r4   rR   z#GraniteMoeSharedForCausalLM.forward”  s|  € ðJ %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð �$—*‘*ð 
ØØ)Ø%Ø+Ø'ñ
ð ñ
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ˜$Ÿ+™+×4Ñ4Ñ4ˆàˆØÐà%�4×%Ñ%ØØñð  Ÿ;™;×1Ñ1ðð ñ	ˆDð ˆÙÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ð!Ø˜×1Ñ1°H·K±KÀÇÁÓ4LÑLÑL�Ü(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r3   )NNNNNNNr   )r)   r*   r+   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr!   r>   r   r   r-   r.   rT   r
   r  r
  r0   rk   r   rR   rU   rV   s   @r4   rr  rr  �  s   ø„ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HðÐ5õ ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø,0Ø-.ñQ
à×#Ñ# dÑ*ðQ
ð Ÿ™ tÑ+ðQ
ð ×&Ñ&¨Ñ-ð	Q
ð
  ™ðQ
ð ×(Ñ(¨4Ñ/ðQ
ð × Ñ  4Ñ'ðQ
ð # T™kðQ
ð ˜eŸl™lÑ*ðQ
ð 
Ð*Ñ	*òQ
ó ó ôQ
r3   rr  )rr  rH  r  )r    )rð   )Nr:   N)JÚcollections.abcr   Útypingr   r   r-   Útorch.nn.functionalr   r�   ry   Ú r   r  Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úintegrationsr   r   r   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   r   r   Úutils.output_capturingr   Úconfiguration_granitemoesharedr!   r#   ÚModuler7   rY   rp   rƒ   r£   r¶   rÁ   rT   r0   rÉ   rn   rÝ   rß   rú   r  r"  rH  rk   rp  rr  Ú__all__r2   r3   r4   ú<module>rš     s¨  ðõ* %ß &ã ß Ð Ý å &Ý !ß .Ý )÷ó õ 0Ý 9ß Qß Kß FÝ &ß 7ß YÑ YÝ 5Ý Bô )°5õ ô0˜"Ÿ)™)ô ñ4 ˜YÓ'ôJ˜bŸi™ió Jó (ðJô(9 §¡ô 9ð, ô$#˜bŸi™ió $#ó ð$#ôN?˜"Ÿ)™)ô ?ò"(ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ñ2 Ð)Ó*ô@) §	¡	ó @)ó +ð@)ôF0Ð#=ô 0ðf ôU oó Uó ðUô4>< b§i¡iô ><ðB ôJ
Ð;ó J
ó ðJ
ð^ #Ø
Ø*.ñ	O&Ø—‘  e§l¡lÑ 3Ñ3°dÑ:ðO&à�t‘ðO&ð —L‘L 4Ñ'ð	O&ð
 ‡\�\�CÑóO&ðd ôe
Ð"AÀ?ó e
ó ðe
òP f�r3   