Ë
    ÜÓ^júž  ã                   ó  — d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZmZ ddlmZ ddlmZmZm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0 ddl1m2Z2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8  ed«       G d„ dejr                  «      «       Z: G d„ de«      Z; G d„ dejr                  «      Z< G d„ dejr                  «      Z=d „ Z> ed!«      dJd"„«       Z?d#ej€                  d$eAd%ej€                  fd&„ZB	 dKd'ejr                  d(ej€                  d)ej€                  d*ej€                  d+ej€                  dz  d,eCd-eCd.e,e.   fd/„ZD ee?«       G d0„ d1ejr                  «      «       ZE G d2„ d3ejr                  «      ZFe G d4„ d5ejr                  «      «       ZG G d6„ d7ejr                  «      ZH G d8„ d9e!«      ZIe/ G d:„ d;e*«      «       ZJe/ G d<„ d=eJ«      «       ZK	 	 	 dLd>ej€                  eLej€                     z  dz  d?eAdz  d+ej€                  dz  d%ej€                  eAz  fd@„ZMe/ G dA„ dBeJe«      «       ZN G dC„ dDeeJ«      ZO G dE„ dFe eJ«      ZP G dG„ dHeeJ«      ZQg dI¢ZRy)Mé    )ÚCallable)ÚOptionalN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGenericForQuestionAnsweringÚ GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚMiniMaxConfigÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚMiniMaxRMSNormÚepsÚreturnNc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z=
        MiniMaxRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer*   Ú	__class__s      €úw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/minimax/modeling_minimax.pyr.   zMiniMaxRMSNorm.__init__;   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕó    Úhidden_statesc                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor0   Úfloat32ÚpowÚmeanÚrsqrtr3   r2   )r4   r9   Úinput_dtypeÚvariances       r7   ÚforwardzMiniMaxRMSNorm.forwardC   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r8   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler2   Úshaper3   )r4   s    r7   Ú
extra_reprzMiniMaxRMSNorm.extra_reprJ   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr8   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Úfloatr.   r0   ÚTensorrF   rJ   Ú__classcell__©r6   s   @r7   r)   r)   9   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJr8   r)   c                   ór   ‡ — e Zd Zˆ fd„Zd„ Zdefd„Zˆ fd„Zdefd„Zde	j                  fd	„Zd
efd„Zˆ xZS )ÚMiniMaxCachec                 ó0   •— t         ‰| �  «        g | _        y ©N)r-   r.   Úlinear_cache©r4   r6   s    €r7   r.   zMiniMaxCache.__init__O   s   ø€ Ü‰ÑÔØ02ˆÕr8   c                 ó¦   — t        t        | j                  «      |dz   «      D ]  }| j                  j                  g «       Œ || j                  |<   y )Nr%   )ÚrangeÚlenrV   Úappend)r4   Ú	layer_idxrV   Ú_s       r7   Úset_linear_cachezMiniMaxCache.set_linear_cacheS   sK   € ä”s˜4×,Ñ,Ó-¨y¸1©}Ó=ò 	)ˆAØ×Ñ×$Ñ$ RÕ(ð	)à'3ˆ×Ñ˜)Ò$r8   r\   c                 ó>   — |t        | «      k  r| j                  |   S y rU   )rZ   rV   )r4   r\   s     r7   Úget_linear_cachezMiniMaxCache.get_linear_cacheY   s"   € Ø”s˜4“yÒ Ø×$Ñ$ YÑ/Ð/Ør8   c                 óZ   •— t        t        ‰| �	  «       t        | j                  «      «      S rU   )Úmaxr-   Ú__len__rZ   rV   rW   s    €r7   rc   zMiniMaxCache.__len__^   s"   ø€ Ü”5‘7‘?Ó$¤c¨$×*;Ñ*;Ó&<Ó=Ð=r8   Úrepeatsc                 óò   — t        t        | «      «      D ]`  }| j                  |   g k7  r.| j                  |   j                  |d¬«      | j                  |<   ŒC| j                  |   j                  |«       Œb y )Nr   ©Údim)rY   rZ   rV   Úrepeat_interleaveÚlayersÚbatch_repeat_interleave)r4   rd   r\   s      r7   rj   z$MiniMaxCache.batch_repeat_interleavea   ss   € Üœs 4›yÓ)ò 	HˆIØ× Ñ  Ñ+¨rÒ1Ø/3×/@Ñ/@ÀÑ/K×/]Ñ/]Ð^eÐklÐ/]Ó/m�×!Ñ! )Ò,à—‘˜IÑ&×>Ñ>¸wÕGñ		Hr8   Úindicesc                 óÚ   — t        t        | «      «      D ]T  }| j                  |   g k7  r"| j                  |   |df   | j                  |<   Œ7| j                  |   j	                  |«       ŒV y )N.)rY   rZ   rV   ri   Úbatch_select_indices)r4   rk   r\   s      r7   rm   z!MiniMaxCache.batch_select_indicesh   sk   € Üœs 4›yÓ)ò 	EˆIØ× Ñ  Ñ+¨rÒ1Ø/3×/@Ñ/@ÀÑ/KÈGÐUXÈLÑ/Y�×!Ñ! )Ò,à—‘˜IÑ&×;Ñ;¸GÕDñ		Er8   Ú
max_lengthc                 ó   — t        d«      ‚)Nz*MiniMaxCache doesnot support `crop` method)ÚRuntimeError)r4   rn   s     r7   ÚcropzMiniMaxCache.cropo   s   € ÜÐGÓHÐHr8   )rK   rL   rM   r.   r^   Úintr`   rc   rj   r0   rO   rm   rq   rP   rQ   s   @r7   rS   rS   N   sL   ø„ ô3ò4ð¨#ó ô
>ðH¨só HðE¨E¯L©Ló EðI˜s÷ Ir8   rS   c                   ó  ‡ — e Zd Zdedefˆ fd„Zd„ Zd„ Z	 ddej                  de
ej                  ej                  f   d	ej                  dz  d
edz  dee   de
ej                  ej                  dz  e
ej                     dz  f   fd„Zˆ xZS )ÚMiniMaxLightningAttentionÚconfigr\   c                 óê  •— t         ‰| �  «        || _        t        |dd «      xs |j                  |j
                  z  | _        |j
                  | _        |j                  | _        |j                  | _        t        |j                     | _        t        | j                  | j
                  z  «      | _        t        j                  |j                  | j
                  | j                  z  dz  d¬«      | _        t        j                  | j
                  | j                  z  |j                  d¬«      | _        t        j                  |j                  | j
                  | j                  z  d¬«      | _        | j'                  «       }| j)                  |«      \  }}}| j+                  d|«       | j+                  d|«       | j+                  d|«       | j+                  d|«       |j,                  |   | _        y )	NÚhead_dimr   F©ÚbiasÚ
slope_rateÚquery_decayÚ	key_decayÚdiagonal_decay)r-   r.   r\   Úgetattrr5   Únum_attention_headsrw   Únum_hidden_layersÚ
block_sizer   Ú
hidden_actÚact_fnr)   Únormr   ÚLinearÚqkv_projÚout_projÚoutput_gateÚget_slope_rateÚdecay_factorsÚregister_bufferÚlayer_typesÚ
layer_type)r4   ru   r\   rz   r{   r|   r}   r6   s          €r7   r.   z"MiniMaxLightningAttention.__init__t   s•  ø€ Ü‰ÑÔØ"ˆŒÜ ¨
°DÓ9Òm¸V×=OÑ=OÐSY×SmÑSmÑ=mˆŒØ#)×#=Ñ#=ˆÔ Ø!'×!9Ñ!9ˆÔØ ×+Ñ+ˆŒä˜V×.Ñ.Ñ/ˆŒÜ" 4§=¡=°4×3KÑ3KÑ#KÓLˆŒ	ÜŸ	™	 &×"4Ñ"4°d×6NÑ6NÐQU×Q^ÑQ^Ñ6^ÐabÑ6bÐinÔoˆŒÜŸ	™	 $×":Ñ":¸T¿]¹]Ñ"JÈF×L^ÑL^ÐejÔkˆŒÜŸ9™9 V×%7Ñ%7¸×9QÑ9QÐTX×TaÑTaÑ9aÐhmÔnˆÔà×(Ñ(Ó*ˆ
Ø15×1CÑ1CÀJÓ1OÑ.ˆ�Y à×Ñ˜\¨:Ô6Ø×Ñ˜]¨KÔ8Ø×Ñ˜[¨)Ô4Ø×ÑÐ-¨~Ô>à ×,Ñ,¨YÑ7ˆ�r8   c                 óæ   — ddd| j                   z  z  z  }t        j                  | j                   «      dz   }d| j                  | j                  dz
  dz   z  z
  dz   }||z  }||z  }|d d …d d f   }|S )Nr%   r;   é   gñhãˆµøä>)r   r0   Úaranger\   r€   )r4   ÚbaseÚexponentÚfactorÚrates        r7   r‰   z(MiniMaxLightningAttention.get_slope_rateŒ   s„   € Ø�A˜!˜d×6Ñ6Ñ6Ñ7Ñ8ˆÜ—<‘< × 8Ñ 8Ó9¸AÑ=ˆØ�T—^‘^ t×'=Ñ'=ÀÑ'AÀDÑ'HÑIÑIÈDÑPˆà�X‰~ˆØ�f‰}ˆØ’A�t˜T�MÑ"ˆàˆr8   c                 ó¦  — t        j                  | j                  «      dz   }t        j                  | |d d …d f   z  «      }t        j                  | | j                  |d d …d f   z
  z  «      }|d d …d f   |d d d …f   z
  }|d d d d …d d …f   }||z  }t        j                  |dk\  | t        d«      «      }t        j                  |«      }|||fS )Nr%   r   z-inf)r0   r�   r�   ÚexpÚwhererN   )r4   rz   Úblock_size_ranger{   r|   r}   s         r7   rŠ   z'MiniMaxLightningAttention.decay_factors—   s×   € Ü Ÿ<™<¨¯©Ó8¸1Ñ<Ðä—i‘i  Ð.>ºqÀ$¸wÑ.GÑ GÓHˆÜ—I‘I˜z˜k¨T¯_©_Ð?OÒPQÐSWÐPWÑ?XÑ-XÑYÓZˆ	à)ª!¨T¨'Ñ2Ð5EÀdÊAÀgÑ5NÑNˆØ'¨¨d²A²qÐ(8Ñ9ˆØ# nÑ4ˆÜŸ™ ^°qÑ%8¸>¸/Ì5ÐQWË=ÓYˆÜŸ™ >Ó2ˆà˜I ~Ð5Ð5r8   Nr9   Úposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsr+   c                 óú  — |j                   \  }}}|| j                  z   dz
  | j                  z  }	| j                  | j                  |«      «      }
|
j	                  ||| j
                  d| j                  z  «      }
t        j                  |
| j                  d¬«      \  }}}|j                  dd«      }|j                  dd«      }|j                  dd«      }d }|�|j                  | j                  «      }|�€t        j                  || j
                  | j                  | j                  «      j                  |«      }|�Q|j                  t        j                  ¬«      }|j                  |j!                  d«      j!                  d«       d«      }g }t#        |	«      D �]b  }|| j                  z  }t%        || j                  z   |«      }||z
  }|d d …d d …||…f   }|d d …d d …||…f   }|d d …d d …||…f   }| j&                  d d …d |…f   }| j(                  d d …| d …f   }| j*                  d d …d d …d |…d |…f   }t        j,                  | j.                   |z  «      }t        j0                  ||j                  dd«      «      }t        j0                  ||z  |«      }t        j0                  ||z  |«      }||z   }|j3                  |«       t        j0                  ||z  j                  dd«      |«      }||z  |z   }�Œe nºt        j,                  | j.                   «      } g }t#        |«      D ]Š  }|d d …d d …||dz   …f   }|d d …d d …||dz   …f   }|d d …d d …||dz   …f   }t        j0                  |j                  dd«      |«      }!| |z  |!z   }t        j0                  ||«      }|j3                  |«       ŒŒ t        j4                  |d¬«      }|j                  dd«      }|j	                  ||| j
                  | j                  z  «      }| j7                  |«      }t9        j:                  | j=                  |«      «      |z  }| j?                  |«      }|�|jA                  | j                  |«       ||fS )	Nr%   r   rf   r;   ©r>   r<   r   éþÿÿÿ)!rI   r�   rƒ   r†   Úreshaper   rw   r0   ÚsplitÚ	transposer`   r\   Úzerosr?   ÚboolÚmasked_fillÚ	unsqueezerY   Úminr{   r|   r}   r–   rz   Úmatmulr[   Úcatr„   ÚFÚsigmoidrˆ   r‡   r^   )"r4   r9   r™   rš   r›   rœ   Ú
batch_sizeÚseq_lenr5   Ú
num_blocksÚ
qkv_statesÚquery_statesÚ
key_statesÚvalue_statesÚattn_weights_interÚattn_outputÚiÚ	start_idxÚend_idxÚcurrent_block_sizeÚcurrent_query_statesÚcurrent_key_statesÚcurrent_value_statesÚcurrent_query_decayÚcurrent_key_decayÚcurrent_diagonal_decayÚblock_decayÚattn_weights_intraÚattn_output_intraÚattn_output_interÚcurrent_attn_outputÚnext_attn_weights_interÚratioÚcurrent_attn_weights_inters"                                     r7   rF   z!MiniMaxLightningAttention.forward¥   s®  € ð ,9×+>Ñ+>Ñ(ˆ
�G˜[Ø §¡Ñ/°!Ñ3¸¿¹ÑGˆ
à—[‘[ §¡¨}Ó!=Ó>ˆ
Ø×'Ñ'¨
°G¸T×=UÑ=UÐWXÐ[_×[hÑ[hÑWhÓiˆ
ä16·±¸ZÈÏÉÐ\]Ô1^Ñ.ˆ�j ,à#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆð "ÐØÐ&Ø!0×!AÑ!AÀ$Ç.Á.Ó!QÐàÑ%Ü!&§¡¨Z¸×9QÑ9QÐSW×S`ÑS`Ðbf×boÑboÓ!p×!sÑ!sØó"Ðð
 Ð)Ø!/×!2Ñ!2¼¿¹Ð!2Ó!D�Ø+×7Ñ7¸×9QÑ9QÐRSÓ9T×9^Ñ9^Ð_aÓ9bÐ8bÐdeÓf�àˆKÜ˜:Ó&ó `�Ø §¡Ñ/�	Ü˜i¨$¯/©/Ñ9¸7ÓC�Ø%,¨yÑ%8Ð"à'3²A²q¸)ÀGÐ:KÐ4KÑ'LÐ$Ø%/²²1°iÀÐ6GÐ0GÑ%HÐ"Ø'3²A²q¸)ÀGÐ:KÐ4KÑ'LÐ$à&*×&6Ñ&6²qÐ:MÐ;MÐ:MÐ7MÑ&NÐ#Ø$(§N¡N²1Ð7IÐ6IÑ6JÐ3JÑ$KÐ!Ø)-×)<Ñ)<ºQÂÐCVÐDVÐCVÐXkÐYkÐXkÐ=kÑ)lÐ&Ü#Ÿi™i¨¯©Ð(8Ð;MÑ(MÓN�ô &+§\¡\Ð2FÐHZ×HdÑHdÐegÐikÓHlÓ%mÐ"Ü$)§L¡LÐ1CÐF\Ñ1\Ð^rÓ$sÐ!ô %*§L¡LÐ1EÐH[Ñ1[Ð]oÓ$pÐ!ð '8Ð:KÑ&KÐ#Ø×"Ñ"Ð#6Ô7ô +0¯,©,Ø'Ð*;Ñ;×FÑFÀrÈ2ÓNÐPdó+Ð'ð &8¸+Ñ%EÐH_Ñ%_Ò"ñ;`ô@ —I‘I˜tŸ™Ð.Ó/ˆEØˆKÜ˜7“^ò 	8�Ø'3²A²q¸!¸aÀ!¹e¸)°OÑ'DÐ$Ø%/²²1°a¸!¸a¹%°i°Ñ%@Ð"Ø'3²A²q¸!¸aÀ!¹e¸)°OÑ'DÐ$ä-2¯\©\Ð:L×:VÑ:VÐWYÐ[]Ó:^Ð`tÓ-uÐ*Ø%*Ð-?Ñ%?ÐB\Ñ%\Ð"Ü&+§l¡lÐ3GÐI[Ó&\Ð#à×"Ñ"Ð#6Õ7ð	8ô —i‘i °Ô4ˆð "×+Ñ+¨A¨qÓ1ˆØ!×)Ñ)¨*°g¸t×?WÑ?WÐZ^×ZgÑZgÑ?gÓhˆØ—i‘i Ó,ˆÜ—i‘i × 0Ñ 0°Ó ?Ó@À;ÑNˆØ—m‘m KÓ0ˆð Ð&Ø×,Ñ,¨T¯^©^Ð=OÔPàÐ.Ð.Ð.r8   rU   )rK   rL   rM   r&   rr   r.   r‰   rŠ   r0   rO   rH   r	   r   r   rF   rP   rQ   s   @r7   rt   rt   s   s»   ø„ ð8˜}ð 8¸õ 8ò0	ò6ð& )-ñ_/à—|‘|ð_/ð # 5§<¡<°·±Ð#=Ñ>ð_/ð Ÿ™ tÑ+ð	_/ð
  ™ð_/ð Ð-Ñ.ð_/ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷_/r8   rt   c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚMiniMaxRotaryEmbeddingÚinv_freqNru   c                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultrÉ   F)Ú
persistentÚoriginal_inv_freq)r-   r.   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenru   Úrope_parametersrË   Úcompute_default_rope_parametersr   Úattention_scalingr‹   Úclone)r4   ru   ÚdeviceÚrope_init_fnrÉ   r6   s        €r7   r.   zMiniMaxRotaryEmbedding.__init__
  s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr8   rÖ   ztorch.devicer­   r+   ztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarw   Nç      ð?r   r;   rž   )rÖ   r>   )	rÒ   r~   r5   r   r0   r�   Úint64r?   rN   )ru   rÖ   r­   r‘   rg   Úattention_factorrÉ   s          r7   rÓ   z6MiniMaxRotaryEmbedding.compute_default_rope_parameters  s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r8   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   r<   r%   ÚmpsÚcpuF)Údevice_typeÚenabledr;   rf   rž   )rÉ   rN   ÚexpandrI   r?   rÖ   Ú
isinstanceÚtypeÚstrr!   r¢   r0   r©   ÚcosrÔ   Úsinr>   )
r4   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrà   ÚfreqsÚembræ   rç   s
             r7   rF   zMiniMaxRotaryEmbedding.forward8  sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$rU   )NNN)rK   rL   rM   r0   rO   Ú__annotations__r&   r.   Ústaticmethodr   rr   rH   rN   rÓ   Úno_gradr   rF   rP   rQ   s   @r7   rÈ   rÈ     s˜   ø… Ø�l‰lÓñV˜}õ Vð  à'+Ø+/Ø"ñ*Ø Ñ$ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r8   rÈ   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr<   r;   rf   )rI   r0   r©   )rè   Úx1Úx2s      r7   Úrotate_halfrô   H  sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r8   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r¦   rô   )ÚqÚkræ   rç   Úunsqueeze_dimÚq_embedÚk_embeds          r7   Úapply_rotary_pos_embrü   O  sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr8   r9   Ún_repr+   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r%   N)rI   râ   r    )r9   rý   ÚbatchÚnum_key_value_headsÚslenrw   s         r7   Ú	repeat_kvr  i  so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr8   ÚmoduleÚqueryÚkeyÚvaluerš   ÚscalingÚdropoutrœ   c                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )Nr;   r   r<   )rg   r>   )ÚpÚtrainingr%   )r  Únum_key_value_groupsr0   r¨   r¢   r   Ú
functionalÚsoftmaxr@   r?   r>   r  r  Ú
contiguous)r  r  r  r  rš   r  r  rœ   r±   r²   Úattn_weightsr´   s               r7   Úeager_attention_forwardr  u  sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r8   c                   óô   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej                  de	ej                  ej                  f   dej                  dz  d	e
dz  d
ee   de	ej                  ej                  dz  f   fd„Zˆ xZS )ÚMiniMaxAttentionz=Multi-headed attention from 'Attention Is All You Need' paperru   r\   c                 ó  •— t         ‰| �  «        || _        || _        t	        |dd «      xs |j
                  |j                  z  | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  d¬«      | _        t        j                  |j
                  |j                  | j                  z  d¬«      | _        t        j                  |j
                  |j                  | j                  z  d¬«      | _        t        j                  |j                  | j                  z  |j
                  d¬«      | _        y )Nrw   g      à¿TFrx   )r-   r.   ru   r\   r~   r5   r   rw   r   r  r  Úattention_dropoutÚ	is_causalr   r…   Úq_projÚk_projÚv_projÚo_proj©r4   ru   r\   r6   s      €r7   r.   zMiniMaxAttention.__init__’  s2  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°DÓ9Òm¸V×=OÑ=OÐSY×SmÑSmÑ=mˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÔkˆ�r8   Nr9   r™   rš   r›   rœ   r+   c           
      ó6  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  sdn| j                   | j"                  t%        | j                  dd «      dœ|¤Ž\  }} |j&                  g |¢d‘­Ž j)                  «       }| j+                  |«      }||fS )Nr<   r%   r;   ç        Úsliding_window)r  r  r  )rI   rw   r  Úviewr¢   r  r  rü   Úupdater\   r   Úget_interfaceru   Ú_attn_implementationr  r  r  r  r~   r    r  r  )r4   r9   r™   rš   r›   rœ   Úinput_shapeÚhidden_shaper°   r±   r²   ræ   rç   Úattention_interfacer´   r  s                   r7   rF   zMiniMaxAttention.forward   s¢  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LÜ" 4§;¡;Ð0@À$ÓGñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r8   rU   )rK   rL   rM   Ú__doc__r&   rr   r.   r0   rO   rH   r	   r   r   rF   rP   rQ   s   @r7   r  r  Ž  s    ø„ áGðl˜}ð l¸õ lð& )-ñ')à—|‘|ð')ð # 5§<¡<°·±Ð#=Ñ>ð')ð Ÿ™ tÑ+ð	')ð
  ™ð')ð Ð-Ñ.ð')ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷')r8   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚMiniMaxTopKRouterc                 ó  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        t        j                  t        j                  | j
                  | j                  «      «      | _        y rU   )r-   r.   Únum_experts_per_tokÚtop_kÚnum_local_expertsÚnum_expertsr5   Ú
hidden_dimr   r/   r0   Úemptyr2   ©r4   ru   r6   s     €r7   r.   zMiniMaxTopKRouter.__init__Ë  s[   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×3Ñ3ˆÔØ ×,Ñ,ˆŒÜ—l‘l¤5§;¡;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆ�r8   c                 óp  — |j                  d| j                  «      }t        j                  || j                  «      }t
        j                  j                  j                  |j                  «       d¬«      }t        j                  || j                  d¬«      \  }}||j                  dd¬«      z  }|}|||fS )Nr<   rf   T)rg   r=   )r    r.  rª   Úlinearr2   r0   r   r  r  rN   Útopkr+  Úsum)r4   r9   Úrouter_logitsÚrouter_probsÚrouter_top_valueÚrouter_indicesÚrouter_scoress          r7   rF   zMiniMaxTopKRouter.forwardÒ  sŸ   € Ø%×-Ñ-¨b°$·/±/ÓBˆÜŸ™ °·±Ó<ˆÜ—x‘x×*Ñ*×2Ñ2°=×3FÑ3FÓ3HÈbÐ2ÓQˆÜ+0¯:©:°lÀDÇJÁJÐTVÔ+WÑ(Ð˜.ØÐ,×0Ñ0°RÀÐ0ÓFÑFÐØ(ˆØ˜m¨^Ð;Ð;r8   )rK   rL   rM   r.   rF   rP   rQ   s   @r7   r(  r(  Ê  s   ø„ ôSö<r8   r(  c                   ó�   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  dej                  dej                  fd„Zˆ xZ	S )	ÚMiniMaxExpertsz2Collection of expert weights stored as 3D tensors.ru   c                 óò  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        t        j                  t        j                  | j                  d| j                  z  | j
                  «      «      | _        t        j                  t        j                  | j                  | j
                  | j                  «      «      | _        t        |j                     | _        y )Nr;   )r-   r.   r,  r-  r5   r.  Úintermediate_sizeÚintermediate_dimr   r/   r0   r/  Úgate_up_projÚ	down_projr   r‚   rƒ   r0  s     €r7   r.   zMiniMaxExperts.__init__à  s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× 8Ñ 8ˆÔÜŸL™L¬¯©°T×5EÑ5EÀqÈ4×K`ÑK`ÑG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸ™¤e§k¡k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Ñ/ˆ�r8   r9   Útop_k_indexÚtop_k_weightsr+   c                 óf  — t        j                  |«      }t        j                  «       5  t         j                  j                  j                  || j                  ¬«      }|j                  ddd«      }t        j                  |j                  d¬«      d«      j                  «       }d d d «       D ]ð  }|d   }|| j                  k(  rŒt        j                  |   «      \  }}	||	   }
t        j                  j                  |
| j                  |   «      j                  dd¬«      \  }}| j                  |«      |z  }t        j                  j                  || j                   |   «      }|||	|d f   z  }|j#                  d|	|j%                  |j&                  «      «       Œò |S # 1 sw Y   �ŒxY w)N)Únum_classesr;   r%   r   )r<   rŸ   rf   r<   )r0   Ú
zeros_likerð   r   r  Úone_hotr-  ÚpermuteÚgreaterr4  Únonzeror—   r2  r?  Úchunkrƒ   r@  Ú
index_add_r?   r>   )r4   r9   rA  rB  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 r7   rF   zMiniMaxExperts.forwardé  s™  € ô $×.Ñ.¨}Ó=ÐÜ�]‰]‹_ñ 	SÜŸ(™(×-Ñ-×5Ñ5°kÈt×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸ™ {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷	Sð
 %ò 
	nˆJØ# A™ˆJØ˜T×-Ñ-Ò-ØÜ#(§;¡;¨{¸:Ñ/FÓ#GÑ ˆI�yØ)¨)Ñ4ˆMÜ—}‘}×+Ñ+¨M¸4×;LÑ;LÈZÑ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆD�"Ø$(§K¡K°Ó$5¸Ñ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9NÐPT×P^ÑP^Ð_iÑPjÓ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÑ<eÑ$eÐ!Ø×*Ñ*¨1¨iÐ9N×9QÑ9QÐRe×RkÑRkÓ9lÕmð
	nð #Ð"÷#	Sñ 	Sús   ªA=F&Æ&F0)
rK   rL   rM   r&  r&   r.   r0   rO   rF   rP   rQ   s   @r7   r;  r;  Ü  sM   ø„ á<ð0˜}õ 0ð#à—|‘|ð#ð —\‘\ð#ð —|‘|ð	#ð
 
�‰÷#r8   r;  c                   ót   ‡ — e Zd Zˆ fd„Zdej
                  deej
                  ej
                  f   fd„Zˆ xZS )ÚMiniMaxSparseMoeBlockc                 ó¦   •— t         ‰| �  «        |j                  | _        |j                  | _        t        |«      | _        t        |«      | _	        y rU   )
r-   r.   r*  r+  Úrouter_jitter_noiseÚjitter_noiser(  rS  r;  Úexpertsr0  s     €r7   r.   zMiniMaxSparseMoeBlock.__init__  sA   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø"×6Ñ6ˆÔÜ% fÓ-ˆŒ	Ü% fÓ-ˆ�r8   r9   r+   c                 ó”  — |j                   \  }}}| j                  rQ| j                  dkD  rB|t        j                  |«      j                  d| j                  z
  d| j                  z   «      z  }|j                  d|j                   d   «      }| j                  |«      \  }}}| j                  |||«      }|j                  |||«      }|S )Nr   rÚ   r<   )
rI   r  rZ  r0   Ú
empty_likeÚuniform_r  rS  r[  r    )r4   r9   r¬   Úsequence_lengthr.  r]   rB  rA  s           r7   rF   zMiniMaxSparseMoeBlock.forward  sÀ   € Ø2?×2EÑ2EÑ/ˆ
�O ZØ�=Š=˜T×.Ñ.°Ò2ØœU×-Ñ-¨mÓ<×EÑEÀcÈD×L]ÑL]ÑF]Ð_bÐei×evÑevÑ_vÓwÑwˆMØ%×*Ñ*¨2¨}×/BÑ/BÀ2Ñ/FÓGˆØ(,¯	©	°-Ó(@Ñ%ˆˆ=˜+ØŸ™ ]°KÀÓOˆØ%×-Ñ-¨j¸/È:ÓVˆØÐr8   )	rK   rL   rM   r.   r0   rO   rH   rF   rP   rQ   s   @r7   rW  rW    s1   ø„ ô.ð U§\¡\ð °e¸E¿L¹LÈ%Ï,É,Ð<VÑ6W÷ r8   rW  c                   óD  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dee   deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚMiniMaxDecoderLayerru   r\   c                 ó   •— t         ‰| �  «        |j                  | _        t        ||«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        || _	        t        |d«      r|j                  |   nd | _        |j                  | _        |j                  | _        t        |«      | _        | j                  dk(  r4t#        ||«      | _        |j$                  | _        |j(                  | _        y t        ||«      | _        |j,                  | _        |j.                  | _        y )N©r*   rŒ   Úlinear_attention)r-   r.   r5   r  Ú	self_attnr)   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr\   ÚhasattrrŒ   Ú
block_typeÚmlp_alpha_factorÚmlp_beta_factorrW  Úmlprt   Úlinear_attn_alpha_factorÚattn_alpha_factorÚlinear_attn_beta_factorÚattn_beta_factorÚfull_attn_alpha_factorÚfull_attn_beta_factorr  s      €r7   r.   zMiniMaxDecoderLayer.__init__  s  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä)¨&°)Ó<ˆŒÜ-¨f×.@Ñ.@Àf×FYÑFYÔZˆÔÜ(6°v×7IÑ7IÈv×ObÑObÔ(cˆÔ%à"ˆŒÜ;BÀ6È=Ô;Y˜&×,Ñ,¨YÒ7Ð_cˆŒØ &× 7Ñ 7ˆÔØ%×5Ñ5ˆÔÜ(¨Ó0ˆŒØ�?‰?Ð0Ò0Ü6°v¸yÓIˆDŒNØ%+×%DÑ%DˆDÔ"Ø$*×$BÑ$BˆDÕ!ä-¨f°iÓ@ˆDŒNØ%+×%BÑ%BˆDÔ"Ø$*×$@Ñ$@ˆDÕ!r8   Nr9   r™   rš   ré   r›   Ú	use_cacherœ   r+   c           
      ó(  — | j                  |«      }|} | j                  d||||||dœ|¤Ž\  }}	|| j                  z  || j                  z  z   }| j	                  |«      }|}| j                  |«      }|| j                  z  || j                  z  z   }|S )N)r9   r™   rš   ré   r›   rt  © )rg  re  ro  rq  rh  rm  rk  rl  )
r4   r9   r™   rš   ré   r›   rt  rœ   Úresidualr]   s
             r7   rF   zMiniMaxDecoderLayer.forward.  sº   € ð ×,Ñ,¨]Ó;ˆØ ˆØ)˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø%Ø+Øñ
ð ñ
Ñˆ�qð ! 4×#9Ñ#9Ñ9¸MÈD×LaÑLaÑ<aÑaˆØ×5Ñ5°mÓDˆØ ˆØŸ™ Ó/ˆØ  4×#8Ñ#8Ñ8¸=È4×K_ÑK_Ñ;_Ñ_ˆàÐr8   )NNNNF)rK   rL   rM   r&   rr   r.   r0   rO   rH   Ú
LongTensorr	   r¤   r   r   ÚFloatTensorrF   rP   rQ   s   @r7   ra  ra    sè   ø„ ðA˜}ð A¸õ Að2 IMØ.2Ø04Ø(,Ø!&ñà—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð  ™ðð ˜$‘;ðð Ð-Ñ.ðð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷r8   ra  c                   óœ   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZ eedd¬	«      eeegd
œZ ej*                  «       ˆ fd„«       Zˆ xZS )ÚMiniMaxPreTrainedModelru   ÚmodelTra  r›   Fzmlp.gater   )Ú
layer_nameÚindex)r5  r9   Ú
attentionsc                 óÊ  •— t         ‰| �  |«       | j                  j                  }t	        |t
        «      rEt        j                  |j                  d|¬«       t        j                  |j                  d|¬«       n2t	        |t        «      r"t        j                  |j                  d|¬«       t	        |t        «      r¦|j                  «       }|j                  |«      \  }}}t        j                  |j                   |«       t        j                  |j"                  |«       t        j                  |j$                  |«       t        j                  |j&                  |«       y y )Nr  )rB   Ústd)r-   Ú_init_weightsru   Úinitializer_rangerã   r;  ÚinitÚnormal_r?  r@  r(  r2   rt   r‰   rŠ   Úcopy_rz   r{   r|   r}   )r4   r  r�  rz   r{   r|   r}   r6   s          €r7   r‚  z$MiniMaxPreTrainedModel._init_weights^  sù   ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœnÔ-Ü�L‰L˜×,Ñ,°3¸CÕ@Ü�L‰L˜×)Ñ)°¸Ö=Ü˜Ô 1Ô2Ü�L‰L˜Ÿ™¨S°cÕ:Ü�fÔ7Ô8Ø×.Ñ.Ó0ˆJØ5;×5IÑ5IÈ*Ó5UÑ2ˆK˜ NÜ�J‰J�v×(Ñ(¨*Ô5Ü�J‰J�v×)Ñ)¨;Ô7Ü�J‰J�v×'Ñ'¨Ô3Ü�J‰J�v×,Ñ,¨nÕ=ð 9r8   )rK   rL   rM   r&   rî   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr#   r(  ra  r  rt   Ú_can_record_outputsr0   rð   r‚  rP   rQ   s   @r7   r{  r{  L  s‚   ø… àÓØÐØ&*Ð#Ø.Ð/ÐØ#4Ð"5ÐØÐØ€NØÐØ"ÐØ"&Ðá'Ð(9ÀjÐXYÔZØ,Ø'Ð)BÐCñÐð €U‡]�]ƒ_ó>ó ô>r8   r{  c                   óî   ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de
dz  dej                  dz  d	edz  d
ee   deez  fd„«       «       Zˆ xZS )ÚMiniMaxModelru   c           	      ó  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t#        |¬«      | _        d| _        | j)                  «        y c c}w )Nrc  )ru   F)r-   r.   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr5   Úembed_tokensÚ
ModuleListrY   r€   ra  ri   r)   rf  r„   rÈ   Ú
rotary_embÚgradient_checkpointingÚ	post_initr  s      €r7   r.   zMiniMaxModel.__init__r  sÆ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜEJÈ6×KcÑKcÓEdÖe¸	Ô  ¨Õ3Òeó
ˆŒô # 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ü0¸Ô?ˆŒØ&+ˆÔ#ð 	�‰Õùò fs   ÂDNÚ	input_idsrš   ré   r›   Úinputs_embedsrt  rœ   r+   c           
      óâ  — |d u |d uz  rt        d«      ‚|r|€t        «       }n*|r(t        |t        «      st        dt        |«      › d�«      ‚|€| j	                  |«      }|€V|�|j                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }| j                  j                  €t        nt        }	 |	| j                  ||||¬«      }
|}| j                  ||«      }t!        | j"                  «      D ]5  \  }}| j                  j$                  |   dk(  r|
}n|} ||f|||||d	œ|¤Ž}Œ7 | j'                  |«      }t)        ||¬
«      S )Nz:You must specify exactly one of input_ids or inputs_embedszSMiniMax uses cache of its own and is not compatible with `past_key_values` of type ú.r   r%   )rÖ   )ru   rž  rš   r›   ré   Úfull_attention)rš   r™   ré   r›   rt  )Úlast_hidden_stater›   )Ú
ValueErrorrS   rã   rä   r˜  Úget_seq_lengthr0   r�   rI   rÖ   r¦   ru   r  r   r   rš  Ú	enumerateri   rŒ   r„   r   )r4   r�  rš   ré   r›   rž  rt  rœ   Úpast_seen_tokensÚmask_functionÚcausal_maskr9   r™   rµ   Údecoder_layerÚinput_attention_masks                   r7   rF   zMiniMaxModel.forward‚  s©  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZá˜Ð0Ü*›n‰OÙœz¨/¼<ÔHÜØeÔfjÐkzÓf{Ðe|Ð|}Ð~óð ð Ð Ø ×-Ñ-¨iÓ8ˆMàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÐ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m¸\ÓJÐä )¨$¯+©+Ó 6ò 	ÑˆAˆ}Ø�{‰{×&Ñ& qÑ)Ð-=Ò=Ø'2Ñ$ð (6Ð$á)Øðà3Ø$7Ø)Ø /Ø#ñð ñ‰Mð	ð" Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r8   )NNNNNN)rK   rL   rM   r&   r.   r"   r$   r0   rx  rO   rS   ry  r¤   r   r   rH   r   rF   rP   rQ   s   @r7   r’  r’  p  sÌ   ø„ ð˜}õ ð   Øð .2Ø.2Ø04Ø/3Ø26Ø!%ñ>
à×#Ñ# dÑ*ð>
ð Ÿ™ tÑ+ð>
ð ×&Ñ&¨Ñ-ð	>
ð
 &¨Ñ,ð>
ð ×(Ñ(¨4Ñ/ð>
ð ˜$‘;ð>
ð Ð+Ñ,ð>
ð 
Ð'Ñ	'ò>
ó ó  ô>
r8   r’  Úgate_logitsr-  c                 ó¾  — | �t        | t        «      syt        | t        «      rC| d   j                  }t        j                  | D �cg c]  }|j                  |«      ‘Œ c}d¬«      }t        j                  j                  j                  d¬«      }t        j                  ||d¬«      \  }}	t        j                  j                  j                  |	|«      }
|€>t        j                  |
j                  «       d¬«      }t        j                  |d¬«      }�n|j                  \  }}|j                  d   ||z  z  }|ddd…dd…ddf   j                  |||||f«      j                  d||«      j                  «      }t        j                   |
j                  «       |z  d¬«      t        j                   |d¬«      z  }|ddd…dd…df   j                  ||||f«      j                  d|«      j                  |«      }t        j                   ||z  d¬«      t        j                   |d¬«      z  }t        j                   ||j#                  d«      z  «      }||z  S c c}w )aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   rf   r<   )rã   rH   rÖ   r0   r©   r?   r   r  r  r3  rF  rB   rN   rI   râ   r    r4  r¦   )r«  r-  r+  rš   Úcompute_deviceÚ
layer_gateÚconcatenated_gate_logitsÚrouting_weightsr]   Úselected_expertsrM  Útokens_per_expertÚrouter_prob_per_expertr¬   r_  r€   Úexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_losss                      r7   Úload_balancing_loss_funcr·  Å  s9  € ð: Ð¤*¨[¼%Ô"@Øä�+œuÔ%Ø$ Q™×.Ñ.ˆÜ#(§9¡9Ð^iÖ-jÐPZ¨j¯m©m¸NÕ.KÒ-jÐpqÔ#rÐ ä—h‘h×)Ñ)×1Ñ1Ð2JÐPRÐ1ÓS€OäŸ*™* _°eÀÔDÑ€AÐä—(‘(×%Ñ%×-Ñ-Ð.>ÀÓL€KàÐä!ŸJ™J {×'8Ñ'8Ó':ÀÔBÐô "'§¡¨OÀÔ!CÒà&4×&:Ñ&:Ñ#ˆ
�OØ4×:Ñ:¸1Ñ=À*ÈÑB^Ñ_Ðð ˜4¢¢A t¨TÐ1Ñ2ß‰VÐ&¨
°OÀUÈKÐXÓYß‰W�R˜ Ó,ß‰R�Óð	 	ô "ŸI™I k×&7Ñ&7Ó&9Ð<QÑ&QÐWXÔYÔ\a×\eÑ\eØ! qô]
ñ 
Ðð ˜4¢¢A tÐ+Ñ,ß‰VÐ&¨
°OÀ[ÐQÓRß‰W�R˜Ó%ß‰R�Óð	 	)ô "'§¡¨?Ð=]Ñ+]ÐcdÔ!eÔhm×hqÑhqØ,°!ôi
ñ "
Ðô —9‘9Ð.Ð1G×1QÑ1QÐRSÓ1TÑTÓU€LØ˜+Ñ%Ð%ùò[ .ks   ÁIc                   óN  ‡ — e Zd ZddiZddiZddgdgfiZˆ fd„Zee	 	 	 	 	 	 	 	 	 dd	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  dedz  dedz  dee	j                  z  dee   defd„«       «       Zˆ xZS )ÚMiniMaxForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr9   Úlogitsc                 óN  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        |j                  | _	        |j                  | _        |j                  | _        | j                  «        y )NFrx   )r-   r.   r’  r|  r–  r   r…   r5   rº  Úrouter_aux_loss_coefr,  r-  r*  rœ  r0  s     €r7   r.   zMiniMaxForCausalLM.__init__  s   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×3Ñ3ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Õr8   Nr�  rš   ré   r›   rž  Úlabelsrt  Úoutput_router_logitsÚlogits_to_keeprœ   r+   c
                 ój  — |�|n| j                   j                  } | j                  d|||||||dœ|
¤Ž}|j                  }t	        |	t
        «      rt        |	 d«      n|	}| j                  |dd…|dd…f   «      }d}|� | j                  ||| j                  fi |
¤Ž}d}|rYt        |j                  | j                  | j                  |«      }|�+|| j                  |j                  |j                   «      z  z  }t#        ||||j$                  |j&                  |j(                  |j                  ¬«      S )a€  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MiniMaxForCausalLM

        >>> model = MiniMaxForCausalLM.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r�  rš   ré   r›   rž  rt  rÀ  )ÚlossÚaux_lossr¼  r›   r9   r  r5  rv  )ru   rÀ  r|  r¢  rã   rr   Úslicerº  Úloss_functionr–  r·  r5  r-  r*  r¾  r?   rÖ   r   r›   r9   r  )r4   r�  rš   ré   r›   rž  r¿  rt  rÀ  rÁ  rœ   Úoutputsr9   Úslice_indicesr¼  rÃ  rÄ  s                    r7   rF   zMiniMaxForCausalLM.forward)  sU  € ðN %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð
 +5¨$¯*©*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ñ	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDàˆÙÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ð!Ø˜×1Ñ1°H·K±KÀÇÁÓ4LÑLÑL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r8   )	NNNNNNNNr   )rK   rL   rM   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr.   r    r   r0   rx  rO   r	   ry  r¤   rr   r   r   r   rF   rP   rQ   s   @r7   r¹  r¹    s5  ø„ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hô
ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ñP
à×#Ñ# dÑ*ðP
ð Ÿ™ tÑ+ðP
ð ×&Ñ&¨Ñ-ð	P
ð
  ™ðP
ð ×(Ñ(¨4Ñ/ðP
ð × Ñ  4Ñ'ðP
ð ˜$‘;ðP
ð # T™kðP
ð ˜eŸl™lÑ*ðP
ð Ð+Ñ,ðP
ð 
#òP
ó ó ôP
r8   r¹  c                   ó   — e Zd Zy)Ú MiniMaxForSequenceClassificationN©rK   rL   rM   rv  r8   r7   rÍ  rÍ  ~  ó   „ Ør8   rÍ  c                   ó   — e Zd Zy)ÚMiniMaxForTokenClassificationNrÎ  rv  r8   r7   rÑ  rÑ  ‚  rÏ  r8   rÑ  c                   ó   — e Zd Zy)ÚMiniMaxForQuestionAnsweringNrÎ  rv  r8   r7   rÓ  rÓ  †  rÏ  r8   rÓ  )r{  r’  r¹  rÍ  rÑ  rÓ  )r%   )r  )Nr;   N)SÚcollections.abcr   Útypingr   r0   Útorch.nn.functionalr   r  rª   Ú r   r„  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   r   r   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r    Úutils.genericr!   r"   Úutils.output_capturingr#   r$   Úconfiguration_minimaxr&   ÚModuler)   rS   rt   rÈ   rô   rü   rO   rr   r  rN   r  r  r(  r;  rW  ra  r{  r’  rH   r·  r¹  rÍ  rÑ  rÓ  Ú__all__rv  r8   r7   ú<module>ré     sä  ðõ, %Ý ã ß Ð Ý å &Ý !ß .Ý )÷ó ÷ SÝ B÷ó ÷ Rß Kß FÝ &ß IÑ Iß Gß EÝ 0ñ ˜YÓ'ôJ�R—Y‘Yó Jó (ðJô("I�<ô "IôJQ/ §	¡	ô Q/ôh><˜RŸY™Yô ><òB(ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ñ2 Ð)Ó*ô8)�r—y‘yó 8)ó +ð8)ôv<˜Ÿ	™	ô <ð$ ô$#�R—Y‘Yó $#ó ð$#ôN˜BŸI™Iô ô&2Ð4ô 2ðj ô >˜_ó  >ó ð >ðF ôQ
Ð)ó Q
ó ðQ
ðl #Ø
Ø*.ñ	O&Ø—‘  e§l¡lÑ 3Ñ3°dÑ:ðO&à�t‘ðO&ð —L‘L 4Ñ'ð	O&ð
 ‡\�\�CÑóO&ðd ôc
Ð/°ó c
ó ðc
ôL	Ð'GÐI_ô 	ô	Ð$AÐCYô 	ô	Ð"=Ð?Uô 	ò�r8   