Ë
    ÜÓ^jòW  ã                   óØ  — d Z ddlZddlmc mZ ddlmZ ddlmZ ddlm	Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5  e"jl                  e7«      Z8 e!d¬«      e G d„ de«      «       «       Z9 G d„ de3«      Z: G d„ de«      Z; G d„ dejx                  «      Z= G d „ d!e)«      Z> G d"„ d#e+«      Z? G d$„ d%e5«      Z@ G d&„ d'e4«      ZA G d(„ d)e,e«      ZB G d*„ d+e2«      ZC G d,„ d-e1«      ZD G d.„ d/e-«      ZE G d0„ d1e/«      ZF G d2„ d3e0«      ZG G d4„ d5e.«      ZHg d6¢ZIy)7zPyTorch MiniMax model.é    N)Ústrict)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚMoeModelOutputWithPast)ÚRopeParameters)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚGemma2RotaryEmbedding)ÚMixtralAttentionÚMixtralDecoderLayerÚMixtralForCausalLMÚMixtralForQuestionAnsweringÚ MixtralForSequenceClassificationÚMixtralForTokenClassificationÚMixtralModelÚMixtralPreTrainedModelÚMixtralRMSNormÚMixtralSparseMoeBlockÚMixtralTopKRouterzMiniMaxAI/MiniMax-Text-01-hf)Ú
checkpointc                   ó®  ‡ — e Zd ZU dZdZdgZdZdddddddd	œZd
gdgfddgdgfdgdgfdœZdddddœZ	ddiZ
dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed <   d!Zeed"<   d#Zeed$<   d%Zeed&<   d'Zeed(<   d)Zeed*<   dZedz  ed+<   d,Zedz  ed-<   d.Zeee   z  dz  ed/<   d0Z eed1<   dZ!edz  ed2<   d3Z"eez  ed4<   d.Z#eed5<   dZ$eed<   d0Z%eed6<   d7Z&eed8<   d3Z'eed9<   dZ(e)e*z  dz  ed:<   dZ+ee   dz  ed;<   d<Z,eed=<   d,Z-eez  ed><   d,Z.eez  ed?<   d,Z/eez  ed@<   d,Z0eez  edA<   d,Z1eez  edB<   d,Z2eez  edC<   ˆ fdD„Z3ˆ xZ4S )EÚMiniMaxConfiga¡  
    block_size (`int`, *optional*, defaults to 256):
        The length of each attention block, determining how queries, keys, and values
        are grouped and processed for intra- and inter-block attention.
    full_attn_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after normal attention.
    full_attn_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after normal attention.
    linear_attn_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after lightning attention.
    linear_attn_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after lightning attention.
    mlp_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after MLP.
    mlp_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after MLP.

    ```python
    >>> from transformers import MiniMaxModel, MiniMaxConfig

    >>> # Initializing a MiniMax style configuration
    >>> configuration = MiniMaxConfig()

    >>> # Initializing a model from the MiniMax style configuration
    >>> model = MiniMaxModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚminimaxÚpast_key_valuesg    €„.AÚcolwiseÚrowwiseÚpacked_colwiseÚmoe_tp_experts)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater.   r/   r0   Únum_expertsÚnum_local_expertsi }  Ú
vocab_sizei   Úhidden_sizei 8  Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   Únum_key_value_headsNÚhead_dimÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeçñhãˆµøä>Úrms_norm_epsTÚ	use_cacheÚpad_token_idé   Úbos_token_idr   Úeos_token_idFÚtie_word_embeddingsÚsliding_windowg        Úattention_dropoutÚnum_experts_per_tokÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefÚrouter_jitter_noiseÚrope_parametersÚlayer_typesé   Ú
block_sizeÚfull_attn_alpha_factorÚfull_attn_beta_factorÚlinear_attn_alpha_factorÚlinear_attn_beta_factorÚmlp_alpha_factorÚmlp_beta_factorc                 óö   •— | j                   €| j                  | _         | j                  €;t        | j                  «      D �cg c]  }t        |dz   dz  «      rdnd‘Œ c}| _        t        ‰| �  di |¤Ž y c c}w )NrM   r   Úfull_attentionÚlinear_attention© )rC   rA   rX   Úranger@   ÚboolÚsuperÚ__post_init__)ÚselfÚkwargsÚiÚ	__class__s      €úv/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/minimax/modular_minimax.pyrh   zMiniMaxConfig.__post_init__‘   sz   ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×ÑÐ#äW\Ð]a×]sÑ]sÓWtö ØRS¤D¨!¨a©%°1©Ô$5Ñ Ð;MÑMò ˆDÔô 	‰ÑÑ' Ó'ùò	 s   ÁA6)5Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚdefault_thetaÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_mapr<   ÚintÚ__annotations__r=   r>   r@   rA   rC   rD   rF   ÚstrrG   rH   ÚfloatrJ   rK   rf   rL   rN   rO   ÚlistrP   rQ   rR   rS   r;   rT   rU   rV   rW   r   ÚdictrX   rZ   r[   r\   r]   r^   r_   r`   rh   Ú__classcell__©rl   s   @rm   r'   r'   6   s;  ø… ñð< €JØ#4Ð"5ÐØ€Mà%.Ø%.Ø%.Ø%.Ø-=Ø*3Ø 0ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð )Ø-;Ø*8Ø 0ñ	Ðð #Ð$7Ð8€Mà€J�ÓØ€K�ÓØ"Ð�sÓ"ØÐ�sÓØ!Ð˜Ó!Ø Ð˜Ó Ø€Hˆc�D‰jÓØ€J�ÓØ#,Ð˜SÓ,Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ#€L�#˜‘*Ó#Ø €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø %Ð˜Ó%Ø!%€N�C˜$‘JÓ%Ø%(Ð�u˜s‘{Ó(Ø Ð˜Ó ØÐ�sÓØ!&Ð˜$Ó&Ø"'Ð˜%Ó'Ø!$Ð˜Ó$Ø48€O�^ dÑ*¨TÑ1Ó8Ø$(€K��c‘˜TÑ!Ó(Ø€J�ÓØ*+Ð˜C %™KÓ+Ø)*Ð˜3 ™;Ó*Ø,-Ð˜c E™kÓ-Ø+,Ð˜S 5™[Ó,Ø$%Ð�c˜E‘kÓ%Ø#$€O�S˜5‘[Ó$÷	(ð 	(ó    r'   c                   ó   — e Zd Zy)ÚMiniMaxRMSNormN©rn   ro   rp   rd   r�   rm   rƒ   rƒ   �   ó   „ Ør�   rƒ   c                   ór   ‡ — e Zd Zˆ fd„Zd„ Zdefd„Zˆ fd„Zdefd„Zde	j                  fd	„Zd
efd„Zˆ xZS )ÚMiniMaxCachec                 ó0   •— t         ‰| �  «        g | _        y ©N)rg   Ú__init__Úlinear_cache©ri   rl   s    €rm   rŠ   zMiniMaxCache.__init__¢   s   ø€ Ü‰ÑÔØ02ˆÕr�   c                 ó¦   — t        t        | j                  «      |dz   «      D ]  }| j                  j                  g «       Œ || j                  |<   y )NrM   )re   Úlenr‹   Úappend)ri   Ú	layer_idxr‹   Ú_s       rm   Úset_linear_cachezMiniMaxCache.set_linear_cache¦   sK   € ä”s˜4×,Ñ,Ó-¨y¸1©}Ó=ò 	)ˆAØ×Ñ×$Ñ$ RÕ(ð	)à'3ˆ×Ñ˜)Ò$r�   r�   c                 ó>   — |t        | «      k  r| j                  |   S y r‰   )rŽ   r‹   )ri   r�   s     rm   Úget_linear_cachezMiniMaxCache.get_linear_cache¬   s"   € Ø”s˜4“yÒ Ø×$Ñ$ YÑ/Ð/Ør�   c                 óZ   •— t        t        ‰| �	  «       t        | j                  «      «      S r‰   )Úmaxrg   Ú__len__rŽ   r‹   rŒ   s    €rm   r—   zMiniMaxCache.__len__±   s"   ø€ Ü”5‘7‘?Ó$¤c¨$×*;Ñ*;Ó&<Ó=Ð=r�   Úrepeatsc                 óò   — t        t        | «      «      D ]`  }| j                  |   g k7  r.| j                  |   j                  |d¬«      | j                  |<   ŒC| j                  |   j                  |«       Œb y )Nr   ©Údim)re   rŽ   r‹   Úrepeat_interleaver6   Úbatch_repeat_interleave)ri   r˜   r�   s      rm   r�   z$MiniMaxCache.batch_repeat_interleave´   ss   € Üœs 4›yÓ)ò 	HˆIØ× Ñ  Ñ+¨rÒ1Ø/3×/@Ñ/@ÀÑ/K×/]Ñ/]Ð^eÐklÐ/]Ó/m�×!Ñ! )Ò,à—‘˜IÑ&×>Ñ>¸wÕGñ		Hr�   Úindicesc                 óÚ   — t        t        | «      «      D ]T  }| j                  |   g k7  r"| j                  |   |df   | j                  |<   Œ7| j                  |   j	                  |«       ŒV y )N.)re   rŽ   r‹   r6   Úbatch_select_indices)ri   rž   r�   s      rm   r    z!MiniMaxCache.batch_select_indices»   sk   € Üœs 4›yÓ)ò 	EˆIØ× Ñ  Ñ+¨rÒ1Ø/3×/@Ñ/@ÀÑ/KÈGÐUXÈLÑ/Y�×!Ñ! )Ò,à—‘˜IÑ&×;Ñ;¸GÕDñ		Er�   Ú
max_lengthc                 ó   — t        d«      ‚)Nz*MiniMaxCache doesnot support `crop` method)ÚRuntimeError)ri   r¡   s     rm   ÚcropzMiniMaxCache.cropÂ   s   € ÜÐGÓHÐHr�   )rn   ro   rp   rŠ   r’   ry   r”   r—   r�   ÚtorchÚTensorr    r¤   r   r€   s   @rm   r‡   r‡   ¡   sL   ø„ ô3ò4ð¨#ó ô
>ðH¨só HðE¨E¯L©Ló EðI˜s÷ Ir�   r‡   c                   ó  ‡ — e Zd Zdedefˆ fd„Zd„ Zd„ Z	 ddej                  de
ej                  ej                  f   d	ej                  dz  d
edz  dee   de
ej                  ej                  dz  e
ej                     dz  f   fd„Zˆ xZS )ÚMiniMaxLightningAttentionÚconfigr�   c                 óê  •— t         ‰| �  «        || _        t        |dd «      xs |j                  |j
                  z  | _        |j
                  | _        |j                  | _        |j                  | _        t        |j                     | _        t        | j                  | j
                  z  «      | _        t        j                  |j                  | j
                  | j                  z  dz  d¬«      | _        t        j                  | j
                  | j                  z  |j                  d¬«      | _        t        j                  |j                  | j
                  | j                  z  d¬«      | _        | j'                  «       }| j)                  |«      \  }}}| j+                  d|«       | j+                  d|«       | j+                  d|«       | j+                  d|«       |j,                  |   | _        y )	NrD   r   F)ÚbiasÚ
slope_rateÚquery_decayÚ	key_decayÚdiagonal_decay)rg   rŠ   r�   Úgetattrr=   rA   rD   r@   rZ   r   rF   Úact_fnrƒ   r7   r   ÚLinearÚqkv_projÚout_projÚoutput_gateÚget_slope_rateÚdecay_factorsÚregister_bufferrX   Ú
layer_type)ri   r©   r�   r¬   r­   r®   r¯   rl   s          €rm   rŠ   z"MiniMaxLightningAttention.__init__Ç   s•  ø€ Ü‰ÑÔØ"ˆŒÜ ¨
°DÓ9Òm¸V×=OÑ=OÐSY×SmÑSmÑ=mˆŒØ#)×#=Ñ#=ˆÔ Ø!'×!9Ñ!9ˆÔØ ×+Ñ+ˆŒä˜V×.Ñ.Ñ/ˆŒÜ" 4§=¡=°4×3KÑ3KÑ#KÓLˆŒ	ÜŸ	™	 &×"4Ñ"4°d×6NÑ6NÐQU×Q^ÑQ^Ñ6^ÐabÑ6bÐinÔoˆŒÜŸ	™	 $×":Ñ":¸T¿]¹]Ñ"JÈF×L^ÑL^ÐejÔkˆŒÜŸ9™9 V×%7Ñ%7¸×9QÑ9QÐTX×TaÑTaÑ9aÐhmÔnˆÔà×(Ñ(Ó*ˆ
Ø15×1CÑ1CÀJÓ1OÑ.ˆ�Y à×Ñ˜\¨:Ô6Ø×Ñ˜]¨KÔ8Ø×Ñ˜[¨)Ô4Ø×ÑÐ-¨~Ô>à ×,Ñ,¨YÑ7ˆ�r�   c                 óæ   — ddd| j                   z  z  z  }t        j                  | j                   «      dz   }d| j                  | j                  dz
  dz   z  z
  dz   }||z  }||z  }|d d …d d f   }|S )NrM   r   rB   rI   )rA   r¥   Úaranger�   r@   )ri   ÚbaseÚexponentÚfactorÚrates        rm   r¶   z(MiniMaxLightningAttention.get_slope_rateß   s„   € Ø�A˜!˜d×6Ñ6Ñ6Ñ7Ñ8ˆÜ—<‘< × 8Ñ 8Ó9¸AÑ=ˆØ�T—^‘^ t×'=Ñ'=ÀÑ'AÀDÑ'HÑIÑIÈDÑPˆà�X‰~ˆØ�f‰}ˆØ’A�t˜T�MÑ"ˆàˆr�   c                 ó¦  — t        j                  | j                  «      dz   }t        j                  | |d d …d f   z  «      }t        j                  | | j                  |d d …d f   z
  z  «      }|d d …d f   |d d d …f   z
  }|d d d d …d d …f   }||z  }t        j                  |dk\  | t        d«      «      }t        j                  |«      }|||fS )NrM   r   z-inf)r¥   r»   rZ   ÚexpÚwherer|   )ri   r¬   Úblock_size_ranger­   r®   r¯   s         rm   r·   z'MiniMaxLightningAttention.decay_factorsê   s×   € Ü Ÿ<™<¨¯©Ó8¸1Ñ<Ðä—i‘i  Ð.>ºqÀ$¸wÑ.GÑ GÓHˆÜ—I‘I˜z˜k¨T¯_©_Ð?OÒPQÐSWÐPWÑ?XÑ-XÑYÓZˆ	à)ª!¨T¨'Ñ2Ð5EÀdÊAÀgÑ5NÑNˆØ'¨¨d²A²qÐ(8Ñ9ˆØ# nÑ4ˆÜŸ™ ^°qÑ%8¸>¸/Ì5ÐQWË=ÓYˆÜŸ™ >Ó2ˆà˜I ~Ð5Ð5r�   Nr3   Úposition_embeddingsr4   r)   rj   Úreturnc                 óú  — |j                   \  }}}|| j                  z   dz
  | j                  z  }	| j                  | j                  |«      «      }
|
j	                  ||| j
                  d| j                  z  «      }
t        j                  |
| j                  d¬«      \  }}}|j                  dd«      }|j                  dd«      }|j                  dd«      }d }|�|j                  | j                  «      }|�€t        j                  || j
                  | j                  | j                  «      j                  |«      }|�Q|j                  t        j                  ¬«      }|j                  |j!                  d«      j!                  d«       d«      }g }t#        |	«      D �]b  }|| j                  z  }t%        || j                  z   |«      }||z
  }|d d …d d …||…f   }|d d …d d …||…f   }|d d …d d …||…f   }| j&                  d d …d |…f   }| j(                  d d …| d …f   }| j*                  d d …d d …d |…d |…f   }t        j,                  | j.                   |z  «      }t        j0                  ||j                  dd«      «      }t        j0                  ||z  |«      }t        j0                  ||z  |«      }||z   }|j3                  |«       t        j0                  ||z  j                  dd«      |«      }||z  |z   }�Œe nºt        j,                  | j.                   «      } g }t#        |«      D ]Š  }|d d …d d …||dz   …f   }|d d …d d …||dz   …f   }|d d …d d …||dz   …f   }t        j0                  |j                  dd«      |«      }!| |z  |!z   }t        j0                  ||«      }|j3                  |«       ŒŒ t        j4                  |d¬«      }|j                  dd«      }|j	                  ||| j
                  | j                  z  «      }| j7                  |«      }t9        j:                  | j=                  |«      «      |z  }| j?                  |«      }|�|jA                  | j                  |«       ||fS )	NrM   r   rš   r   )Údtypeéÿÿÿÿr   éþÿÿÿ)!ÚshaperZ   r±   r³   ÚreshaperA   rD   r¥   ÚsplitÚ	transposer”   r�   ÚzerosÚtorf   Úmasked_fillÚ	unsqueezere   Úminr­   r®   r¯   rÁ   r¬   Úmatmulr�   Úcatr7   ÚFÚsigmoidrµ   r´   r’   )"ri   r3   rÄ   r4   r)   rj   Ú
batch_sizeÚseq_lenr=   Ú
num_blocksÚ
qkv_statesÚquery_statesÚ
key_statesÚvalue_statesÚattn_weights_interÚattn_outputrk   Ú	start_idxÚend_idxÚcurrent_block_sizeÚcurrent_query_statesÚcurrent_key_statesÚcurrent_value_statesÚcurrent_query_decayÚcurrent_key_decayÚcurrent_diagonal_decayÚblock_decayÚattn_weights_intraÚattn_output_intraÚattn_output_interÚcurrent_attn_outputÚnext_attn_weights_interÚratioÚcurrent_attn_weights_inters"                                     rm   Úforwardz!MiniMaxLightningAttention.forwardø   s®  € ð ,9×+>Ñ+>Ñ(ˆ
�G˜[Ø §¡Ñ/°!Ñ3¸¿¹ÑGˆ
à—[‘[ §¡¨}Ó!=Ó>ˆ
Ø×'Ñ'¨
°G¸T×=UÑ=UÐWXÐ[_×[hÑ[hÑWhÓiˆ
ä16·±¸ZÈÏÉÐ\]Ô1^Ñ.ˆ�j ,à#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆð "ÐØÐ&Ø!0×!AÑ!AÀ$Ç.Á.Ó!QÐàÑ%Ü!&§¡¨Z¸×9QÑ9QÐSW×S`ÑS`Ðbf×boÑboÓ!p×!sÑ!sØó"Ðð
 Ð)Ø!/×!2Ñ!2¼¿¹Ð!2Ó!D�Ø+×7Ñ7¸×9QÑ9QÐRSÓ9T×9^Ñ9^Ð_aÓ9bÐ8bÐdeÓf�àˆKÜ˜:Ó&ó `�Ø §¡Ñ/�	Ü˜i¨$¯/©/Ñ9¸7ÓC�Ø%,¨yÑ%8Ð"à'3²A²q¸)ÀGÐ:KÐ4KÑ'LÐ$Ø%/²²1°iÀÐ6GÐ0GÑ%HÐ"Ø'3²A²q¸)ÀGÐ:KÐ4KÑ'LÐ$à&*×&6Ñ&6²qÐ:MÐ;MÐ:MÐ7MÑ&NÐ#Ø$(§N¡N²1Ð7IÐ6IÑ6JÐ3JÑ$KÐ!Ø)-×)<Ñ)<ºQÂÐCVÐDVÐCVÐXkÐYkÐXkÐ=kÑ)lÐ&Ü#Ÿi™i¨¯©Ð(8Ð;MÑ(MÓN�ô &+§\¡\Ð2FÐHZ×HdÑHdÐegÐikÓHlÓ%mÐ"Ü$)§L¡LÐ1CÐF\Ñ1\Ð^rÓ$sÐ!ô %*§L¡LÐ1EÐH[Ñ1[Ð]oÓ$pÐ!ð '8Ð:KÑ&KÐ#Ø×"Ñ"Ð#6Ô7ô +0¯,©,Ø'Ð*;Ñ;×FÑFÀrÈ2ÓNÐPdó+Ð'ð &8¸+Ñ%EÐH_Ñ%_Ò"ñ;`ô@ —I‘I˜tŸ™Ð.Ó/ˆEØˆKÜ˜7“^ò 	8�Ø'3²A²q¸!¸aÀ!¹e¸)°OÑ'DÐ$Ø%/²²1°a¸!¸a¹%°i°Ñ%@Ð"Ø'3²A²q¸!¸aÀ!¹e¸)°OÑ'DÐ$ä-2¯\©\Ð:L×:VÑ:VÐWYÐ[]Ó:^Ð`tÓ-uÐ*Ø%*Ð-?Ñ%?ÐB\Ñ%\Ð"Ü&+§l¡lÐ3GÐI[Ó&\Ð#à×"Ñ"Ð#6Õ7ð	8ô —i‘i °Ô4ˆð "×+Ñ+¨A¨qÓ1ˆØ!×)Ñ)¨*°g¸t×?WÑ?WÐZ^×ZgÑZgÑ?gÓhˆØ—i‘i Ó,ˆÜ—i‘i × 0Ñ 0°Ó ?Ó@À;ÑNˆØ—m‘m KÓ0ˆð Ð&Ø×,Ñ,¨T¯^©^Ð=OÔPàÐ.Ð.Ð.r�   r‰   )rn   ro   rp   r'   ry   rŠ   r¶   r·   r¥   r¦   Útupler   r   r   rñ   r   r€   s   @rm   r¨   r¨   Æ   s»   ø„ ð8˜}ð 8¸õ 8ò0	ò6ð& )-ñ_/à—|‘|ð_/ð # 5§<¡<°·±Ð#=Ñ>ð_/ð Ÿ™ tÑ+ð	_/ð
  ™ð_/ð Ð-Ñ.ð_/ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷_/r�   r¨   c                   ó   — e Zd Zy)ÚMiniMaxRotaryEmbeddingNr„   rd   r�   rm   rô   rô   Z  r…   r�   rô   c                   ó   — e Zd Zy)ÚMiniMaxAttentionNr„   rd   r�   rm   rö   rö   ^  r…   r�   rö   c                   ó   — e Zd Zy)ÚMiniMaxTopKRouterNr„   rd   r�   rm   rø   rø   b  r…   r�   rø   c                   ó   — e Zd Zy)ÚMiniMaxSparseMoeBlockNr„   rd   r�   rm   rú   rú   f  r…   r�   rú   c                   óD  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dee   deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚMiniMaxDecoderLayerr©   r�   c                 óÌ  •— t         ‰| �  ||«       || _        t        |d«      r|j                  |   nd | _        |j                  | _        |j                  | _        | `t        |«      | _        | j
                  dk(  r4t        ||«      | _        |j                  | _        |j                  | _        y t!        ||«      | _        |j"                  | _        |j$                  | _        y )NrX   rc   )rg   rŠ   r�   ÚhasattrrX   Ú
block_typer_   r`   Úmlprú   r¨   Ú	self_attnr]   Úattn_alpha_factorr^   Úattn_beta_factorrö   r[   r\   )ri   r©   r�   rl   s      €rm   rŠ   zMiniMaxDecoderLayer.__init__k  sÅ   ø€ Ü‰Ñ˜ Ô+à"ˆŒÜ;BÀ6È=Ô;Y˜&×,Ñ,¨YÒ7Ð_cˆŒØ &× 7Ñ 7ˆÔØ%×5Ñ5ˆÔØˆHÜ(¨Ó0ˆŒØ�?‰?Ð0Ò0Ü6°v¸yÓIˆDŒNØ%+×%DÑ%DˆDÔ"Ø$*×$BÑ$BˆDÕ!ä-¨f°iÓ@ˆDŒNØ%+×%BÑ%BˆDÔ"Ø$*×$@Ñ$@ˆDÕ!r�   Nr3   rÄ   r4   Úposition_idsr)   rK   rj   rÅ   c           
      ó(  — | j                  |«      }|} | j                  d||||||dœ|¤Ž\  }}	|| j                  z  || j                  z  z   }| j	                  |«      }|}| j                  |«      }|| j                  z  || j                  z  z   }|S )N)r3   rÄ   r4   r  r)   rK   rd   )Úinput_layernormr  r  r  Úpost_attention_layernormr   r_   r`   )
ri   r3   rÄ   r4   r  r)   rK   rj   Úresidualr‘   s
             rm   rñ   zMiniMaxDecoderLayer.forward}  sº   € ð ×,Ñ,¨]Ó;ˆØ ˆØ)˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø%Ø+Øñ
ð ñ
Ñˆ�qð ! 4×#9Ñ#9Ñ9¸MÈD×LaÑLaÑ<aÑaˆØ×5Ñ5°mÓDˆØ ˆØŸ™ Ó/ˆØ  4×#8Ñ#8Ñ8¸=È4×K_ÑK_Ñ;_Ñ_ˆàÐr�   )NNNNF)rn   ro   rp   r'   ry   rŠ   r¥   r¦   rò   Ú
LongTensorr   rf   r   r   ÚFloatTensorrñ   r   r€   s   @rm   rü   rü   j  sè   ø„ ðA˜}ð A¸õ Að* IMØ.2Ø04Ø(,Ø!&ñà—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð  ™ðð ˜$‘;ðð Ð-Ñ.ðð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷r�   rü   c                   óD   ‡ — e Zd ZdZ eedd¬«      eeegdœZ	ˆ fd„Z
ˆ xZS )ÚMiniMaxPreTrainedModelFzmlp.gater   )Ú
layer_nameÚindex)Úrouter_logitsr3   Ú
attentionsc                 ó�  •— t         ‰| �  |«       t        |t        «      r¦|j	                  «       }|j                  |«      \  }}}t        j                  |j                  |«       t        j                  |j                  |«       t        j                  |j                  |«       t        j                  |j                  |«       y y r‰   )rg   Ú_init_weightsÚ
isinstancer¨   r¶   r·   ÚinitÚcopy_r¬   r­   r®   r¯   )ri   Úmoduler¬   r­   r®   r¯   rl   s         €rm   r  z$MiniMaxPreTrainedModel._init_weights£  s•   ø€ Ü‰Ñ˜fÔ%Ü�fÔ7Ô8Ø×.Ñ.Ó0ˆJØ5;×5IÑ5IÈ*Ó5UÑ2ˆK˜ NÜ�J‰J�v×(Ñ(¨*Ô5Ü�J‰J�v×)Ñ)¨;Ô7Ü�J‰J�v×'Ñ'¨Ô3Ü�J‰J�v×,Ñ,¨nÕ=ð 9r�   )rn   ro   rp   Ú_can_compile_fullgraphr   rø   rü   rö   r¨   Ú_can_record_outputsr  r   r€   s   @rm   r  r  ›  s5   ø„ Ø"Ðá'Ð(9ÀjÐXYÔZØ,Ø'Ð)BÐCñÐ÷>ð >r�   r  c                   óÖ   — e Zd Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  de
dz  dee   d	eez  fd
„«       «       Zy)ÚMiniMaxModelNr1   r4   r  r)   r2   rK   rj   rÅ   c           
      óâ  — |d u |d uz  rt        d«      ‚|r|€t        «       }n*|r(t        |t        «      st        dt        |«      › d�«      ‚|€| j	                  |«      }|€V|�|j                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }| j                  j                  €t        nt        }	 |	| j                  ||||¬«      }
|}| j                  ||«      }t!        | j"                  «      D ]5  \  }}| j                  j$                  |   dk(  r|
}n|} ||f|||||d	œ|¤Ž}Œ7 | j'                  |«      }t)        ||¬
«      S )Nz:You must specify exactly one of input_ids or inputs_embedszSMiniMax uses cache of its own and is not compatible with `past_key_values` of type ú.r   rM   )Údevice)r©   r2   r4   r)   r  rb   )r4   rÄ   r  r)   rK   )Úlast_hidden_stater)   )Ú
ValueErrorr‡   r  Útyper5   Úget_seq_lengthr¥   r»   rÊ   r  rÑ   r©   rQ   r   r   Ú
rotary_embÚ	enumerater6   rX   r7   r   )ri   r1   r4   r  r)   r2   rK   rj   Úpast_seen_tokensÚmask_functionÚcausal_maskr3   rÄ   rk   Údecoder_layerÚinput_attention_masks                   rm   rñ   zMiniMaxModel.forward¯  s©  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZá˜Ð0Ü*›n‰OÙœz¨/¼<ÔHÜØeÔfjÐkzÓf{Ðe|Ð|}Ð~óð ð Ð Ø ×-Ñ-¨iÓ8ˆMàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÐ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m¸\ÓJÐä )¨$¯+©+Ó 6ò 	ÑˆAˆ}Ø�{‰{×&Ñ& qÑ)Ð-=Ò=Ø'2Ñ$ð (6Ð$á)Øðà3Ø$7Ø)Ø /Ø#ñð ñ‰Mð	ð" Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r�   )NNNNNN)rn   ro   rp   r   r   r¥   r	  r¦   r‡   r
  rf   r   r   rò   r   rñ   rd   r�   rm   r  r  ®  s½   „ ØØð .2Ø.2Ø04Ø/3Ø26Ø!%ñ>
à×#Ñ# dÑ*ð>
ð Ÿ™ tÑ+ð>
ð ×&Ñ&¨Ñ-ð	>
ð
 &¨Ñ,ð>
ð ×(Ñ(¨4Ñ/ð>
ð ˜$‘;ð>
ð Ð+Ñ,ð>
ð 
Ð'Ñ	'ò>
ó ó  ñ>
r�   r  c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚMiniMaxForCausalLMc                 ó"   •— t        ‰| �  di |¤ŽS )a€  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MiniMaxForCausalLM

        >>> model = MiniMaxForCausalLM.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```rd   )rg   rñ   )ri   Úsuper_kwargsrl   s     €rm   rñ   zMiniMaxForCausalLM.forwardó  s   ø€ ô. ‰w‰Ñ. Ñ.Ð.r�   )rn   ro   rp   rñ   r   r€   s   @rm   r*  r*  ò  s   ø„ ÷/ð /r�   r*  c                   ó   — e Zd Zy)Ú MiniMaxForSequenceClassificationNr„   rd   r�   rm   r.  r.    r…   r�   r.  c                   ó   — e Zd Zy)ÚMiniMaxForTokenClassificationNr„   rd   r�   rm   r0  r0    r…   r�   r0  c                   ó   — e Zd Zy)ÚMiniMaxForQuestionAnsweringNr„   rd   r�   rm   r2  r2    r…   r�   r2  )r'   r  r  r*  r.  r0  r2  )Jrq   r¥   Útorch.nn.functionalr   Ú
functionalrÕ   Úhuggingface_hub.dataclassesr   Ú r   r  Úactivationsr   Úcache_utilsr   r	   Úconfiguration_utilsr
   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úgemma2.modeling_gemma2r   Úmixtral.modeling_mixtralr   r   r   r   r   r   r    r!   r"   r#   r$   Ú
get_loggerrn   Úloggerr'   rƒ   r‡   ÚModuler¨   rô   rö   rø   rú   rü   r  r  r*  r.  r0  r2  Ú__all__rd   r�   rm   ú<module>rI     sf  ðñ ã ß Ð Ý .Ý å &Ý !ß .Ý 3ß RÝ BÝ 9Ý 6Ý 1Ý &ß @Ñ @Ý 7ß EÝ :÷÷ ÷ ñ ð 
ˆ×	Ñ	˜HÓ	%€ñ Ð9Ô:Øôb(Ð$ó b(ó ó ;ðb(ôJ	�^ô 	ô"I�<ô "IôJQ/ §	¡	ô Q/ôh	Ð2ô 	ô	Ð'ô 	ô	Ð)ô 	ô	Ð1ô 	ô.Ð-Ð/Iô .ôb>Ð3ô >ô&A
�<ô A
ôH/Ð+ô /ô6	Ð'Gô 	ô	Ð$Aô 	ô	Ð"=ô 	ò�r�   