
    ^jW                        d Z ddlZddlmc mZ ddlmZ ddlmZ ddlm	Z
 ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5  e"jl                  e7      Z8 e!d      e G d de                    Z9 G d de3      Z: G d de      Z; G d dejx                        Z= G d  d!e)      Z> G d" d#e+      Z? G d$ d%e5      Z@ G d& d'e4      ZA G d( d)e,e      ZB G d* d+e2      ZC G d, d-e1      ZD G d. d/e-      ZE G d0 d1e/      ZF G d2 d3e0      ZG G d4 d5e.      ZHg d6ZIy)7zPyTorch MiniMax model.    N)strict)nn   )initialization)ACT2FN)CacheDynamicCache)PreTrainedConfig)create_causal_mask!create_sliding_window_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)MoeModelOutputWithPast)RopeParameters)Unpack)TransformersKwargsauto_docstringlogging)merge_with_config_defaults)OutputRecordercapture_outputs   )Gemma2RotaryEmbedding)MixtralAttentionMixtralDecoderLayerMixtralForCausalLMMixtralForQuestionAnswering MixtralForSequenceClassificationMixtralForTokenClassificationMixtralModelMixtralPreTrainedModelMixtralRMSNormMixtralSparseMoeBlockMixtralTopKRouterzMiniMaxAI/MiniMax-Text-01-hf)
checkpointc                       e Zd ZU dZdZdgZdZdddddddd	Zd
gdgfddgdgfdgdgfdZdddddZ	ddiZ
dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed <   d!Zeed"<   d#Zeed$<   d%Zeed&<   d'Zeed(<   d)Zeed*<   dZedz  ed+<   d,Zedz  ed-<   d.Zeee   z  dz  ed/<   d0Z eed1<   dZ!edz  ed2<   d3Z"eez  ed4<   d.Z#eed5<   dZ$eed<   d0Z%eed6<   d7Z&eed8<   d3Z'eed9<   dZ(e)e*z  dz  ed:<   dZ+ee   dz  ed;<   d<Z,eed=<   d,Z-eez  ed><   d,Z.eez  ed?<   d,Z/eez  ed@<   d,Z0eez  edA<   d,Z1eez  edB<   d,Z2eez  edC<    fdDZ3 xZ4S )EMiniMaxConfiga  
    block_size (`int`, *optional*, defaults to 256):
        The length of each attention block, determining how queries, keys, and values
        are grouped and processed for intra- and inter-block attention.
    full_attn_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after normal attention.
    full_attn_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after normal attention.
    linear_attn_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after lightning attention.
    linear_attn_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after lightning attention.
    mlp_alpha_factor (`float`, *optional*, defaults to 1):
        Weight for residual value in residual connection after MLP.
    mlp_beta_factor (`float`, *optional*, defaults to 1):
        Weight for hidden state value in residual connection after MLP.

    ```python
    >>> from transformers import MiniMaxModel, MiniMaxConfig

    >>> # Initializing a MiniMax style configuration
    >>> configuration = MiniMaxConfig()

    >>> # Initializing a model from the MiniMax style configuration
    >>> model = MiniMaxModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```minimaxpast_key_valuesg    .Acolwiserowwisepacked_colwisemoe_tp_experts)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_proj!layers.*.mlp.experts.gate_up_projlayers.*.mlp.experts.down_projlayers.*.mlp.experts	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnorm	ep_routergrouped_gemm)zlayers.*.mlp.gater.   r/   r0   num_expertsnum_local_expertsi }  
vocab_sizei   hidden_sizei 8  intermediate_size    num_hidden_layersnum_attention_heads   num_key_value_headsNhead_dimsilu
hidden_acti   max_position_embeddingsg{Gz?initializer_rangeh㈵>rms_norm_epsT	use_cachepad_token_id   bos_token_idr   eos_token_idFtie_word_embeddingssliding_windowg        attention_dropoutnum_experts_per_tokoutput_router_logitsgMbP?router_aux_loss_coefrouter_jitter_noiserope_parameterslayer_types   
block_sizefull_attn_alpha_factorfull_attn_beta_factorlinear_attn_alpha_factorlinear_attn_beta_factormlp_alpha_factormlp_beta_factorc                     | j                   | j                  | _         | j                  ;t        | j                        D cg c]  }t        |dz   dz        rdnd c}| _        t        |   di | y c c}w )NrM   r   full_attentionlinear_attention )rC   rA   rX   ranger@   boolsuper__post_init__)selfkwargsi	__class__s      v/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/minimax/modular_minimax.pyrh   zMiniMaxConfig.__post_init__   sz    ##+'+'?'?D$#W\]a]s]sWt RSD!a%1$5 ;MM D 	''	 s   A6)5__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencedefault_thetabase_model_tp_planbase_model_pp_planbase_model_ep_planattribute_mapr<   int__annotations__r=   r>   r@   rA   rC   rD   rF   strrG   rH   floatrJ   rK   rf   rL   rN   rO   listrP   rQ   rR   rS   r;   rT   rU   rV   rW   r   dictrX   rZ   r[   r\   r]   r^   r_   r`   rh   __classcell__rl   s   @rm   r'   r'   6   s;   < J#4"5M%.%.%.%.-=*3 0 &(9:#%568IJ!"_$56 )-;*8 0	 #$78MJK"s"s!!  HcDjJ#,S,#u#L%It#L#*# L#* +,L#S	/D(, %%!%NC$J%%(us{(  s!&$&"'%'!$$48O^d*T18$(KcT!(J*+C%K+)*3;*,-cEk-+,S5[,$%cEk%#$OS5[$	( 	(    r'   c                       e Zd Zy)MiniMaxRMSNormNrn   ro   rp   rd   r   rm   r   r          r   r   c                   r     e Zd Z fdZd ZdefdZ fdZdefdZde	j                  fd	Zd
efdZ xZS )MiniMaxCachec                 0    t         |           g | _        y N)rg   __init__linear_cacheri   rl   s    rm   r   zMiniMaxCache.__init__   s    02r   c                     t        t        | j                        |dz         D ]  }| j                  j                  g         || j                  |<   y )NrM   )re   lenr   append)ri   	layer_idxr   _s       rm   set_linear_cachezMiniMaxCache.set_linear_cache   sK    s4,,-y1}= 	)A$$R(	)'3)$r   r   c                 >    |t        |       k  r| j                  |   S y r   )r   r   )ri   r   s     rm   get_linear_cachezMiniMaxCache.get_linear_cache   s"    s4y $$Y//r   c                 Z    t        t        | 	         t        | j                              S r   )maxrg   __len__r   r   r   s    rm   r   zMiniMaxCache.__len__   s"    57?$c$*;*;&<==r   repeatsc                     t        t        |             D ]`  }| j                  |   g k7  r.| j                  |   j                  |d      | j                  |<   C| j                  |   j                  |       b y )Nr   dim)re   r   r   repeat_interleaver6   batch_repeat_interleave)ri   r   r   s      rm   r   z$MiniMaxCache.batch_repeat_interleave   ss    s4y) 	HI  +r1/3/@/@/K/]/]^ekl/]/m!!),I&>>wG		Hr   indicesc                     t        t        |             D ]T  }| j                  |   g k7  r"| j                  |   |df   | j                  |<   7| j                  |   j	                  |       V y )N.)re   r   r   r6   batch_select_indices)ri   r   r   s      rm   r   z!MiniMaxCache.batch_select_indices   sk    s4y) 	EI  +r1/3/@/@/KGUXL/Y!!),I&;;GD		Er   
max_lengthc                     t        d      )Nz*MiniMaxCache doesnot support `crop` method)RuntimeError)ri   r   s     rm   cropzMiniMaxCache.crop   s    GHHr   )rn   ro   rp   r   r   ry   r   r   r   torchTensorr   r   r   r   s   @rm   r   r      sL    34# 
>Hs HEELL EIs Ir   r   c                       e Zd Zdedef fdZd Zd Z	 ddej                  de
ej                  ej                  f   d	ej                  dz  d
edz  dee   de
ej                  ej                  dz  e
ej                     dz  f   fdZ xZS )MiniMaxLightningAttentionconfigr   c                    t         |           || _        t        |dd       xs |j                  |j
                  z  | _        |j
                  | _        |j                  | _        |j                  | _        t        |j                     | _        t        | j                  | j
                  z        | _        t        j                  |j                  | j
                  | j                  z  dz  d      | _        t        j                  | j
                  | j                  z  |j                  d      | _        t        j                  |j                  | j
                  | j                  z  d      | _        | j'                         }| j)                  |      \  }}}| j+                  d|       | j+                  d|       | j+                  d|       | j+                  d|       |j,                  |   | _        y )	NrD   r   F)bias
slope_ratequery_decay	key_decaydiagonal_decay)rg   r   r   getattrr=   rA   rD   r@   rZ   r   rF   act_fnr   r7   r   Linearqkv_projout_projoutput_gateget_slope_ratedecay_factorsregister_bufferrX   
layer_type)ri   r   r   r   r   r   r   rl   s          rm   r   z"MiniMaxLightningAttention.__init__   s   "
D9mV=O=OSYSmSm=m#)#=#= !'!9!9 ++V../"4==43K3K#KL			&"4"4d6N6NQUQ^Q^6^ab6bino		$":":T]]"JFL^L^ejk99V%7%79Q9QTXTaTa9ahmn((*
151C1CJ1O.Y\:6]K8[)4-~> ,,Y7r   c                     ddd| j                   z  z  z  }t        j                  | j                         dz   }d| j                  | j                  dz
  dz   z  z
  dz   }||z  }||z  }|d d d d f   }|S )NrM   r   rB   rI   )rA   r   aranger   r@   )ri   baseexponentfactorrates        rm   r   z(MiniMaxLightningAttention.get_slope_rate   s    A!d66678<< 8 89A=T^^t'='='AD'HIIDPX~f}AtTM"r   c                    t        j                  | j                        dz   }t        j                  | |d d d f   z        }t        j                  | | j                  |d d d f   z
  z        }|d d d f   |d d d f   z
  }|d d d d d d f   }||z  }t        j                  |dk\  | t        d            }t        j                  |      }|||fS )NrM   r   z-inf)r   r   rZ   expwherer|   )ri   r   block_size_ranger   r   r   s         rm   r   z'MiniMaxLightningAttention.decay_factors   s     <<81<ii.>q$w.G GHIIzkT__?OPQSWPW?X-XYZ	)!T'25EdAg5NN'dAq(89#n4^q%8>/5QW=Y>2I~55r   Nr3   position_embeddingsr4   r)   rj   returnc                    |j                   \  }}}|| j                  z   dz
  | j                  z  }	| j                  | j                  |            }
|
j	                  ||| j
                  d| j                  z        }
t        j                  |
| j                  d      \  }}}|j                  dd      }|j                  dd      }|j                  dd      }d }||j                  | j                        }|t        j                  || j
                  | j                  | j                        j                  |      }|Q|j                  t        j                        }|j                  |j!                  d      j!                  d       d      }g }t#        |	      D ]b  }|| j                  z  }t%        || j                  z   |      }||z
  }|d d d d ||f   }|d d d d ||f   }|d d d d ||f   }| j&                  d d d |f   }| j(                  d d | d f   }| j*                  d d d d d |d |f   }t        j,                  | j.                   |z        }t        j0                  ||j                  dd            }t        j0                  ||z  |      }t        j0                  ||z  |      }||z   }|j3                  |       t        j0                  ||z  j                  dd      |      }||z  |z   }e nt        j,                  | j.                         } g }t#        |      D ]  }|d d d d ||dz   f   }|d d d d ||dz   f   }|d d d d ||dz   f   }t        j0                  |j                  dd      |      }!| |z  |!z   }t        j0                  ||      }|j3                  |        t        j4                  |d      }|j                  dd      }|j	                  ||| j
                  | j                  z        }| j7                  |      }t9        j:                  | j=                  |            |z  }| j?                  |      }||jA                  | j                  |       ||fS )	NrM   r   r   r   )dtyper   )!shaperZ   r   r   reshaperA   rD   r   split	transposer   r   zerostorf   masked_fill	unsqueezere   minr   r   r   r   r   matmulr   catr7   Fsigmoidr   r   r   )"ri   r3   r   r4   r)   rj   
batch_sizeseq_lenr=   
num_blocks
qkv_statesquery_states
key_statesvalue_statesattn_weights_interattn_outputrk   	start_idxend_idxcurrent_block_sizecurrent_query_statescurrent_key_statescurrent_value_statescurrent_query_decaycurrent_key_decaycurrent_diagonal_decayblock_decayattn_weights_intraattn_output_intraattn_output_intercurrent_attn_outputnext_attn_weights_interratiocurrent_attn_weights_inters"                                     rm   forwardz!MiniMaxLightningAttention.forward   s    ,9+>+>(
G[/!3G
[[}!=>
''
GT=U=UWX[_[h[hWhi
16Z\]1^.j,#--a3))!Q/
#--a3 "&!0!A!A$..!Q%!&Z9Q9QSWS`S`bfbobo!p!s!s"
 )!/!2!2!2!D+779Q9QRS9T9^9^_a9b8bdefK:& `/	i$//97C%,y%8"'3Aq)G:K4K'L$%/1i6G0G%H"'3Aq)G:K4K'L$&*&6&6q:M;M:M7M&N#$(NN17I6I6J3J$K!)-)<)<QCVDVCVXkYkXk=k)l&#ii(8;M(MN &+\\2FHZHdHdegikHl%m"$)LL1CF\1\^r$s! %*LL1EH[1[]o$p! '8:K&K#""#67 +0,,'*;;FFr2NPd+' &8+%EH_%_";`@ IIt./EK7^ 	8'3Aq!a!e)O'D$%/1a!a%i%@"'3Aq!a!e)O'D$-2\\:L:V:VWY[]:^`t-u*%*-?%?B\%\"&+ll3GI[&\#""#67	8 ii4 "++Aq1!))*gt?W?WZ^ZgZg?ghii,ii 0 0 ?@;NmmK0 &,,T^^=OP...r   r   )rn   ro   rp   r'   ry   r   r   r   r   r   tupler   r   r   r   r   r   s   @rm   r   r      s    8} 8 80	6& )-_/||_/ #5<<#=>_/ t+	_/
 _/ -._/ 
u||U\\D0%2E2LL	M_/r   r   c                       e Zd Zy)MiniMaxRotaryEmbeddingNr   rd   r   rm   r   r   Z  r   r   r   c                       e Zd Zy)MiniMaxAttentionNr   rd   r   rm   r   r   ^  r   r   r   c                       e Zd Zy)MiniMaxTopKRouterNr   rd   r   rm   r   r   b  r   r   r   c                       e Zd Zy)MiniMaxSparseMoeBlockNr   rd   r   rm   r   r   f  r   r   r   c                   D    e Zd Zdedef fdZ	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dee   deej                  eej                  ej                  f   dz  f   fdZ xZS )MiniMaxDecoderLayerr   r   c                    t         |   ||       || _        t        |d      r|j                  |   nd | _        |j                  | _        |j                  | _        | `t        |      | _        | j
                  dk(  r4t        ||      | _        |j                  | _        |j                  | _        y t!        ||      | _        |j"                  | _        |j$                  | _        y )NrX   rc   )rg   r   r   hasattrrX   
block_typer_   r`   mlpr   r   	self_attnr]   attn_alpha_factorr^   attn_beta_factorr   r[   r\   )ri   r   r   rl   s      rm   r   zMiniMaxDecoderLayer.__init__k  s    +";B6=;Y&,,Y7_c & 7 7%55H(0??006vyIDN%+%D%DD"$*$B$BD!-fi@DN%+%B%BD"$*$@$@D!r   Nr3   r   r4   position_idsr)   rK   rj   r   c           
      (   | j                  |      }|} | j                  d||||||d|\  }}	|| j                  z  || j                  z  z   }| j	                  |      }|}| j                  |      }|| j                  z  || j                  z  z   }|S )N)r3   r   r4   r  r)   rK   rd   )input_layernormr  r  r  post_attention_layernormr   r_   r`   )
ri   r3   r   r4   r  r)   rK   rj   residualr   s
             rm   r   zMiniMaxDecoderLayer.forward}  s     ,,]; )4>> 
' 3)%+
 
q !4#9#99MDLaLa<aa55mD / 4#8#88=4K_K_;__r   )NNNNF)rn   ro   rp   r'   ry   r   r   r   r   
LongTensorr   rf   r   r   FloatTensorr   r   r   s   @rm   r   r   j  s    A} A A* IM.204(,!&|| #5<<#=>E t+	
 &&-  $; -. 
u  %(9(95;L;L(L"MPT"TT	Ur   r   c                   D     e Zd ZdZ eedd      eeegdZ	 fdZ
 xZS )MiniMaxPreTrainedModelFzmlp.gater   )
layer_nameindex)router_logitsr3   
attentionsc                    t         |   |       t        |t              r|j	                         }|j                  |      \  }}}t        j                  |j                  |       t        j                  |j                  |       t        j                  |j                  |       t        j                  |j                  |       y y r   )rg   _init_weights
isinstancer   r   r   initcopy_r   r   r   r   )ri   moduler   r   r   r   rl   s         rm   r  z$MiniMaxPreTrainedModel._init_weights  s    f%f78..0J5;5I5I*5U2KNJJv((*5JJv));7JJv''3JJv,,n= 9r   )rn   ro   rp   _can_compile_fullgraphr   r   r   r   r   _can_record_outputsr  r   r   s   @rm   r  r    s5    "'(9jXYZ,')BC> >r   r  c                       e Zd Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  de
dz  dee   d	eez  fd
              Zy)MiniMaxModelNr1   r4   r  r)   r2   rK   rj   r   c           
         |d u |d uz  rt        d      |r|t               }n*|r(t        |t              st        dt        |       d      || j	                  |      }|V||j                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }| j                  j                  t        nt        }	 |	| j                  ||||      }
|}| j                  ||      }t!        | j"                        D ]5  \  }}| j                  j$                  |   dk(  r|
}n|} ||f|||||d	|}7 | j'                  |      }t)        ||
      S )Nz:You must specify exactly one of input_ids or inputs_embedszSMiniMax uses cache of its own and is not compatible with `past_key_values` of type .r   rM   )device)r   r2   r4   r)   r  rb   )r4   r   r  r)   rK   )last_hidden_stater)   )
ValueErrorr   r  typer5   get_seq_lengthr   r   r   r  r   r   rQ   r   r   
rotary_emb	enumerater6   rX   r7   r   )ri   r1   r4   r  r)   r2   rK   rj   past_seen_tokensmask_functioncausal_maskr3   r   rk   decoder_layerinput_attention_masks                   rm   r   zMiniMaxModel.forward  s    -t";<YZZ0*nOz/<Hefjkzf{e||}~    --i8MCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L.2kk.H.H.P*Vw#;;')+%
 &"oom\J )$++ 6 	A}{{&&q)-=='2$ (6$)3$7) /# M	" 		-0%++
 	
r   )NNNNNN)rn   ro   rp   r   r   r   r	  r   r   r
  rf   r   r   r   r   r   rd   r   rm   r  r    s     .2.204/326!%>
##d*>
 t+>
 &&-	>

 &,>
 ((4/>
 $;>
 +,>
 
'	'>
   >
r   r  c                        e Zd Z fdZ xZS )MiniMaxForCausalLMc                 "    t        |   di |S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MiniMaxForCausalLM

        >>> model = MiniMaxForCausalLM.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("MiniMaxAI/MiniMax-Text-01-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```rd   )rg   r   )ri   super_kwargsrl   s     rm   r   zMiniMaxForCausalLM.forward  s    . w...r   )rn   ro   rp   r   r   r   s   @rm   r*  r*    s    / /r   r*  c                       e Zd Zy) MiniMaxForSequenceClassificationNr   rd   r   rm   r.  r.    r   r   r.  c                       e Zd Zy)MiniMaxForTokenClassificationNr   rd   r   rm   r0  r0    r   r   r0  c                       e Zd Zy)MiniMaxForQuestionAnsweringNr   rd   r   rm   r2  r2    r   r   r2  )r'   r  r  r*  r.  r0  r2  )Jrq   r   torch.nn.functionalr   
functionalr   huggingface_hub.dataclassesr    r   r  activationsr   cache_utilsr   r	   configuration_utilsr
   masking_utilsr   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   modeling_rope_utilsr   processing_utilsr   utilsr   r   r   utils.genericr   utils.output_capturingr   r   gemma2.modeling_gemma2r   mixtral.modeling_mixtralr   r   r   r   r   r   r    r!   r"   r#   r$   
get_loggerrn   loggerr'   r   r   Moduler   r   r   r   r   r   r  r  r*  r.  r0  r2  __all__rd   r   rm   <module>rI     sf       .  & ! . 3 R B 9 6 1 & @ @ 7 E :    
		H	% 9:b($ b(  ;b(J	^ 	"I< "IJQ/		 Q/h	2 		' 		) 		1 	.-/I .b>3 >&A
< A
H/+ /6	'G 		$A 		"= 	r   