
    ^j4*                     X   d dl mZ d dlZd dlmc mZ d dlmZ ddlmZ	 ddl
mZ ddlmZ ddlmZmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z.  ej^                  e0      Z1 G d de%      Z2 G d de&      Z3 G d de,      Z4 G d de      Z5 G d de      Z6 G d d e       Z7 G d! d"e      Z8 G d# d$e      Z9 G d% d&e      Z: G d' d(e$      Z; G d) d*e#      Z< G d+ d,ee:      Z= G d- d.ee:      Z>g d/Z?y)0    )CallableN)nn   )initialization)Cache)FlashAttentionKwargs) GenericForSequenceClassificationGenericForTokenClassification)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)logging)is_flash_attention_requested   )DeepseekV2TopkRouter)DeepseekV3AttentionDeepseekV3DecoderLayerDeepseekV3ExpertsDeepseekV3MoEapply_rotary_pos_emb_interleave)LlamaForCausalLM
LlamaModelLlamaRMSNormLlamaRotaryEmbeddingapply_rotary_pos_embeager_attention_forward)get_llama_4_attn_scale)Qwen2MoeMLP   )Mistral4Configc                       e Zd Zy)Mistral4RMSNormN__name__
__module____qualname__     x/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mistral4/modular_mistral4.pyr"   r"   4       r(   r"   c                       e Zd Zy)Mistral4RotaryEmbeddingNr#   r'   r(   r)   r,   r,   8   r*   r(   r,   c                       e Zd Zy)Mistral4MLPNr#   r'   r(   r)   r.   r.   <   r*   r(   r.   c                   $     e Zd Z fdZd Z xZS )Mistral4TopkRouterc                 l    t         |   |       | `|j                  | _        |j
                  | _        y N)super__init__topk_methodnum_local_expertsnum_expertsnorm_topk_prob)selfconfig	__class__s     r)   r4   zMistral4TopkRouter.__init__A   s2     !33$33r(   c                    |j                  d| j                        }t        j                  || j                        }|j                  d      }|j                  d| j                  | j                  | j                  z        j                  dd      d   j                  d      }t        j                  || j                  dd      d   }t        j                  |      }|j                  d|d       |j                  d      j                  d| j                  | j                  | j                  z        j!                  d| j                        }|j#                  |j%                          d      }t        j                  || j&                  dd      d   }	|j)                  d|	      }
| j*                  r|
j                  dd	
      dz   }|
|z  }
|
| j,                  z  }
||
|	fS )Nr   dimr   F)kr?   sortedr           T)r?   keepdimg#B;)view
hidden_dimFlinearweightsoftmax	num_groupr7   topksumtorch
topk_group
zeros_likescatter_	unsqueezeexpandreshapemasked_fillbooltop_kgatherr8   routed_scaling_factor)r9   hidden_statesrouter_logitsscoresgroup_scores	group_idx
group_mask
score_maskscores_for_choicetopk_indicestopk_weightsdenominators               r)   forwardzMistral4TopkRouter.forwardG   s   %**2t?<&&r*KKDNND,<,<,NOTTUV\^T_`abffkmfn 	 JJ|tBuUVWX	%%l3
Ay!,  $VB(8(8DNN(JKWR))* 	
 #..
0A/A3Gzz"3tzzrRWXYZ[}}Q5&**r4*@5HKK'L#d&@&@@lL88r(   )r$   r%   r&   r4   rd   __classcell__r;   s   @r)   r0   r0   @   s    49r(   r0   c                       e Zd Zy)Mistral4ExpertsNr#   r'   r(   r)   rh   rh   `   r*   r(   rh   c                       e Zd Zy)Mistral4MoENr#   r'   r(   r)   rj   rj   d   r*   r(   rj   c                      e Zd ZdedefdZ	 ddej                  deej                  ej                  f   dej                  dz  dej                  d	e	dz  d
e
e   deej                  ej                  dz  eej                     dz  f   fdZy)Mistral4Attentionr:   	layer_idxc                    t         j                  j                  |        || _        || _        |j
                  |j                  z  | _        |j                  | _        |j
                  | _	        |j                  | _
        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        d| _        | j                  ?t        j"                  |j$                  | j                  | j                  z  d      | _        nt        j"                  |j$                  |j                  |j(                        | _        t-        |j                        | _        t        j"                  |j                  | j                  | j                  z  d      | _        t        j"                  |j$                  | j                  | j                  z   |j(                        | _        t-        | j                        | _        t        j"                  | j                  | j                  | j                  | j                  z   z  d      | _        t        j"                  | j                  | j                  z  |j$                  |j(                        | _        | j                  dz  | _        y )NTF)biasg      )r   Moduler4   r:   rm   num_attention_headsnum_key_value_headsnum_key_value_groupsattention_dropout	num_headsq_lora_rankqk_rope_head_dimkv_lora_rank
v_head_dimqk_nope_head_dimqk_head_dim	is_causalLinearhidden_sizeq_projattention_biasq_a_projr"   q_a_layernormq_b_projkv_a_proj_with_mqakv_a_layernorm	kv_b_projo_projscalingr9   r:   rm   s      r)   r4   zMistral4Attention.__init__i   s   
		4 "$*$>$>&B\B\$\!!'!9!933!-- & 7 7"// ++ & 7 7!--#))F$6$6IYIY8Y`efDKIIf&8&8&:L:LSYShShiDM!01C1C!DDIIf&8&8$..4K[K[:[bghDM"$)) 5 55&&#

 .d.?.?@NNd33dooEF
 iiNNT__,&&
 ''D1r(   NrY   position_embeddingsattention_maskposition_idspast_key_valueskwargsreturnc                     |j                   d d \  }}||d| j                  f}	||d| j                  | j                  z   f}
| j                  | j                  |      }n/| j                  | j                  | j                  |                  }|j                  |	      j                  dd      }t        j                  || j                  | j                  gd      \  }}| j                  |      }t        j                  || j                  | j                  gd      \  }}| j!                  | j#                  |            j                  |
      j                  dd      }t        j                  || j                  | j                  gd      \  }}|j                  |d|| j                        }|\  }}| j$                  j&                  rt)        ||||      \  }}nt+        ||||      \  }} |j,                  g |j                   d d d }t        j.                  ||fd      }t        j.                  ||fd      }|t1        || j$                  j2                  j5                  d      | j$                  j2                  j5                  d            j7                  |j8                        z  }| |j;                  ||| j<                        \  }}t?        | j$                        rH| j                  | j                  k7  r/tA        jB                  |d| j                  | j                  z
  g      }tE        jF                  | j$                  jH                  tJ              } || ||||f| jL                  sdn| jN                  | jP                  d	|\  }}t?        | j$                        r4| j                  | j                  k7  r|d d d d d d d | j                  f   }|jS                  ||d      jU                         }| jW                  |      }||fS )
Nr=   r   r   r>   llama_4_scaling_beta original_max_position_embeddingsr   rB   )dropoutr   ),shaper{   rz   ry   rv   r   r   r   r   rD   	transposerM   splitrw   r   rx   r   r   r:   rope_interleaver   r   rR   catr   rope_parametersgettodtypeupdaterm   r   rF   padr   get_interface_attn_implementationr   trainingrt   r   rS   
contiguousr   )r9   rY   r   r   r   r   r   
batch_size
seq_lengthquery_shape	key_shapeq_statesq_passq_rotcompressed_kvk_passk_rotvalue_statescossinquery_states
key_statesattention_interfaceattn_outputattn_weightss                            r)   rd   zMistral4Attention.forward   s    "/!4!4Sb!9
J!:r43C3CDR1F1F1XY	#{{=1H}}T%7%7m8T%UVH==-771=Ht/D/DdF[F[.\bde//>MD4E4EtG\G\3]cef 3 3F ;<AA)LVVWXZ[\${{6D4I4I4??3[acd

:q*d6K6KL&S;;&&:5%cRLE5/uc3GLE54fll3B/44yy&%b9YYB7
#&<KK''++,BCKK''++,NO'
 "\
 	! &'6'='=j,X\XfXf'g$J'49I9IT__9\5543C3Cdoo3U/VWL(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ (49I9IT__9\%aA/@/@&@AK!))*j"EPPRkk+.L((r(   r2   )r$   r%   r&   r    intr4   rM   Tensortupler   r   r   rd   r'   r(   r)   rl   rl   h   s    )2~ )2# )2b )-F)||F) #5<<#=>F) t+	F)
 llF) F) -.F) 
u||U\\D0%2E2LL	MF)r(   rl   c                       e Zd ZdedefdZy)Mistral4DecoderLayerr:   rm   c                    t         j                  j                  |        |j                  | _        t	        ||      | _        ||j                  k\  rt        |      | _        nt        |      | _        t        |j                  |j                        | _        t        |j                  |j                        | _        y )N)r:   rm   )eps)r   rp   r4   r~   rl   	self_attnfirst_k_dense_replacerj   mlpr.   r"   rms_norm_epsinput_layernormpost_attention_layernormr   s      r)   r4   zMistral4DecoderLayer.__init__   s    
		4 !--*&IN444"6*DH"6*DH.v/A/AvGZGZ[(78J8JPVPcPc(d%r(   N)r$   r%   r&   r    r   r4   r'   r(   r)   r   r      s    e~ e# er(   r   c                        e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedZg Zg Z ej(                          fd       Z xZS )Mistral4PreTrainedModelr:   modelTr   r   )rY   
attentionsc                    t         |   |       t        |t              r7t	        j
                  |j                  d| j                  j                         y t        |t              rmt	        j
                  |j                  d| j                  j                         t	        j
                  |j                  d| j                  j                         y y )NrB   )meanstd)r3   _init_weights
isinstancer0   initnormal_rH   r:   initializer_rangerh   gate_up_proj	down_proj)r9   moduler;   s     r)   r   z%Mistral4PreTrainedModel._init_weights   s    f%f01LLSdkk6S6ST0LL,,3DKK<Y<YZLL))9V9VW 1r(   )r$   r%   r&   r    __annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendr   rl   _can_record_outputs_keep_in_fp32_modules_strict"_keys_to_ignore_on_load_unexpectedrM   no_gradr   re   rf   s   @r)   r   r      s|    &*#/0#4"5N!"&-' $& )+&U]]_X Xr(   r   c                       e Zd Zy)Mistral4ModelNr#   r'   r(   r)   r   r   
  r*   r(   r   c                       e Zd Zy)Mistral4ForCausalLMNr#   r'   r(   r)   r   r     r*   r(   r   c                       e Zd Zy)!Mistral4ForSequenceClassificationNr#   r'   r(   r)   r   r     r*   r(   r   c                       e Zd Zy)Mistral4ForTokenClassificationNr#   r'   r(   r)   r   r     r*   r(   r   )r   r   r   r   r   )@collections.abcr   rM   torch.nn.functionalr   
functionalrF    r   r   cache_utilsr   modeling_flash_attention_utilsr   modeling_layersr	   r
   modeling_utilsr   r   processing_utilsr   utilsr   utils.genericr    deepseek_v2.modeling_deepseek_v2r    deepseek_v3.modeling_deepseek_v3r   r   r   r   r   llama.modeling_llamar   r   r   r   r   r   ministral3.modeling_ministral3r   qwen2_moe.modeling_qwen2_moer   configuration_mistral4r    
get_loggerr$   loggerr"   r,   r.   r0   rh   rj   rl   r   r   r   r   r   r   __all__r'   r(   r)   <module>r      s   %     &   B ^ F &  9 C   D 6 2 
		H	%	l 		2 		+ 	9- 9@	' 		- 	r)+ r)je1 e Xo X:	J 		* 		(HJa 		%BD[ 	r(   