Ë
    ÜÓ^j4*  ã                   óX  — d dl mZ d dlZd dlmc mZ d dlmZ ddlmZ	 ddl
mZ ddlmZ ddlmZmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z.  ej^                  e0«      Z1 G d„ de%«      Z2 G d„ de&«      Z3 G d„ de,«      Z4 G d„ de«      Z5 G d„ de«      Z6 G d„ d e «      Z7 G d!„ d"e«      Z8 G d#„ d$e«      Z9 G d%„ d&e«      Z: G d'„ d(e$«      Z; G d)„ d*e#«      Z< G d+„ d,ee:«      Z= G d-„ d.ee:«      Z>g d/¢Z?y)0é    )ÚCallableN)Únné   )Úinitialization)ÚCache)ÚFlashAttentionKwargs)Ú GenericForSequenceClassificationÚGenericForTokenClassification)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úlogging)Úis_flash_attention_requestedé   )ÚDeepseekV2TopkRouter)ÚDeepseekV3AttentionÚDeepseekV3DecoderLayerÚDeepseekV3ExpertsÚDeepseekV3MoEÚapply_rotary_pos_emb_interleave)ÚLlamaForCausalLMÚ
LlamaModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)Úget_llama_4_attn_scale)ÚQwen2MoeMLPé   )ÚMistral4Configc                   ó   — e Zd Zy)ÚMistral4RMSNormN©Ú__name__Ú
__module__Ú__qualname__© ó    úx/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mistral4/modular_mistral4.pyr"   r"   4   ó   „ Ør(   r"   c                   ó   — e Zd Zy)ÚMistral4RotaryEmbeddingNr#   r'   r(   r)   r,   r,   8   r*   r(   r,   c                   ó   — e Zd Zy)ÚMistral4MLPNr#   r'   r(   r)   r.   r.   <   r*   r(   r.   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚMistral4TopkRouterc                 ól   •— t         ‰| �  |«       | `|j                  | _        |j
                  | _        y ©N)ÚsuperÚ__init__Útopk_methodÚnum_local_expertsÚnum_expertsÚnorm_topk_prob)ÚselfÚconfigÚ	__class__s     €r)   r4   zMistral4TopkRouter.__init__A   s2   ø€ Ü‰Ñ˜Ô ØÐØ!×3Ñ3ˆÔØ$×3Ñ3ˆÕr(   c                 óÊ  — |j                  d| j                  «      }t        j                  || j                  «      }|j                  d«      }|j                  d| j                  | j                  | j                  z  «      j                  dd¬«      d   j                  d¬«      }t        j                  || j                  dd¬«      d   }t        j                  |«      }|j                  d|d«       |j                  d«      j                  d| j                  | j                  | j                  z  «      j!                  d| j                  «      }|j#                  |j%                  «        d«      }t        j                  || j&                  dd¬«      d   }	|j)                  d|	«      }
| j*                  r|
j                  dd	¬
«      dz   }|
|z  }
|
| j,                  z  }
||
|	fS )Néÿÿÿÿr   ©Údimr   F)Úkr?   Úsortedr   ç        T)r?   Úkeepdimg#B’¡œÇ;)ÚviewÚ
hidden_dimÚFÚlinearÚweightÚsoftmaxÚ	num_groupr7   ÚtopkÚsumÚtorchÚ
topk_groupÚ
zeros_likeÚscatter_Ú	unsqueezeÚexpandÚreshapeÚmasked_fillÚboolÚtop_kÚgatherr8   Úrouted_scaling_factor)r9   Úhidden_statesÚrouter_logitsÚscoresÚgroup_scoresÚ	group_idxÚ
group_maskÚ
score_maskÚscores_for_choiceÚtopk_indicesÚtopk_weightsÚdenominators               r)   ÚforwardzMistral4TopkRouter.forwardG   s¬  € Ø%×*Ñ*¨2¨t¯©Ó?ˆÜŸ™ °·±Ó<ˆØ×&Ñ& rÓ*ˆà�K‰K˜˜DŸN™N¨D×,<Ñ,<ÀÇÁÑ,NÓO×TÑTÐUVÐ\^ÐTÓ_Ð`aÑb×fÑfÐkmÐfÓnð 	ô —J‘J˜|¨t¯©ÀBÈuÔUÐVWÑXˆ	Ü×%Ñ% lÓ3ˆ
Ø×Ñ˜A˜y¨!Ô,à× Ñ  Ó$ß‰V�B˜Ÿ™¨×(8Ñ(8¸D¿N¹NÑ(JÓKß‰W�R˜×)Ñ)Ó*ð 	ð
 #×.Ñ.°
·±Ó0AÐ/AÀ3ÓGÐÜ—z‘zÐ"3°t·z±zÀrÐRWÔXÐYZÑ[ˆØ—}‘} Q¨Ó5ˆØ×ÒØ&×*Ñ*¨r¸4Ð*Ó@À5ÑHˆKØ˜KÑ'ˆLØ# d×&@Ñ&@Ñ@ˆØ˜l¨LÐ8Ð8r(   )r$   r%   r&   r4   rd   Ú__classcell__©r;   s   @r)   r0   r0   @   s   ø„ ô4ö9r(   r0   c                   ó   — e Zd Zy)ÚMistral4ExpertsNr#   r'   r(   r)   rh   rh   `   r*   r(   rh   c                   ó   — e Zd Zy)ÚMistral4MoENr#   r'   r(   r)   rj   rj   d   r*   r(   rj   c                   ó  — e Zd Zdedefd„Z	 ddej                  deej                  ej                  f   dej                  dz  dej                  d	e	dz  d
e
e   deej                  ej                  dz  eej                     dz  f   fd„Zy)ÚMistral4Attentionr:   Ú	layer_idxc                 ó†  — t         j                  j                  | «       || _        || _        |j
                  |j                  z  | _        |j                  | _        |j
                  | _	        |j                  | _
        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        d| _        | j                  €?t        j"                  |j$                  | j                  | j                  z  d¬«      | _        n“t        j"                  |j$                  |j                  |j(                  ¬«      | _        t-        |j                  «      | _        t        j"                  |j                  | j                  | j                  z  d¬«      | _        t        j"                  |j$                  | j                  | j                  z   |j(                  ¬«      | _        t-        | j                  «      | _        t        j"                  | j                  | j                  | j                  | j                  z   z  d¬«      | _        t        j"                  | j                  | j                  z  |j$                  |j(                  ¬«      | _        | j                  dz  | _        y )NTF)Úbiasg      à¿)r   ÚModuler4   r:   rm   Únum_attention_headsÚnum_key_value_headsÚnum_key_value_groupsÚattention_dropoutÚ	num_headsÚq_lora_rankÚqk_rope_head_dimÚkv_lora_rankÚ
v_head_dimÚqk_nope_head_dimÚqk_head_dimÚ	is_causalÚLinearÚhidden_sizeÚq_projÚattention_biasÚq_a_projr"   Úq_a_layernormÚq_b_projÚkv_a_proj_with_mqaÚkv_a_layernormÚ	kv_b_projÚo_projÚscaling©r9   r:   rm   s      r)   r4   zMistral4Attention.__init__i   s  € Ü
�	‰	×Ñ˜4Ô ØˆŒØ"ˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø!'×!9Ñ!9ˆÔØ×3Ñ3ˆŒà!×-Ñ-ˆÔØ &× 7Ñ 7ˆÔØ"×/Ñ/ˆÔØ ×+Ñ+ˆŒØ &× 7Ñ 7ˆÔØ!×-Ñ-ˆÔàˆŒØ×ÑÐ#ÜŸ)™) F×$6Ñ$6¸¿¹È×IYÑIYÑ8YÐ`eÔfˆD�KäŸI™I f×&8Ñ&8¸&×:LÑ:LÐSY×ShÑShÔiˆDŒMÜ!0°×1CÑ1CÓ!DˆDÔÜŸI™I f×&8Ñ&8¸$¿.¹.È4×K[ÑK[Ñ:[ÐbgÔhˆDŒMä"$§)¡)Ø×ÑØ×Ñ × 5Ñ 5Ñ5Ø×&Ñ&ô#
ˆÔô
 .¨d×.?Ñ.?Ó@ˆÔÜŸ™Ø×ÑØ�N‰N˜d×3Ñ3°d·o±oÑEÑFØô
ˆŒô —i‘iØ�N‰N˜TŸ_™_Ñ,Ø×ÑØ×&Ñ&ô
ˆŒð ×'Ñ'¨DÑ1ˆ�r(   NrY   Úposition_embeddingsÚattention_maskÚposition_idsÚpast_key_valuesÚkwargsÚreturnc                 ó   — |j                   d d \  }}||d| j                  f}	||d| j                  | j                  z   f}
| j                  €| j                  |«      }n/| j                  | j                  | j                  |«      «      «      }|j                  |	«      j                  dd«      }t        j                  || j                  | j                  gd¬«      \  }}| j                  |«      }t        j                  || j                  | j                  gd¬«      \  }}| j!                  | j#                  |«      «      j                  |
«      j                  dd«      }t        j                  || j                  | j                  gd¬«      \  }}|j                  |d|| j                  «      }|\  }}| j$                  j&                  rt)        ||||«      \  }}nt+        ||||«      \  }} |j,                  g |j                   d d ¢d‘­Ž }t        j.                  ||fd¬«      }t        j.                  ||fd¬«      }|t1        || j$                  j2                  j5                  d«      | j$                  j2                  j5                  d«      «      j7                  |j8                  «      z  }|� |j;                  ||| j<                  «      \  }}t?        | j$                  «      rH| j                  | j                  k7  r/tA        jB                  |d| j                  | j                  z
  g«      }tE        jF                  | j$                  jH                  tJ        «      } || ||||f| jL                  sdn| jN                  | jP                  d	œ|¤Ž\  }}t?        | j$                  «      r4| j                  | j                  k7  r|d d …d d …d d …d | j                  …f   }|jS                  ||d«      jU                  «       }| jW                  |«      }||fS )
Nr=   r   r   r>   Úllama_4_scaling_betaÚ original_max_position_embeddingsr   rB   )Údropoutrˆ   ),Úshaper{   rz   ry   rv   r   rƒ   r‚   r�   rD   Ú	transposerM   Úsplitrw   r„   rx   r†   r…   r:   Úrope_interleaver   r   rR   Úcatr   Úrope_parametersÚgetÚtoÚdtypeÚupdaterm   r   rF   Úpadr   Úget_interfaceÚ_attn_implementationr   Útrainingrt   rˆ   rS   Ú
contiguousr‡   )r9   rY   rŠ   r‹   rŒ   r�   rŽ   Ú
batch_sizeÚ
seq_lengthÚquery_shapeÚ	key_shapeÚq_statesÚq_passÚq_rotÚcompressed_kvÚk_passÚk_rotÚvalue_statesÚcosÚsinÚquery_statesÚ
key_statesÚattention_interfaceÚattn_outputÚattn_weightss                            r)   rd   zMistral4Attention.forward”   sž  € ð "/×!4Ñ!4°S°bÐ!9Ñˆ
�JØ! :¨r°4×3CÑ3CÐDˆØ ¨R°×1FÑ1FÈÏÉÑ1XÐYˆ	à×ÑÐ#Ø—{‘{ =Ó1‰Hà—}‘} T×%7Ñ%7¸¿¹ÀmÓ8TÓ%UÓVˆHØ—=‘= Ó-×7Ñ7¸¸1Ó=ˆÜŸ™ H¨t×/DÑ/DÀd×F[ÑF[Ð.\ÐbdÔe‰ˆ�à×/Ñ/°Ó>ˆÜŸ™ M°D×4EÑ4EÀt×G\ÑG\Ð3]ÐceÔf‰ˆ�à—‘ × 3Ñ 3°FÓ ;Ó<×AÑAÀ)ÓL×VÑVÐWXÐZ[Ó\ˆÜ$Ÿ{™{¨6°D×4IÑ4IÈ4Ï?É?Ð3[ÐacÔdÑˆ�à—
‘
˜: q¨*°d×6KÑ6KÓLˆà&‰ˆˆSØ�;‰;×&Ò&Ü:¸5À%ÈÈcÓR‰LˆE‘5ä/°°u¸cÀ3ÓG‰LˆE�5Ø�—‘Ð4˜fŸl™l¨3¨BÐ/Ð4°Ò4ˆä—y‘y &¨% °bÔ9ˆÜ—Y‘Y ¨˜°BÔ7ˆ
à#Ô&<ØØ�K‰K×'Ñ'×+Ñ+Ð,BÓCØ�K‰K×'Ñ'×+Ñ+Ð,NÓOó'
÷ ‰"ˆ\×ÑÓ
 ñ	!ˆð Ð&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä'¨¯©Ô4¸×9IÑ9IÈTÏ_É_Ò9\ÜŸ5™5 °°4×3CÑ3CÀdÇoÁoÑ3UÐ/VÓWˆLä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ô (¨¯©Ô4¸×9IÑ9IÈTÏ_É_Ò9\Ø%¢aªªAÐ/@°·±Ð/@Ð&@ÑAˆKà!×)Ñ)¨*°jÀ"ÓE×PÑPÓRˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r(   r2   )r$   r%   r&   r    Úintr4   rM   ÚTensorÚtupler   r   r   rd   r'   r(   r)   rl   rl   h   sÀ   „ ð)2˜~ð )2¸#ó )2ðb )-ñF)à—|‘|ðF)ð # 5§<¡<°·±Ð#=Ñ>ðF)ð Ÿ™ tÑ+ð	F)ð
 —l‘lðF)ð  ™ðF)ð Ð-Ñ.ðF)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	MôF)r(   rl   c                   ó   — e Zd Zdedefd„Zy)ÚMistral4DecoderLayerr:   rm   c                 ó€  — t         j                  j                  | «       |j                  | _        t	        ||¬«      | _        ||j                  k\  rt        |«      | _        nt        |«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y )N)r:   rm   )Úeps)r   rp   r4   r~   rl   Ú	self_attnÚfirst_k_dense_replacerj   Úmlpr.   r"   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr‰   s      r)   r4   zMistral4DecoderLayer.__init__Þ   s�   € Ü
�	‰	×Ñ˜4Ô Ø!×-Ñ-ˆÔä*°&ÀIÔNˆŒà˜×4Ñ4Ò4Ü" 6Ó*ˆD�Hä" 6Ó*ˆDŒHä.¨v×/AÑ/AÀv×GZÑGZÔ[ˆÔÜ(7¸×8JÑ8JÐPV×PcÑPcÔ(dˆÕ%r(   N)r$   r%   r&   r    rµ   r4   r'   r(   r)   r¹   r¹   Ý   s   „ ðe˜~ð e¸#ô er(   r¹   c                   óŒ   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZg Zg Z ej(                  «       ˆ fd„«       Zˆ xZS )ÚMistral4PreTrainedModelr:   ÚmodelTr¹   r�   )rY   Ú
attentionsc                 ó¬  •— t         ‰| �  |«       t        |t        «      r7t	        j
                  |j                  d| j                  j                  ¬«       y t        |t        «      rmt	        j
                  |j                  d| j                  j                  ¬«       t	        j
                  |j                  d| j                  j                  ¬«       y y )NrB   )ÚmeanÚstd)r3   Ú_init_weightsÚ
isinstancer0   ÚinitÚnormal_rH   r:   Úinitializer_rangerh   Úgate_up_projÚ	down_proj)r9   Úmoduler;   s     €r)   rÉ   z%Mistral4PreTrainedModel._init_weights   sˆ   ø€ ä‰Ñ˜fÔ%Ü�fÔ0Ô1Ü�L‰L˜Ÿ™¨S°d·k±k×6SÑ6SÖTÜ˜¤Ô0Ü�L‰L˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�L‰L˜×)Ñ)°¸¿¹×9VÑ9VÖWð 1r(   )r$   r%   r&   r    Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr¹   rl   Ú_can_record_outputsÚ_keep_in_fp32_modules_strictÚ"_keys_to_ignore_on_load_unexpectedrM   Úno_gradrÉ   re   rf   s   @r)   rÃ   rÃ   í   s|   ø… ØÓØÐØ&*Ð#Ø/Ð0ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà-Ø'ñÐð $&Ð Ø)+Ð&à€U‡]�]ƒ_óXó ôXr(   rÃ   c                   ó   — e Zd Zy)ÚMistral4ModelNr#   r'   r(   r)   rà   rà   
  r*   r(   rà   c                   ó   — e Zd Zy)ÚMistral4ForCausalLMNr#   r'   r(   r)   râ   râ     r*   r(   râ   c                   ó   — e Zd Zy)Ú!Mistral4ForSequenceClassificationNr#   r'   r(   r)   rä   rä     r*   r(   rä   c                   ó   — e Zd Zy)ÚMistral4ForTokenClassificationNr#   r'   r(   r)   ræ   ræ     r*   r(   ræ   )rÃ   rà   râ   rä   ræ   )@Úcollections.abcr   rM   Útorch.nn.functionalr   Ú
functionalrF   Ú r   rË   Úcache_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr	   r
   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   Úutils.genericr   Ú deepseek_v2.modeling_deepseek_v2r   Ú deepseek_v3.modeling_deepseek_v3r   r   r   r   r   Úllama.modeling_llamar   r   r   r   r   r   Úministral3.modeling_ministral3r   Úqwen2_moe.modeling_qwen2_moer   Úconfiguration_mistral4r    Ú
get_loggerr$   Úloggerr"   r,   r.   r0   rh   rj   rl   r¹   rÃ   rà   râ   rä   ræ   Ú__all__r'   r(   r)   ú<module>rû      s  ðõ %ã ß Ð Ý å &Ý  Ý Bß ^ß FÝ &Ý Ý 9Ý C÷õ ÷÷ õ DÝ 6Ý 2ð 
ˆ×	Ñ	˜HÓ	%€ô	�lô 	ô	Ð2ô 	ô	�+ô 	ô9Ð-ô 9ô@	Ð'ô 	ô	�-ô 	ôr)Ð+ô r)ôjeÐ1ô eô X˜oô Xô:	�Jô 	ô	Ð*ô 	ô	Ð(HÐJaô 	ô	Ð%BÐD[ô 	ò�r(   