
    ^jkl                        d dl mZmZ  e       r
ddlZddlmZ d dlmZmZ d dlm	Z	m
Z
mZ  ej                  e      Zg dZ G d d	e      Z G d
 de      Z G d de      Z G d de      Zd Zd Zej,                  dddej.                  dedej2                  fdZej,                  dddej.                  dedej2                  fdZ G d dej8                        Zd Zd Zd Z d Z!d Z"d  Z#d#d!e$e%   dz  fd"Z&y)$   )is_torch_availablelogging    N)nn)ConversionOps_IdentityOp)get_module_from_name	on_deviceshould_convert_module)g        g      ?g      ?g      ?g       @g      @g      @g      @g       g      g      g      g       g      g      g      c                       e Zd Zd Z	 	 	 d	deeej                  f   dej                  j                  dz  de
e   dz  dedz  deeej                  f   f
dZy)
Mxfp4Quantizec                     || _         y Nhf_quantizerselfr   s     j/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/integrations/mxfp4.py__init__zMxfp4Quantize.__init__1   
    (    N
input_dictmodelmissing_keysfull_layer_namereturnc                    t        |j                               d   \  }}t        |t              r|d   n|}t	        ||      \  }}t        j                  |j                        5  t        |t              rt        |j                  dd      t              \  }	}
t        j                  j                  t        j                  j                  t        j                  j                  }}}t        |	|
t              \  }	}
d|v rdnd}||j                   v r|j                   |= t#        |||	       t#        || d ||
 | |                          |j%                  |        d	|_        i cd d d        S 	 d d d        y # 1 sw Y   y xY w)
Nr   gate_up_proj	down_proj_precision_configrhs_dataweight_scaleflex_ctxT)tupleitems
isinstancelistr	   torchdeviceMxfp4GptOssExpertsquantize_to_mxfp4	transposetriton_kernels_hub
matmul_ogsPrecisionConfigFlexCtx
InFlexDataswizzle_mxfp4_parameterssetattrdiscard_is_hf_initialized)r   r   r   r   r   kwargs_valuemoduletriton_weight_tensorr&   r3   r4   r5   projs                  r   convertzMxfp4Quantize.convert4   ss    ))+,Q/5&ud3a(@	\\%,,' 	&"455FuWY[]G^`r5s2$l&11AA&1199&11<< +5
 6C(,8J62$l *8?)J~P[6---**40&:;f-.#YcYeHfg $$'8:,0)9	 	5	 	 	s   %DE==FNNN__name__
__module____qualname__r   dictstrr,   Tensorr   Moduler+   rA    r   r   r   r   0   s}    ) )-)-&*)ell*+) xx%) 3i$&	)
 t) 
c5<<	 )r   r   c                       e Zd Zd Z	 	 	 d
deeej                  f   dej                  j                  dz  dedz  de
e   dz  deeej                  f   f
dZedd	       Zy)Mxfp4Dequantizec                     || _         y r   r   r   s     r   r   zMxfp4Dequantize.__init__a   r   r   Nr   r   r   r   r   c                 z   i }d|v rdnd}| d|j                         v r6t        || d   t              r|| d   d   || d<   n|| d   || d<   | d|j                         v r6t        || d   t              r|| d   d   || d<   n|| d   || d<   t        || d   || d         }||iS )Nr    r!   _blocksr   _scales)keysr*   r+   dequantize_convertops)	r   r   r   r   r   r;   
param_datar@   dequantizeds	            r   rA   zMxfp4Dequantize.convertd   s!    
!/?!B~V7z00*vW%56=/9TF':J/KA/N
dV7+,/9TF':J/K
dV7+,V7z00*vW%56=/9TF':J/KA/N
dV7+,/9TF':J/K
dV7+, ,J$w7G,H*X\W]]dUeJfg--r   c                     t               S r   )r   r   s    r   
reverse_opzMxfp4Dequantize.reverse_op}   s
    }r   rB   )r   r   )rD   rE   rF   r   rG   rH   r,   rI   r   rJ   r+   rA   propertyrX   rK   r   r   rM   rM   `   s    ) )-&*)-.ell*+. xx%. t	.
 3i$&. 
c5<<	 .2  r   rM   c                       e Zd Zd Z	 	 	 d
deeej                  f   dej                  j                  dz  dedz  de
e   dz  deeej                  f   f
dZedefd	       Zy)Mxfp4Deserializec                     || _         y r   r   r   s     r   r   zMxfp4Deserialize.__init__   r   r   Nr   r   r   r   r   c           	         i }d|v rdnd}| d|j                         v r6t        || d   t              r|| d   d   || d<   n|| d   || d<   | d|j                         v r6t        || d   t              r|| d   d   || d<   n|| d   || d<   t        ||      \  }}	t	        || d   || d   |||| d   j
                  t               |j                  |        d|_        i S )Nr    r!   rP   r   rQ   T)	rR   r*   r+   r	   swizzle_mxfp4_convertopsr-   r1   r9   r:   )
r   r   r   r   r   r;   rT   r@   r>   r<   s
             r   rA   zMxfp4Deserialize.convert   se    
!/?!B~V7z00*vW%56=/9TF':J/KA/N
dV7+,/9TF':J/K
dV7+,V7z00*vW%56=/9TF':J/KA/N
dV7+,/9TF':J/K
dV7+, )@	 $w'($w'($w'(//	
 	02$(! 	r   c                 ,    t        | j                        S r   )Mxfp4ReverseDeserializer   rW   s    r   rX   zMxfp4Deserialize.reverse_op   s    &t'8'899r   rB   )rD   rE   rF   r   rG   rH   r,   rI   r   rJ   r+   rA   rY   r   rX   rK   r   r   r[   r[      s    ) )-&*)-%ell*+% xx%% t	%
 3i$&% 
c5<<	 %N :M : :r   r[   c                       e Zd Zd Z	 	 	 d	deeej                  f   dej                  j                  dz  dedz  de
e   dz  deeej                  f   f
dZy)
r`   c                     || _         y r   r   r   s     r   r   z Mxfp4ReverseDeserialize.__init__   r   r   Nr   r   r   r   r   c                    t        |j                  dd      }t        |j                  dd      }d|v rdnd}|j                  dd      d	   }	t        ||      \  }
}i }t	        |
t
              rd
|v r&|j                  dd      }	t        |
|dz         ||	<   |S d|v r|
j                  j                  j                  j                  |
j                  j                  j                        j                  dd      j                  |ddd      ||	 d<   |
j                  j                  j                  j                  j                  |
j                  j                  j                  j                        j                  dd      ||	 d<   |S |
j                   j                  j                  j                  |
j                   j                  j                        j                  dd      j                  ||dd      ||	 d<   |
j"                  j                  j                  j                  j                  |
j"                  j                  j                  j                        j                  dd      ||	 d<   |S )Nnum_local_experts    hidden_sizei@  r    r!   r<      r   biasrP    _biasr   r   Z      rQ   )getattrconfigrsplitr	   r*   r.   replacer    storagelayoutunswizzle_datadatar0   reshapegate_up_proj_precision_configr&   r!   down_proj_precision_config)r   r   r   r   r   r;   rd   rf   r@   namer>   r<   
state_dicts                r   rA   zMxfp4ReverseDeserialize.convert   s>    $ELL2ErJellM4@!/?!B~%%c1-a0(@	
f01(&..y"=#*64'>#B
4 !!0''//66EEfFYFYFaFaFfFfgYr2&W.B; dV7+, 88EEMMTTcc<<IIQQVViB' dV7+,"  $$,,33BB6CSCSC[C[C`C`aYr2&W.RD dV7+, 55BBJJQQ``99FFNNSSiB' dV7+, r   rB   rC   rK   r   r   r`   r`      s}    ) )-&*)-,ell*+, xx%, t	,
 3i$&, 
c5<<	 ,r   r`   c                     |j                   j                  j                  } || j                  t        j
                        t        j                  d      \  } }| |fS )Nrg   )axis)numerics_detailsmxfpdowncast_to_mxfp_torchtor,   bfloat16uint8)wr1   r~   w_scales       r   r/   r/      sH    /@@EE\\'U^^(<ekkPQRJAwg:r   c                 f   |j                   j                  |j                   j                  |j                   j                  }}}|j                  j
                  }|j                  j
                  j                  }|j                  d      \  }}	 | || |      |fi |	}  | ||      |      }| |fS )zE
    Changes the layout of the tensors depending on the hardware
    rg   )mx_axisdtype)tensorFP4convert_layoutwrap_torch_tensortensor_detailsrr   StridedLayout"make_default_matmul_mxfp4_w_layout)
r   r   r1   r   r   r   rr   r   value_layoutvalue_layout_optss
             r   r6   r6      s    
 	!!%%!!00!!33 +C
  ..55F&55<<JJM&,&O&OXY&O&Z#L#(#6ZHYZA.w7GGg:r   i   r   rows_per_chunkr   r   r   c                4   ddl }| j                  t        j                        } |j                  t        j                        dz
  }| j
                  dd |j
                  k(  s$J d| j
                  dd d|j
                         t        j                  t        || j                        }| j
                  ^ }}}|j                  |      |z  }	| j                  |	|      } |j                  |	d      }t        j                  |	|d	z  || j                        }
t        d|	|      D ]  }t        ||z   |	      }| || }||| }|
|| }|d
z  j                  t        j                        }||   |ddddd	f<   ~|dz	  j                  t        j                        }||   |ddddd	f<   ~t        j                  |||       ~~~   |
j                  g |||d	z   j                   g |||z  d	z   }
|
j#                  dd	      j%                         S )w
    Convert the mxfp4 weights again, dequantizing and makes them compatible with the forward
    pass of GPT_OSS.
    r   N   r   zblocks.shape[:-1]=z does not match scales.shape=)r   r-   rg   r         )out)mathr   r,   r   int32shaper   
FP4_VALUESr-   prodru   emptyrangeminintldexpviewr0   
contiguous)blocksscalesr   r   r   lutprefix_shapeGB
rows_totalr   r0r1blkexpsubidx_loidx_his                     r   _convert_moe_packed_tensorsr      s    YYu{{#FYYu{{#c)F<<,d1Ccr1B0DDbU[UaUaTc.dd,
,,zv}}
EC ,,\1a<(1,J^^J*F^^J*F
++j!a%uV]]
KCAz>2 n$j1RmRm"Rj *+6{Aqt!tG (uyy)6{Aqt!tG 	C#&c'* 4+#++
.|
.Q
.A
.
3
3
M\
M1q519
MC==A))++r   c                    	 t        | |||      S # t        j                  $ r4 | j                  d      } |j                  d      }t        | |||      cY S w xY w)r   r   cpu)r   r,   OutOfMemoryErrorr   )r   r   r   r   s       r   convert_moe_packed_tensorsr   6  sa    g*66Weff !! g5!5!*66Weffgs    AAAc                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )r.   c           	         t         |           |j                  | _        |j                  | _        |j
                  | _        t        j                  t        j                  | j                  d| j                  z  | j
                  dz  dt        j                        d      | _        t        j                  t        j                  | j                  d| j                  z  t        j                        d      | _        t        j                  t        j                  | j                  | j
                  | j                  dz  dft        j                        d      | _        t        j                  t        j                  | j                  | j
                  t        j                        d      | _        d| _        t#        |dd	      | _        d | _        d | _        t#        |dd	      | _        y )
Nr   re   rl   r   Frequires_gradgZd;?swiglu_limitg      @)superr   rd   num_expertsintermediate_sizerf   r   	Parameterr,   zerosr   r    float32gate_up_proj_biasr!   down_proj_biasalpharm   limitrv   rw   )r   rn   	__class__s     r   r   zMxfp4GptOssExperts.__init__P  s{   !33!'!9!9!--LLKK((!d.D.D*DdFVFVZ\F\^`hmhshst

 "$KK((!d.D.D*DEMMZjo"
 KK))4+;+;T=S=SWY=Y[]^fkfqfqr

 !llKK(($*:*:%--P`e
 
V^S9
-1**.'V^S9
r   hidden_statesr   c                    t         j                  j                  t         j                  j                  t         j                  j                  }}}t         j                  j
                  }t        |j                        5   | |d|d      | j                  | j                  fd      }	 ||| j                  | j                  j                  t        j                        ||| j                  d |	      }
 ||
| j                   | j"                  j                  t        j                        ||| j$                  |j&                        }d d d        |S # 1 sw Y   S xY w)Nswiglu)r   r   r   )gather_indxprecision_configgammasfused_activation)scatter_indxr   r   )r1   r2   FnSpecsFusedActivationr   	swiglu_fnr
   r-   r   r   r    r   r   r,   r   rv   r!   r   rw   	gate_scal)r   r   routing_data
gather_idxscatter_idxr   r   r2   r   actintermediate_cache1intermediate_cache3s               r   forwardzMxfp4GptOssExperts.forwardn  s0   ))11))99))44 #-
 '--77	}++, 	!'(I?Q"RUYU_U_aeakakTlnopC",!!&&))%--8&!%!C!C!$	# #-###&&u}}5(!%!@!@#--#	. #"/	. #"s   >CEE)rD   rE   rF   r   r,   rI   r   __classcell__)r   s   @r   r.   r.   O  s&    :<#U\\ #]b]i]i #r   r.   c                 B   dd l }t        j                  j                  t        j                  j                  t        j                  j
                  t        j                  j                  f\  }}}}t        | j                        5  t        j                  j                         }t        |j                  j                  dd            }d}	| j                  d   }
| j                  d   }||z  }||z  }|dz   |z  }|
|z  }d } || |      \  }}t        j                   |d      }t        j"                  |d      \  }}t        j$                  |d|      }|j'                  d      }t        j(                  |||dz
        || }|j+                  d      j-                  t        j.                        }d	}t        j0                  ||k  ||      }t        j2                  |d
      j-                  t        j.                        }t        j2                  |      j-                  t        j.                        }t        j0                  ||k  ||	      }t        j0                  ||k  ||	      }t        j0                  ||	k(  |	|      }||   }t        j0                  ||   |	k(  |	|      } ||j                         |j                               } ||j                         |j                               } ||||      }|}d d d         |      fS # 1 sw Y   xY w)Nr   
LOCAL_RANK0r   rg   c                     t        j                  |  dd      d d d |f   }|j                         }t        j                  | |d      }||j	                         fS )Nrg   T)dimstabler   )r,   argsortlongtake_along_dimr   )valsktk_indxtk_vals       r   topkz routing_torch_dist.<locals>.topk  sS    mmTEq>q"1"uEGllnG))$Q?F7;;=((r   r   )binsmaxi  T)r   )src_indxdst_indx)osr1   routing
GatherIndxRoutingDataScatterIndxcompute_expt_data_torchr
   r-   r,   distributedget_world_sizer   environgetr   softmaxsortgatherru   histcr   r   r   wherer   )logitsn_expts_actr   r   r   r   r   
world_sizerankreplace_valuen_tokensn_expts_totn_local_expertslocal_expert_startlocal_expert_endn_gates_padr   	expt_scal	expt_indxsort_indiceshistvar	topk_indx	gate_indxr   r   r   	expt_datahit_expertss                                r   routing_torch_distr    s     	""--""..""..""::	EAJ[*A 
6==	! 3"&&557
2::>>,45<<?ll1o%3!O3 1H7,	)  $FK8	9MM)4	"'**YA">	<LLA|<	 %%b)	{{9;K!OLM_`pqNN2&))%++6	 KK	,> >YO	MM)D9<<U[[I	MM),//<	KK	,< <iW	KK 2i ?MZ	KK	] :M9U	i(	KK	) 4 E}V_`	 !)--/IMMOT"IMMOimmoV+D/;O	!g3"h y$iPR]_kkki3" 3"s   I9LLc                    dd l m} |j                         r#|j                         rt	        | d      rt
        }nt        j                  j                  }|j                  d   }|j                  d| j                  j                        }t        j                  j                  || j                  j                  | j                  j                         }t#        |j$                        5   ||| j                  j&                        \  }}}d d d        | j)                  |      }	|	j                  |d| j                  j                        }	|	|fS # 1 sw Y   IxY w)Nr   
_is_hookedr   )r   )torch.distributedr   is_availableis_initializedhasattrr  r1   r   r   ru   router
hidden_dimr   
functionallinearweightrh   r
   r-   top_kexperts)
r   r   distr   
batch_sizerouter_logitsr   r   r   
routed_outs
             r   mlp_forwardr$    s   $t224|9T$$,,44$$Q'J!))"dkk.D.DEMMM((8J8JDKKL\L\]M	=''	( Z07t{{GXGX0Y-j+Z m\:S^_J##JDKK4J4JKJ}$$Z Zs   '"EEc                    ddl m} |j                  d      }|j                  d      }|j                  d      }	|j                  d      }
|j                  d      }|j                  d      }d	D ]  }||v s| ||||||	|
||      }| d
}| d}t        | |j	                  dd      d   |       t        | |      sPt        | |      s]t        t        | |      t        | |            }t        | |t        j                  j                  |j                  |                   t        | |       t        | |        y )Nr   shard_and_distribute_moduler   empty_paramcasting_dtypeto_contiguousr  device_mesh)r    r!   rP   rQ   .rg   )integrations.tensor_parallelr'  r   r8   ro   r  r   rm   r,   r   r   r   delattr)r>   
param_nameparam_valuetarget_devicedq_param_namer;   r'  r   r(  r)  r*  r  r+  r@   blocks_attrscales_attrrU   s                    r   
dequantizer5    s8   JJJwE**]+KJJ/MJJ/M::fD**]+K- -:&9!!!	 "F'*K!F'*KFJ--c15a8+Fv{+0L89UW^_egrWstehh&8&89V&WX,,)-r   c                 X    t        | |      }t        j                  j                  |      S r   )r   r,   r   r   )r   r   rU   s      r   rS   rS     s#    ,VV<K88k**r   c                 Z   |j                   j                  |j                   j                  |j                   j                  }}}ddlm}	 |j                  d      }
|j                  d      }|j                  d      }|j                  d      }|j                  d      }|j                  d      }d	|v r&|j                  d
      d   j                  d      d   }d|v r&|j                  d
      d   j                  d      d   }| |	|
|||||||       n?t        | |j                  d
d      d   t        j                  j                  |d              d}| d}t        | |      }t        | |      }|j                  j                  dk7  r|j                  j                  dk7  r|j!                  d      }|dk(  r!|j#                  || j$                  dz  d      }n |j#                  |d| j$                  dz        }t        |d|      dk(  rVt'        t        d      rFt        j(                  j+                         (t        j(                  j+                         j                  }|j-                  |      j/                         }|j-                  |      j/                         }t1        |      5  t3        |j5                  dd      |j5                  dd      |      \  }}ddd       |dk(  r5t        j6                  || j8                  | j$                  dz  g      _        n1t        j6                  || j$                  | j8                  g      _        t        | ||       t        | | d | | |                          t=        | |       t=        | |       ~yyy# 1 sw Y   xY w)q
    This transforms the weights obtained using `convert_gpt_oss.py` to load them into `Mxfp4GptOssExperts`.
    r   r&  r   r(  r)  r*  r  r+  r   r,  r   rP   r   r   rQ   Nrg   Fr   metar    typer   acceleratorr   r"   r#   r%   )r2   r3   r4   r5   r-  r'  r   splitr8   ro   r,   r   r   rm   r-   r:  sizeru   r   r  r;  current_acceleratorr   r   r
   r6   r0   Sizerf   r   r.  )r>   r/  r0  r1  r1   r;   r3   r4   r5   r'  r   r(  r)  r*  r  r+  r@   r3  r4  r   r   local_expertsr?   r&   s                           r   load_and_swizzle_mxfp4rA    sf   
 	%%55%%--%%00 )WO
 KJJwE**]+KJJ/MJJ/M::fD**]+K:$R(..y9!<:$R(..y9!<#;ZW[]h	
 	
))#q1!4ehh6H6Hdi6H6jkF'"KF'"KV[)FV[)F}}V#(:(:f(DA>!^^M63K3Ka3OQSTF^^M2v7O7OST7TUFM6=9UB}-!!557C!--AACHHM=)446=)446}% 	1>  R(&*:*:2r*BDV2. ,	 >!).]FDVDVX^XpXpstXt4u)v &).]FD\D\^d^p^p4q)r & 	23f%&Q[Q]@^_	
 	$$I )E#	 	s   01N!!N*c                    |j                   j                  |j                   j                  |j                   j                  }}}| j	                  d      }	t        |d|      dk(  rVt        t        d      rFt        j                  j                         (t        j                  j                         j                  }| j                  |      j                         } |j                  |      j                         }|dk(  r!| j                  |	|j                  dz  d      } n | j                  |	d|j                  dz        } t        |      5  t!        | j#                  d	d      |j#                  d	d      |      \  }
}ddd       |dk(  r5t        j$                  |	|j&                  |j                  dz  g      
_        n1t        j$                  |	|j                  |j&                  g      
_        ||j*                  v r|j*                  |= t-        |||
       t-        || d
 | | |                          y# 1 sw Y   xY w)r8  r   r:  r   r;  Nr    r   r   r   r"   r#   r%   )r2   r3   r4   r5   r=  rm   r  r,   r;  r>  r:  r   r   ru   r   r
   r6   r0   r?  rf   r   r7   r8   )r   r   r>   r@   r1  r1   r3   r4   r5   r@  r?   r&   s               r   r^   r^   W  s    
 	%%55%%--%%00 )WO KKNMv}5>E=)113?))==?DDYY}%002FYY}%002F~v/G/G!/KRPr63K3Kq3PQ	=	! 
-:R$f&6&6r2&>@R.
*l

 ~%*ZZ@R@RTZTlTlopTp0q%r"%*ZZ@X@XZ`ZlZl0m%n" v!!!t$FD./&!"\GZ\<Z[!
 
s   	1H::Imodules_to_not_convertc                 (   |j                   r| S ddlm}  |dd      ad}| j	                         D ]  \  }}t        ||      s|j                  j                  dk(  rQ|j                   sEt        j                  d      5  | j                  |t        | j                               d}d	d	d	       |j                  j                  d
k(  s|j                   rddlm}  |t        |      |_         |st"        j%                  d       | S # 1 sw Y   cxY w)aD  
    Public method that replaces the expert layers of the given model with mxfp4 quantized layers.

    Args:
        model (`torch.nn.Module`):
            The model to convert, can be any `torch.nn.Module` instance.
        quantization_config (`Mxfp4Config`, defaults to `None`):
            The quantization config object that contains the quantization parameters.
        modules_to_not_convert (`list`, *optional*, defaults to `None`):
            A list of modules to not convert. If a module name is in the list (e.g. `lm_head`), it will not be
            converted.
    rg   )
get_kernelz(kernels-community/gpt-oss-triton-kernels)versionFGptOssExpertsr9  TN	GptOssMLPr   )
MethodTypezYou are loading your model using mixed-precision FP4 quantization but no linear modules were found in your model. Please double check your model architecture, or submit an issue on github if you think this is a bug.)r5  hub_kernelsrE  r1   named_modulesr   r   rD   r,   r-   set_submoduler.   rn   typesrI  r$  r   loggerwarning)r   quantization_configrC  rE  has_been_replacedmodule_namer>   rI  s           r   replace_with_mxfp4_linearrS    s    %%' $$NXYZ$224 
=V$[2HI$$7@S@^@^f% )##K1CELL1QR$(!) $$3<O<Z<Z('V<FN
= 	
 L) )s   >(DD	)NN)'utilsr   r   r,   r   core_model_loadingr   r   quantizers.quantizers_utilsr	   r
   r   
get_loggerrD   rN  r   r   rM   r[   r`   r/   r6   r   r   r   rI   r   r   rJ   r.   r  r$  r5  rS   rA  r^   r+   rH   rS  rK   r   r   <module>rX     s<   0  ; ` ` 
		H	%
(-M -`m D-:} -:`0m 0h0 &3, ;;	3,
 3, \\3,t &g ;;	g
 g \\g2># >#FAlH%(-B+
DN.b)W[\_W`cgWg )r   