Ë
    ÛÓ^jÃ  ã                  ó:  — d dl mZ d dlZd dlZd dlmZ d dlmZ d dlZd dl	m
Z
 d dl	mZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZmZm Z  ddl!m"Z" ddl#m$Z$m%Z% ddl&m'Z'  ejP                  e)«      Z*ejV                  Z, ejZ                  e,«      j\                  Z/ ejZ                  e,«      j`                  Z1ejd                  d2d„«       Z3d„ Z4 ed¬«       G d„ d«      «       Z5ejd                  d3d„«       Z6ejn                  jp                  d4d„«       Z9d3d„Z:d5d„Z;	 	 	 	 d6	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d7d„Z< edd¬«      	 	 	 	 d8	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d9d „«       Z= edd¬«      	 	 	 	 	 d:	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d;d!„«       Z> G d"„ d#e
j~                  «      Z@ G d$„ d%e@«      ZA	 	 	 	 	 	 	 	 	 	 d<d&„ZB	 	 	 	 	 	 	 	 	 	 d<d'„ZC G d(„ d)e
jˆ                  «      ZE G d*„ d+e$«      ZF eF«       ZGd=d,„ZH	 d>	 d?d-„ZI G d.„ d/e«      ZJ G d0„ d1e«      ZKy)@é    )ÚannotationsN)ÚCallable)Ú	dataclass)Ú
functionalé   )ÚACT2FN)ÚConversionOps)Úget_module_from_nameÚshould_convert_module)Úlogging)Údeprecate_kwarg)ÚKERNELS_MAX_VERSIONÚKERNELS_MIN_VERSIONÚis_kernels_availableÚis_torchdynamo_compilingé   )Ú deepgemm_fp8_fp4_experts_forwardÚdeepgemm_fp8_fp4_linearÚ(deepgemm_fp8_fp4_megamoe_experts_forward)Úlazy_load_kernel)ÚExpertsInterfaceÚuse_experts_implementation)Úto_localc                 ó|   — t        t        d«      st        dt        j                  › d�«      ‚t        j                  S )uS  Return ``torch.float8_e8m0fnu`` or raise a clear error on torch without FP8 support.

    UE8M0 scales are always stored/consumed as this single dtype â€” the kernels (Triton
    finegrained + DeepGEMM) read it natively, and supporting the same scales in mixed
    container dtypes would be a mess â€” so fail loudly rather than fall back.Úfloat8_e8m0fnuzbscale_fmt='ue8m0' requires torch.float8_e8m0fnu, which is only available in PyTorch >= 2.7 (found z.). Upgrade torch to use UE8M0 FP8 checkpoints.)ÚhasattrÚtorchÚRuntimeErrorÚ__version__r   © ó    út/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/integrations/finegrained_fp8.pyÚ_get_ue8m0_dtyper#   6   sD   € ô ”5Ð*Ô+Üð%Ü%*×%6Ñ%6Ð$7Ð7eðgó
ð 	
ô ×ÑÐr!   c                óŠ   — |D ]  }t        | |«      sŒt        | |«      c S  t        t        | «      j                  › d|› �«      ‚)Nz has none of: )r   ÚgetattrÚAttributeErrorÚtypeÚ__name__)ÚobjÚnamesÚnames      r"   Ú_first_attrr,   E   sK   € Øò &ˆÜ�3˜ÕÜ˜3 Ó%Ò%ð&ô œD ›I×.Ñ.Ð/¨~¸e¸WÐEÓ
FÐFr!   T)Úfrozenc                  ó0   — e Zd ZU dZded<   ded<   ded<   y)ÚFineGrainedFP8zNEntry points exposed by the `kernels-community/finegrained-fp8` Triton kernel.r   ÚmatmulÚbatched_matmulÚgrouped_matmulN)r(   Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__r    r!   r"   r/   r/   L   s   … áXàÓØÓØÔr!   r/   c                 óº  — t        «       s+t        «       s!t        dt        › dt        › dt        › d�«      ‚t        d«      } | €t        d«      ‚t        | dd«      }t        | d	d«      }t        | d
d«      }d|fd	|fd
|ffD ��cg c]	  \  }}|€|‘Œ }}}|r3t        ddj                  |«      › dt        › dt        › dt        › d�	«      ‚t        |||¬«      S c c}}w )zÈ
    Load the finegrained-fp8 Triton kernel once and return its entry points.

    Raises `ImportError` if the `kernels` package is missing, or the kernel or required
    symbols cannot be found.
    z\finegrained-fp8 kernel requires the `kernels` package. Please install a compatible version (z <= version < z), e.g. `pip install kernels==ú`zfinegrained-fp8Nu‰   Failed to load the finegrained-fp8 kernel â€” check that `kernels-community/finegrained-fp8` has a build matching the current torch/CUDA.Ú	matmul_2dÚmatmul_batchedÚmatmul_groupedz4finegrained-fp8 kernel is missing required symbols: ú, z'. Please install a compatible version ()r0   r1   r2   )	r   r   ÚImportErrorr   r   r   r%   Újoinr/   )Úkernelr0   r1   r2   r+   ÚattrÚmissings          r"   Ú_load_finegrained_fp8_kernelrB   U   sB  € ô $Ô%Ü#Ô%Üð8Ü8KÐ7LÈNÔ[nÐZoð p.Ü.AÐ-BÀ!ðEóð ô Ð/Ó0€FØ€~Üð;ó
ð 	
ô
 �V˜[¨$Ó/€FÜ˜VÐ%5°tÓ<€NÜ˜VÐ%5°tÓ<€Nð
 ˜&Ð!Ø˜~Ð.Ø˜~Ð.ð
÷áˆD�$ð
 ˆ<ò 	ð€Gñ ñ ÜØBÀ4Ç9Á9ÈWÓCUÐBVð W4Ü4GÐ3HÈÔWjÐVkð l*Ü*=Ð)>¸aðAó
ð 	
ô ØØ%Ø%ôð ùó!s   ÂCc                 ó   — t        «       } y ©N)rB   )Ú_s    r"   Ú _populate_finegrained_fp8_kernelrF   ‡   s   € ä$Ó&€AØr!   c                 ó>   — t        «       r
t        «        t        «       S rD   )r   rF   rB   r    r!   r"   Úload_finegrained_fp8_kernelrH   �   s   € ÜÔ!Ü(Ô*Ü'Ó)Ð)r!   c                ó   — | |z   dz
  |z  S )zCeiling division.r   r    )ÚaÚbs     r"   Ú_cdivrL   “   s   € à�‰E�A‰I˜!ÑÐr!   c	                ó\  — t        j                  | |||z  |¬«      }	t        j                  |	|	j	                  «       ¬«      }
t        |�t        ||«      nd|«      }|�t        ||«      nd}t        j                  | |||¬«      }t        j                  ||j	                  «       ¬«      }|
|fS )u×  Allocate `(weight, weight_scale_inv)` parameters for one expert projection.

    `weight_k_div` halves the K dim for FP4-packed storage (2 e2m1 values per byte).
    `sf_gran_n` / `sf_gran_k` set per-block (None â†’ per-row/per-tensor) SF granularity.
    `min_sf_out` floors the SF tensor's output dim â€” used by the fused gate_up
    projection to keep room for both halves (pass `2`) even when `proj_out < sf_gran_n`
    would otherwise collapse the SF dim to 1.
    ©Údtype©Úrequires_gradr   )r   ÚemptyÚnnÚ	ParameterÚis_floating_pointÚmaxrL   )Únum_expertsÚproj_outÚproj_inÚweight_dtypeÚsf_dtypeÚweight_k_divÚ	sf_gran_nÚ	sf_gran_kÚ
min_sf_outÚweight_tÚweightÚsf_outÚsf_inÚsf_tÚsfs                  r"   Ú_alloc_expert_projrf   ˜   s›   € ô& �{‰{˜;¨°'¸\Ñ2IÐQ]Ô^€HÜ�\‰\˜(°(×2LÑ2LÓ2NÔO€FÜ¨yÐ/D”�x Ô+È!ÈZÓX€FØ)2Ð)>ŒE�'˜9Ô%ÀA€EÜ�;‰;�{ F¨E¸ÔB€DÜ	�‰�d¨$×*@Ñ*@Ó*BÔ	C€BØ�2ˆ:Ðr!   Úoutput_dtypezv5.16)Úversionc                ó‚   — t        «       }|j                  | |||| j                  |¬«      }|�|j                  |«       |S )u0  Triton FP8/FP4 linear: fused act-quant + matmul, then optional bias add.

    ``activation_scale=None`` â†’ dynamic per-K-block scales (inline); set it for
    static per-tensor quant. ``weight_scale_inv`` accepts fp32 or UE8M0; the
    dispatcher routes FP4 (``int8``-packed) weights automatically.
    ©Úactivation_scale)rH   r0   rO   Úadd_)	Úinputra   Úweight_scale_invÚ
block_sizeÚbiasrk   rg   Úfinegrained_fp8Úoutputs	            r"   Úfinegrained_fp8_linearrs   ´   sP   € ô  2Ó3€OØ×#Ñ#ØØØØØ�‰Ø)ð $ó €Fð ÐØ�‰�DÔØ€Mr!   c                ó  — |xr® |du xr¨ |j                   j                  dk(  xr� t        j                  j	                  «       j
                  dk\  xr` |j                  t        j                  k(  xs |duxr |d   |d   cxk(  xr dk(  nc xr# t        j                  j                  dd«      d	k7  }|r	 t        | |||||¬
«      S t        | |||||«      S # t        $ r#}	t        j                  d|	› d�«       Y d}	~	Œ7d}	~	ww xY w)u  End-to-end FP8/FP4 linear used by `FP8Linear` and the eager `FP8Experts` loop.

    Dispatch order â€” both backends handle FP8 and FP4 weights with fp32 or UE8M0 scales:
      1. DeepGEMM (`deepgemm_fp8_fp4_linear`) â€” 3-6Ã— faster on the shapes it supports.
         Preferred for FP4, UE8M0 SFs, and 128Ã—128 block FP8.
      2. Triton finegrained-fp8 fallback â€” used when DeepGEMM is unavailable, when the
         caller passes ``activation_scale`` (DeepGEMM is dynamic-only), or for any
         shape DeepGEMM declined.

    Args:
        input: (..., K) bf16/fp16 activations.
        weight: (N, K) `float8_e4m3fn` or (N, K // 2) `int8` (FP4-packed).
        weight_scale_inv: per-block weight scales â€” `float32` (V3-style) or `float8_e8m0fnu`
            (V4-style; reinterpreted as int32 at the DeepGEMM kernel boundary).
        block_size: [block_n, block_k] for FP8 block-wise quant, or None/[N, K] for per-tensor.
            Ignored for FP4 weights (the kernel infers SF granularity from the dtype).
        bias: optional bias added to the matmul output.
        activation_scale: pass a per-tensor scalar to use static activation quant; leave `None`
            for dynamic (per-token) quant.
        allow_deepgemm: set ``False`` to force the Triton fallback for this call. Used when the
            model spans multiple CUDA devices in one process â€” DeepGEMM's cached kernels are bound
            to a single CUDA context and produce garbage across devices (see the multi-device guard
            in ``quantizer_finegrained_fp8.py``).
    NÚcudaé	   r   r   é€   Ú$TRANSFORMERS_DISABLE_DEEPGEMM_LINEARÚ0Ú1)ro   rk   rp   zDDeepGEMM unavailable for this call, falling back to Triton. Reason: zW Set `TRANSFORMERS_DISABLE_DEEPGEMM_LINEAR=1` to skip DeepGEMM for FP8 linear entirely.)Údevicer'   r   ru   Úget_device_propertiesÚmajorrO   Úint8ÚosÚenvironÚgetr   r=   ÚloggerÚwarning_oncers   )
rm   ra   rn   ro   rp   rk   rg   Úallow_deepgemmÚdeepgemm_preferredÚes
             r"   Ú
fp8_linearr‡   Ò   s3  € ðV 	ò 	OØ Ð$ò	Oà�M‰M×Ñ &Ñ(ò	Oô �J‰J×,Ñ,Ó.×4Ñ4¸Ñ9ò	Oð �\‰\œUŸZ™ZÑ'Òm¨J¸dÐ,BÒ,lÀzÐRSÁ}ÐXbÐcdÑXeÖGlÐilÔGlò		Oô
 �J‰J�N‰NÐAÀ3ÓGÈ3ÑNð ñ ð	Ü*ØØØ Ø%Ø!1Øôð ô  " %¨Ð1AÀ:ÈtÐUeÓfÐføô ò 	ô ×ÑØVÐWXÐVYð Zið i÷ñ ûð	ús   Â6C Ã	DÃ C>Ã>Dc                  óL   ‡ — e Zd ZdZ	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zdd„Zˆ xZS )Ú	FP8LinearFc                ó  •— t         ‰
| �  ||«       || _        || _        || _        t
        j                  j                  t        j                  ||t        ¬«      «      | _
        | j                  €>t        j                  t        j                  dt
        j                  ¬«      «      | _        n¥|dk(  r
t        «       nt
        j                  }|| j                  d   z   dz
  | j                  d   z  }|| j                  d   z   dz
  | j                  d   z  }	t        j                  t        j                  ||	|¬«      |j                  ¬«      | _        | j                  dk(  r>t        j                  t        j                  dt
        j                  ¬«      «      | _        n| j#                  dd «       | j                  r8t        j                  t        j                  | j$                  «      «      | _        y | j#                  d	d «       y )
NrN   ç      ð?Úue8m0r   r   rP   Ústaticrk   rp   )ÚsuperÚ__init__Úhas_biasro   Úactivation_schemer   rS   rT   rR   Ú
_FP8_DTYPEra   ÚtensorÚfloat32rn   r#   rU   rk   Úregister_parameterÚout_featuresrp   )ÚselfÚin_featuresr–   ro   r‘   Ú	scale_fmtr�   r[   Úscale_out_featuresÚscale_in_featuresÚ	__class__s             €r"   r�   zFP8Linear.__init__   s�  ø€ ô 	‰Ñ˜ lÔ3à ˆŒØ$ˆŒØ!2ˆÔÜ—h‘h×(Ñ(¬¯©°\À;ÔV`Ô)aÓbˆŒà�?‰?Ð"ä$&§L¡L´·±¸cÌÏÉÔ1WÓ$XˆDÕ!à-6¸'Ò-AÔ'Ô)ÄuÇ}Á}ˆHØ".°·±ÀÑ1CÑ"CÀaÑ"GÈDÏOÉOÐ\]ÑL^Ñ!^ÐØ!,¨t¯©¸qÑ/AÑ!AÀAÑ!EÈ$Ï/É/ÐZ[ÑJ\Ñ \ÐÜ$&§L¡LÜ—‘Ð.Ð0AÈÔRØ&×8Ñ8ô%ˆDÔ!ð
 ×!Ñ! XÒ-Ü$&§L¡L´·±¸cÌÏÉÔ1WÓ$XˆDÕ!à×#Ñ#Ð$6¸Ô=à�=Š=ÜŸ™¤U§[¡[°×1BÑ1BÓ%CÓDˆD�Ià×#Ñ# F¨DÕ1r!   c           	     ó\  — | j                   j                  «       dkD  r+t        j                  || j                   | j                  «      S t        | j                   «      }t        | j                  «      }t        |||| j                  | j                  | j                  | j                   ¬«      S )Nr   )ro   rk   rp   r„   )ra   Úelement_sizeÚFÚlinearrp   r   rn   r‡   ro   rk   Ú_deepgemm_disabled)r—   rm   ra   Ú	scale_invs       r"   ÚforwardzFP8Linear.forwardF  s‰   € Ø�;‰;×#Ñ#Ó%¨Ò)Ü—8‘8˜E 4§;¡;°·	±	Ó:Ð:ä˜$Ÿ+™+Ó&ˆÜ˜T×2Ñ2Ó3ˆ	äØØØØ—‘Ø!×2Ñ2Ø—‘Ø#×6Ñ6Ð6ô
ð 	
r!   ©NÚdynamicÚfloatF)r˜   Úintr–   r§   ro   útuple[int, int] | Noner‘   Ústrr™   r©   r�   Úbool)rm   útorch.TensorÚreturnr«   )r(   r3   r4   r¡   r�   r£   Ú__classcell__©rœ   s   @r"   r‰   r‰     s^   ø„ ð Ðð .2Ø!*Ø Øð$2àð$2ð ð$2ð +ð	$2ð
 ð$2ð ð$2ð õ$2÷L
r!   r‰   c                  óP   ‡ — e Zd ZdZ	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zdd„Zˆ xZS )ÚFP8GroupedLinearuï  FP8 drop-in for block-diagonal grouped linears.

    The underlying nn.Linear stores a single `(n_groups * out_per_group, in_per_group)`
    weight; logically that's `n_groups` independent `(out_per_group, in_per_group)`
    sub-matrices, each consuming a disjoint slice of the input's last-but-one dim.
    Forward expects input of shape `(..., n_groups, in_per_group)` and returns
    `(..., n_groups, out_per_group)` â€” same contract as the vanilla bf16 grouped
    linear it replaces.

    c                ó>   •— t         ‰| �  ||||||¬«       || _        y )N©r˜   r–   ro   r‘   r™   r�   )rŽ   r�   Ún_groups)	r—   Úin_features_per_groupr–   r³   ro   r‘   r™   r�   rœ   s	           €r"   r�   zFP8GroupedLinear.__init__d  s2   ø€ ô 	‰ÑØ-Ø%Ø!Ø/ØØð 	ô 	
ð !ˆ�r!   c                ó$  — |j                   d d }|j                   d   }| j                  j                  «       dkD  rí| j                  j                  | j                  d|«      j                  dd«      }|j                  d| j                  |«      j                  dd«      }t        j                  ||«      j                  dd«      } |j                  g |¢| j                  ‘d‘­Ž }| j                  r5|j                  | j                  j                  | j                  d«      «       |S t        | j                  «      }t        | j                  «      }|j                  | j                  d|«      }|j                  dd«      j                  d|«      }|j                  | j                  |j                  d«      | j                  z  |j                  d«      «      }|j                  d«      | j                  z  }t        j                   | j                  f||j"                  t        j$                  ¬«      }t        j&                  d| j                  dz   |j"                  t        j$                  ¬«      |z  }	t)        «       }
|
j+                  ||||	|| j,                  ¬«      } |j                  | j                  g|¢d‘­Ž j                  dd«      }| j                  r5|j                  | j                  j                  | j                  d«      «       |S )Néþÿÿÿéÿÿÿÿr   r   r   )r{   rO   ©ÚoffsetsÚtokens_per_expertro   )Úshapera   rž   Úviewr³   Ú	transposeÚreshaper   Úbmmr�   rl   rp   r   rn   ÚmovedimÚsizeÚfullr{   Úint32ÚarangerH   r2   ro   )r—   ÚxÚinput_shapeÚ
hidden_dimÚwÚyr¢   Útokens_per_grouprº   r¹   rq   s              r"   r£   zFP8GroupedLinear.forwardx  se  € Ø—g‘g˜c˜r�lˆØ—W‘W˜R‘[ˆ
à�;‰;×#Ñ#Ó%¨Ò)Ø—‘× Ñ  §¡°°JÓ?×IÑIÈ!ÈQÓOˆAØ—	‘	˜"˜dŸm™m¨ZÓ8×BÑBÀ1ÀaÓHˆAÜ—	‘	˜!˜Q“×)Ñ)¨!¨QÓ/ˆAØ�—	‘	Ð:˜;Ð:¨¯©Ð:°rÒ:ˆAØ�}Š}Ø—‘�t—y‘y—~‘~ d§m¡m°RÓ8Ô9ØˆHä�T—[‘[Ó!ˆÜ˜T×2Ñ2Ó3ˆ	à�F‰F�4—=‘= " jÓ1ˆØ�I‰I�b˜!Ó×$Ñ$ R¨Ó4ˆØ—N‘N 4§=¡=°)·.±.ÀÓ2CÀtÇ}Á}Ñ2TÐV_×VdÑVdÐefÓVgÓhˆ	àŸ6™6 !›9¨¯©Ñ5ÐÜ!ŸJ™J¨¯©Ð'7Ð9IÐRS×RZÑRZÔbg×bmÑbmÔnÐÜ—,‘,˜q $§-¡-°!Ñ"3¸A¿H¹HÌEÏKÉKÔXÐ[kÑkˆä5Ó7ˆØ×*Ñ*ØØØØØ/Ø—‘ð +ó 
ˆð ˆA�I‰I�d—m‘mÐ6 kÐ6°2Ò6×>Ñ>¸qÀ"ÓEˆØ�=Š=Ø�F‰F�4—9‘9—>‘> $§-¡-°Ó4Ô5Øˆr!   r¤   )r´   r§   r–   r§   r³   r§   ro   r¨   r‘   r©   r™   r©   r�   rª   )rÅ   r«   r¬   r«   )r(   r3   r4   r5   r�   r£   r­   r®   s   @r"   r°   r°   X  sd   ø„ ñ	ð  .2Ø!*Ø Øð!à"ð!ð ð!ð ð	!ð
 +ð!ð ð!ð ð!ð õ!÷($r!   r°   c                ó6  — | j                   dk(  rt        d«      ‚t        «       }|j                  d«      }|j                  d«      }|j                  d«      }|j	                  |d¬«      }|j                  d«      }	|j                  d«      }
|
| j                  k\  j                  d«      }t        | j                  r| j                  n| j                  «      }t        | j                  r| j                  n| j                  «      }t        | j                  «      }t        | j                  «      }|j!                  |||| j"                  |
¬«      }| j                  r| j%                  |«      }n| j'                  |«      }|j!                  |||| j"                  |
¬«      }||	j)                  |j*                  «      j                  d«      z  }|j-                  |d«       |j/                  |||«      j1                  d¬«      }|j)                  |j*                  «      S )	Nr�   z‘batched_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.r·   r   ©Údim)ro   Ú
expert_idsç        r   )r‘   ÚNotImplementedErrorrH   rÁ   Úrepeat_interleaver¾   rW   Ú	unsqueezer   Úhas_gateÚgate_up_projÚup_projÚgate_up_proj_scale_invÚup_proj_scale_invÚ	down_projÚdown_proj_scale_invr1   ro   Ú_apply_gateÚact_fnÚtorO   Úmasked_fill_r¼   Úsum)r—   Úhidden_statesÚtop_k_indexÚtop_k_weightsrq   Ú	num_top_kÚ
num_tokensrÇ   Úselected_hidden_statesÚsample_weightsrÎ   Úsentinel_maskÚ	weight_upÚweight_scale_upÚweight_downÚweight_scale_downrX   Úweighted_outÚfinal_hidden_statess                      r"   Úfp8_batched_mm_experts_forwardrí   Ÿ  s   € ð ×Ñ Ò)Ü!ðWó
ð 	
ô
 2Ó3€Oà× Ñ  Ó$€IØ×#Ñ# AÓ&€JØ×#Ñ# BÓ'€Jð +×<Ñ<¸YÈAÐ<ÓNÐØ"×*Ñ*¨2Ó.€NØ×$Ñ$ RÓ(€Jð
   4×#3Ñ#3Ñ3×>Ñ>¸rÓB€Mä¨d¯mªm˜×*Ò*ÀÇÁÓN€IÜ¸d¿mºm˜t×:Ò:ÐQU×QgÑQgÓh€OÜ˜4Ÿ>™>Ó*€KÜ  ×!9Ñ!9Ó:Ðð ×-Ñ-ØØØØ—?‘?Øð .ó €Hð ‡}‚}à×#Ñ# HÓ-‰ð —;‘;˜xÓ(ˆð ×-Ñ-ØØØØ—?‘?Øð .ó €Hð ˜n×/Ñ/°·±Ó?×IÑIÈ"ÓMÑM€Lð ×Ñ˜m¨SÔ1ð '×+Ñ+¨J¸	À:ÓN×RÑRÐWXÐRÓYÐà×!Ñ! -×"5Ñ"5Ó6Ð6r!   c                ó  — | j                   dk(  rt        d«      ‚t        «       }|j                  }|j	                  d«      }|j	                  d«      }|j	                  d«      }|j                  d«      }	|j                  d«      }
t        j                  |
«      \  }}|||z     }|	|   }|j                  dk(  r|j                  «       n|j                  «       }t        j                  || j                  d| j                  dz
  ¬«      }t        j                  |dt        j                  ¬«      }|| j                  k\  j                  d«      }t!        | j"                  r| j$                  n| j&                  «      }t!        | j"                  r| j(                  n| j*                  «      }t!        | j,                  «      }t!        | j.                  «      }|j1                  |||||| j2                  ¬	«      }| j"                  r| j5                  |«      }n| j7                  |«      }|j1                  |||||| j2                  ¬	«      }||j9                  |j:                  «      j                  d«      z  }|j=                  |d
«       t        j>                  |«      }t        j@                  |j	                  d«      |¬«      ||<   ||   }|jC                  |||«      jE                  d¬«      }|j9                  |j:                  «      S )Nr�   z‘grouped_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.r·   r   Úcpur   )ÚbinsÚminrV   )rÍ   rO   r¸   rÏ   )r{   rÌ   )#r‘   rÐ   rH   r{   rÁ   r¾   r   Úsortr'   r¦   r§   ÚhistcrW   ÚcumsumrÃ   rÒ   r   rÓ   rÔ   rÕ   rÖ   r×   rØ   rÙ   r2   ro   rÚ   rÛ   rÜ   rO   rÝ   Ú
empty_likerÄ   r¼   rÞ   )r—   rß   rà   rá   rq   r{   râ   rã   rÇ   rå   rÎ   Úexpert_ids_gÚpermÚselected_hidden_states_gÚsample_weights_gÚhistc_inputrº   r¹   ræ   rç   rè   ré   rê   rX   rë   Úinv_permrì   s                              r"   Úfp8_grouped_mm_experts_forwardrü   é  sÉ  € ð ×Ñ Ò)Ü!ðWó
ð 	
ô
 2Ó3€Oà×!Ñ!€FØ× Ñ  Ó$€IØ×#Ñ# AÓ&€JØ×#Ñ# BÓ'€Jð #×*Ñ*¨2Ó.€NØ×$Ñ$ RÓ(€Jô Ÿ™ JÓ/Ñ€L�$Ø,¨T°YÑ->Ñ?ÐØ% dÑ+Ðð
 +1¯+©+¸Ò*>�,×$Ñ$Ô&ÀL×DTÑDTÓDV€KÜŸ™ K°d×6FÑ6FÈAÐSW×ScÑScÐfgÑSgÔhÐÜ�l‰lÐ,°!¼5¿;¹;ÔG€Gð " T×%5Ñ%5Ñ5×@Ñ@ÀÓD€Mä¨d¯mªm˜×*Ò*ÀÇÁÓN€IÜ¸d¿mºm˜t×:Ò:ÐQU×QgÑQgÓh€OÜ˜4Ÿ>™>Ó*€KÜ  ×!9Ñ!9Ó:Ðð ×-Ñ-Ø ØØØØ+Ø—?‘?ð .ó €Hð ‡}‚}à×#Ñ# HÓ-‰ð —;‘;˜xÓ(ˆð ×-Ñ-ØØØØØ+Ø—?‘?ð .ó €Hð Ð.×1Ñ1°(·.±.ÓA×KÑKÈBÓOÑO€Lð ×Ñ˜m¨SÔ1ô ×Ñ Ó%€HÜ—\‘\ $§)¡)¨A£,°vÔ>€HˆT�NØ Ñ)€Lð '×+Ñ+¨J¸	À:ÓN×RÑRÐWXÐRÓYÐà×!Ñ! -×"5Ñ"5Ó6Ð6r!   c                  óš   ‡ — e Zd ZU dZddddœiZded<   	 	 	 	 	 d	 	 	 	 	 	 	 	 	 dˆ fd„Zdd	„Z	 	 	 	 	 	 	 	 dd
„Z	 d	 	 	 	 	 	 	 	 	 dd„Z	ˆ xZ
S )Ú
FP8ExpertsFÚdeepgemm_megamoeÚmegamoe_expertsÚmegamoe_router)Úmoe_tp_expertsÚ	ep_routerzdict[str, dict[str, str]]Ú_impl_tp_layer_overridesc                ój  •— t         ‰
| �  «        |du sJ d«       ‚|| _        || _        || _        || _        |j                  | _        || _        t        |dd«      | _
        t        |dd«      | _        t        |dd «      | _        t        |dd «      | _        t        t        |d	d
«         | _        t        |dd «      | _        t        |dd«      dk(  }|dk(  r
t%        «       nt&        j(                  }|rt&        j*                  |ddddœ}	nt,        ||�|d   nd |�|d   nd dœ}	| j                  rQt/        | j                  d| j                  z  | j                  fddi|	¤Ž\  | _        | _        | j5                  dd «       nKt/        | j                  | j                  | j                  fi |	¤Ž\  | _        | _        | j5                  dd «       t/        | j                  | j                  | j                  fi |	¤Ž\  | _        | _        | j5                  dd «       | j                  dk(  r�t?        j@                  t'        jB                  | j                  t&        j(                  ¬«      «      | _"        t?        j@                  t'        jB                  | j                  t&        j(                  ¬«      «      | _#        y y )NFzWFP8Experts does not support bias for now, please open an issue if you want this featureÚnum_local_expertsrW   Úmoe_intermediate_sizeÚintermediate_sizeÚswiglu_alphaÚswiglu_limitÚhidden_activationÚ
hidden_actÚexpert_dtypeÚfp8Úfp4rŒ   r   r   é    )rZ   r[   r\   r]   r^   r   )rZ   r[   r]   r^   r_   Úgate_up_proj_biasÚup_proj_biasÚdown_proj_biasr�   rN   )$rŽ   r�   Úconfigr�   rÓ   ro   Úhidden_sizerÇ   r‘   r,   rW   Úintermediate_dimr%   r	  r
  r   rÛ   Úlimitr#   r   r”   r~   r’   rf   rÔ   rÖ   r•   rÕ   r×   rØ   rÙ   rS   rT   ÚonesÚgate_up_proj_activation_scaleÚdown_proj_activation_scale)r—   r  ro   r‘   r™   r�   rÓ   Úis_fp4r[   Úalloc_kwargsrœ   s             €r"   r�   zFP8Experts.__init__[  sˆ  ø€ ô 	‰ÑÔà˜5Ñ ð 	
Øeó	
Ð ð ˆŒØ ˆŒØ ˆŒØ$ˆŒØ ×,Ñ,ˆŒØ!2ˆÔÜ& vÐ/BÀMÓRˆÔÜ +¨FÐ4KÐM`Ó aˆÔÜ# F¨N¸DÓAˆÔÜ# F¨N¸DÓAˆÔÜœ[¨Ð1DÀlÓSÑTˆŒÜ˜V ^°TÓ:ˆŒ
ô ˜ °Ó7¸5Ñ@ˆØ)2°gÒ)=Ô#Ô%Ä5Ç=Á=ˆÙä %§
¡
Ø$Ø !ØØñ‰Lô !+Ø$Ø.8Ð.D˜Z¨š]È$Ø.8Ð.D˜Z¨š]È$ñ	ˆLð �=Š=Ü=OØ× Ñ  ! d×&;Ñ&;Ñ";¸T¿_¹_ñ>ØYZð>Ø^jñ>Ñ:ˆDÔ˜tÔ:ð ×#Ñ#Ð$7¸Õ>ä3EØ× Ñ  $×"7Ñ"7¸¿¹ñ4ØLXñ4Ñ0ˆDŒL˜$Ô0ð ×#Ñ# N°DÔ9ä3EØ×Ñ˜dŸo™o¨t×/DÑ/Dñ4
ØHTñ4
Ñ0ˆŒ˜Ô0ð 	×ÑÐ 0°$Ô7à×!Ñ! XÒ-Ü13·±¼e¿j¹jÈ×IYÑIYÔaf×anÑanÔ>oÓ1pˆDÔ.Ü.0¯l©l¼5¿:¹:Àd×FVÑFVÔ^c×^kÑ^kÔ;lÓ.mˆDÕ+ð .r!   c                óð  — |j                  dd¬«      \  }}| j                  �q|j                  | j                  ¬«      }|j                  | j                   | j                  ¬«      }|t	        j
                  || j                  z  «      z  }|dz   |z  S | j                  �D|j                  | j                  ¬«      }|j                  | j                   | j                  ¬«      }| j                  |«      |z  S )Nr   r·   rÌ   )rV   ©rñ   rV   r‹   )Úchunkr	  Úclampr
  r   Úsigmoidr  rÛ   )r—   Úgate_upÚgateÚupÚglus        r"   rÚ   zFP8Experts._apply_gate¡  sÖ   € Ø—=‘= ¨�=Ó+‰ˆˆbØ×ÑÐ(à—:‘: $×"3Ñ"3�:Ó4ˆDØ—‘˜t×0Ñ0Ð0°d×6GÑ6G�ÓHˆBØœŸ™ t¨d×.?Ñ.?Ñ'?Ó@Ñ@ˆCØ˜‘H Ñ#Ð#Ø�Z‰ZÐ#Ø—:‘: $§*¡*�:Ó-ˆDØ—‘˜tŸz™z˜k¨t¯z©z�Ó:ˆBØ�{‰{˜4Ó  2Ñ%Ð%r!   c                ó  — t        j                  |t         j                  ¬«      }t        j                  «       5  t         j                  j
                  j                  || j                  dz   ¬«      }|j                  ddd«      }t        j                  |j                  d¬«      d«      j                  d¬	«      j                  d
«      }d d d «       D �]‡  }|| j                  k(  rŒt        j                  |   «      \  }}	||	   }
| j                  dk(  r| j                  |   nd }| j!                  |
| j"                  r| j$                  |   n| j&                  |   | j"                  r| j(                  |   n| j*                  |   |¬«      }| j"                  r| j-                  |«      n| j/                  |«      }| j                  dk(  r| j0                  |   nd }| j!                  || j2                  |   | j4                  |   |¬«      }||	|d f   }||j7                  |j8                  «      z  }|j;                  d|	|j7                  |j8                  «      «       �ŒŠ |j7                  |j8                  «      S # 1 sw Y   �Œ²xY w)NrN   r   )Únum_classesr   r   )r·   r¶   rÌ   F)Úas_tupler·   r�   rj   )r   Ú
zeros_liker”   Úno_gradrS   r   Úone_hotrW   ÚpermuteÚgreaterrÞ   Únonzeror¼   Úwherer‘   r  r    rÓ   rÔ   rÕ   rÖ   r×   rÚ   rÛ   r  rØ   rÙ   rÜ   rO   Ú
index_add_)r—   rß   rà   rá   rì   Úexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgate_up_act_scalerX   Údown_act_scaleÚrouting_weightsrë   s                   r"   r£   zFP8Experts.forward®  s^  € ô
 $×.Ñ.¨}ÄEÇMÁMÔRÐä�]‰]‹_ñ 	jÜŸ(™(×-Ñ-×5Ñ5°kÈt×O_ÑO_ÐbcÑOcÐ5ÓdˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸ™ {§¡¸8 Ó'DÀaÓH×PÑPÐZ_ÐPÓ`×eÑeÐfhÓiˆJ÷	jð
 %ó 	eˆJØ˜T×-Ñ-Ò-Øä#(§;¡;¨{¸:Ñ/FÓ#GÑ ˆI�yØ)¨)Ñ4ˆMàBF×BXÑBXÐ\dÒBd�×2Ñ2°:Ò>Ðjnð ð —{‘{ØØ15·²�×!Ñ! *Ò-ÀDÇLÁLÐQ[ÑD\Ø;?¿=º=�×+Ñ+¨JÒ7Èd×NdÑNdÐeoÑNpØ!2ð	 #ó ˆHð 6:·]²]�t×'Ñ'¨Ô1ÈÏÉÐT\ÓH]ˆHà?C×?UÑ?UÐYaÒ?a�×/Ñ/°
Ò;Ðgkð ð —{‘{ØØ—‘˜zÑ*Ø×(Ñ(¨Ñ4Ø!/ð	 #ó ˆHð ,¨I°yÀ$Ð,FÑGˆOØ# o×&8Ñ&8¸¿¹Ó&HÑHˆLØ×*Ñ*¨1¨i¸¿¹ÐI\×IbÑIbÓ9cÖdð7	eð8 #×%Ñ% m×&9Ñ&9Ó:Ð:÷C	jñ 	jús   ºBI;É;Jc                ó¢   — |j                  «       dkD  rt        j                  ||d «      S t        |||| j                  || j
                   ¬«      S )Nr   )rk   r„   )rž   rŸ   r    r‡   ro   r¡   )r—   rm   ra   rn   rk   s        r"   r    zFP8Experts.linearØ  sT   € ð ×ÑÓ  1Ò$Ü—8‘8˜E 6¨4Ó0Ð0äØØØØ�O‰OØ-Ø#×6Ñ6Ð6ô
ð 	
r!   )Nr¥   r¦   FT)
ro   r¨   r‘   r©   r™   r©   r�   rª   rÓ   rª   )r"  r«   r¬   r«   )rß   r«   rà   r«   rá   r«   r¬   r«   rD   )
rm   r«   ra   r«   rn   r«   rk   útorch.Tensor | Noner¬   r«   )r(   r3   r4   r¡   r  r6   r�   rÚ   r£   r    r­   r®   s   @r"   rþ   rþ   G  så   ø… ð Ðð 	Ø/Ø)ñ
ð;ÐÐ7ó ð .2Ø!*Ø ØØðDnð +ðDnð ð	Dnð
 ðDnð ðDnð õDnóL&ð(;Ø)ð(;Ø8Dð(;ØUað(;à	ó(;ð^ 15ð
àð
ð ð
ð 'ð	
ð
 .ð
ð 
÷
r!   rþ   c                  ó   — e Zd ZdZeeeedœZy)ÚFP8ExpertsInterfacez?Interface for registering custom FP8 experts forward functions.)Ú
batched_mmÚ
grouped_mmÚdeepgemmrÿ   N)	r(   r3   r4   r5   rí   rü   r   r   Ú_global_mappingr    r!   r"   r=  r=  ì  s   „ ÙIð 5Ø4Ø4ØDñ	�Or!   r=  c                ó´  — | j                  «       D �cg c]  }t        |t        t        f«      sŒ|‘Œ }}t	        «       }|D ]^  }t        |j                  «       d«      }|€Œ |j                  j                  dk(  sŒ:|j                  |j                  j                  «       Œ` t        |«      dk  ry|D ]	  }d|_        Œ t        j                  d«       yc c}w )a4  Internal, temporary helper (not public API): flag every FP8 module to skip DeepGEMM when the
    model spans >1 CUDA device in one process.

    DeepGEMM loads each kernel via `cuKernelGetFunction`, which binds the `CUfunction` handle to the
    CUDA context live at load time; driving that cached handle from another device launches it against
    the wrong context and produces garbage. (Build-time fix: compile DeepGEMM with
    `DG_JIT_USE_RUNTIME_API=1` for a context-free `cudaKernel_t` loader; until our wheel picks that up
    we avoid single-process multi-device.) Setting `_deepgemm_disabled` routes both the linear and
    experts paths through Triton/grouped_mm. A model that fits on one device keeps DeepGEMM even with
    other GPUs visible; TP/EP put one device per process, so this is a no-op there.
    Nru   r   TaB  This FP8 model spans multiple CUDA devices in one process; routing its FP8 linear and experts layers through Triton/grouped_mm instead of DeepGEMM (DeepGEMM's cached kernels are bound to a single CUDA context and corrupt across devices). Run tensor/expert parallel (one device per process) to use the faster DeepGEMM path.)ÚmodulesÚ
isinstancer‰   rþ   ÚsetÚnextÚ
parametersr{   r'   ÚaddÚindexÚlenr¡   r‚   rƒ   )ÚmodelÚmÚfp8_modulesÚcuda_devicesÚparams        r"   Ú!_disable_deepgemm_on_multi_devicerP  ú  sÄ   € ð $Ÿm™m›oÖX˜´¸AÄ	Ì:Ð?VÕ1W’1ÐX€KÐXÜ“5€LØò 1ˆÜ�Q—\‘\“^ TÓ*ˆØÑ §¡×!2Ñ!2°fÓ!<Ø×Ñ˜UŸ\™\×/Ñ/Õ0ð1ô ˆ<Ó˜AÒØØò $ˆØ#ˆÕð$ä
×Ñð	4õùò Ys
   “C¯Cc                ó<  — |j                   r| S d}| j                  «       D �]Ó  \  }}t        ||«      sŒd}t        j                  d«      5  |j                  d«      rƒt        |dd«      }t        |dd«      }	t        |d| j                  j                  «       «      }
t        t        t        |	|¬	«      } ||
|j                  |j                  |j                  |	|¬
«      }nöt        |«      t         j"                  u rPt%        |j&                  |j(                  |j                  |j                  |j                  |j*                  du¬«      }n‹t-        |t         j"                  «      rqdt        |«      j.                  v rZt1        |j&                  |j(                  |j2                  |j                  |j                  |j                  |j*                  du¬«      }|�| j5                  ||«       d}ddd«       �ŒÖ |st6        j9                  d«       | S # 1 sw Y   �ŒúxY w)a®  
    A helper function to replace all `torch.nn.Linear` modules by `FP8Linear` modules.

    Parameters:
        model (`torch.nn.Module`):
            Input model or `torch.nn.Module` as the function is run recursively.
        modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):
            Names of the modules to not convert. In practice we keep the `lm_head` in full precision for numerical stability reasons.
        quantization_config (`FineGrainedFP8Config`):
            The quantization config object that contains the quantization parameters.
        pre_quantized (`book`, defaults to `False`):
            Whether the model is pre-quantized or not
    FNÚmetaz.expertsrÓ   Tr�   r  )Úexperts_classÚexperts_interfacer�   rÓ   )r  ro   r‘   r™   r�   rÓ   r²   ÚGroupedLinear)r´   r–   r³   ro   r‘   r™   r�   z�You are loading your model using fp8 but no linear modules were found in your model. Please double check your model architecture.)Ú
dequantizeÚnamed_modulesr   r   r{   Úendswithr%   r  Úget_text_configr   rþ   ÚALL_FP8_EXPERTS_FUNCTIONSÚweight_block_sizer‘   r™   r'   rS   ÚLinearr‰   r˜   r–   rp   rD  r(   r°   r³   Úset_submoduler‚   Úwarning)rK  Úmodules_to_not_convertÚquantization_configÚpre_quantizedÚhas_been_replacedÚmodule_nameÚmoduleÚ
new_modulerÓ   r�   r  Ú	new_classs               r"   Úreplace_with_fp8_linearrg    sù  € ð" ×%Ò%ØˆàÐØ$×2Ñ2Ó4ó 4)Ñˆ�VÜ$ [Ð2HÔIØàˆ
Ü�\‰\˜&Ó!ñ /	)Ø×#Ñ# JÔ/Ü" 6¨:°tÓ<�Ü" 6¨:°uÓ=�Ü  ¨°5·<±<×3OÑ3OÓ3QÓR�Ü6Ü",Ü&?Ø%Ø%ô	�	ñ 'Ø!Ø2×DÑDØ&9×&KÑ&KØ1×;Ñ;Ø%Ø%ô‘
ô �f“¤§¡Ñ*ä&Ø &× 2Ñ 2Ø!'×!4Ñ!4Ø2×DÑDØ&9×&KÑ&KØ1×;Ñ;Ø#Ÿ[™[°Ð4ô‘
ô ˜F¤B§I¡IÔ.°?ÄdÈ6Ãl×F[ÑF[Ñ3[ô .Ø*0×*<Ñ*<Ø!'×!4Ñ!4Ø#Ÿ_™_Ø2×DÑDØ&9×&KÑ&KØ1×;Ñ;Ø#Ÿ[™[°Ð4ô�
ð Ð%Ø×#Ñ# K°Ô<Ø$(Ð!÷_/	)ñ /	)ð4)ñl Ü�‰ð<ô	
ð €L÷m/	)ñ /	)ús   ÁF!HÈH	c                  ó@   — e Zd ZdZd„ Zdd„Zd	d„Zd
d„Zedd„«       Z	y)ÚFp8Quantizez^
    A quantization operation that creates two tensors, weight and scale out of a weight.
    c                ó   — || _         y rD   ©Úhf_quantizer©r—   rl  s     r"   r�   zFp8Quantize.__init__p  ó
   € Ø(ˆÕr!   c                ó^  — d }| j                   j                  �kt        | j                   j                  t        «      r&| j                   j                  j	                  d«      }n!t        | j                   j                  dd «      }|€|j                  d   |j                  d   f}t        |«      S )Nr[  r¶   r·   )rl  r`  rD  Údictr�   r%   r»   Útuple)r—   Úvaluero   s      r"   Ú_resolve_block_sizezFp8Quantize._resolve_block_sizes  s“   € Øˆ
Ø×Ñ×0Ñ0Ð<Ü˜$×+Ñ+×?Ñ?ÄÔFØ!×.Ñ.×BÑB×FÑFÐGZÓ[‘
ä$ T×%6Ñ%6×%JÑ%JÐL_ÐaeÓf�
ØÐØŸ+™+ b™/¨5¯;©;°r©?Ð;ˆJÜ�ZÓ Ð r!   c                óž  — |j                   dk  r||iS | j                  |«      \  }}|j                  d   |j                  d   }}||z  dk7  s||z  dk7  r||iS |j                  d d }||z  }||z  }	|j                  }
|j                  t        j
                  «      } |j                  g |¢|‘|‘|	‘|‘­Ž }|j                  «       j                  d¬«      }t	        j                  |dkD  |t	        j                  |«      «      }t        |z  }t	        j                  |dkD  |t	        j                  |«      «      }d|z  j                  t        j
                  «      }| j                  j                  j                  dk(  r²t	        j                  d	t	        j                   t	        j"                  |j%                  t	        j&                  t        j
                  «      j(                  ¬
«      «      «      «      }|j                  t+        «       «      }d|j                  t        j
                  «      z  }|j-                  d«      j-                  d«      }||z  }t	        j$                  |t.        t        ¬«      j                  t0        «      }|j                  |
«      }|j3                  d«      r|j5                  dd«      d   dz   n|dz   }||||iS )Nr   r¶   r·   r   )éýÿÿÿr·   rÌ   r‹   rŒ   ç       @)rñ   ru  r  ú.weightú.r   ú.weight_scale_invÚ
_scale_inv)Úndimrs  r»   rÜ   r   r”   r¾   ÚabsÚamaxr/  Ú	ones_likeÚ_FP8_MAXrl  r`  r™   ÚpowÚceilÚlog2r   ÚfinfoÚtinyr#   rÒ   Ú_FP8_MINr’   rX  Úrsplit)r—   Úkeyrr  Úblock_mÚblock_nÚrowsÚcolsÚleading_shapeÚ
rows_tilesÚ
cols_tilesÚoriginal_shapeÚ
value_fp32ÚreshapedÚmax_absÚsafe_max_absÚscalesÚ
inv_scalesÚscales_broadcastÚscaledÚ	quantizedÚ	scale_keys                        r"   Ú_quantize_onezFp8Quantize._quantize_one~  sk  € ð �:‰:˜Š>Ø˜�<ÐØ×3Ñ3°EÓ:Ñˆ�Ø—[‘[ ‘_ e§k¡k°"¡oˆdˆØ�'‰>˜QÒ $¨¡.°AÒ"5Ø˜�<Ðð Ÿ™ C RÐ(ˆØ˜W‘_ˆ
Ø˜W‘_ˆ
ØŸ™ˆØ—X‘XœeŸm™mÓ,ˆ
à%�:×%Ñ%Ð_ }Ð_°jÐ_À'Ð_È:Ð_ÐW^Ò_ˆà—,‘,“.×%Ñ%¨(Ð%Ó3ˆÜ—{‘{ 7¨Q¡;°¼¿¹ÈÓ9QÓRˆä˜LÑ(ˆÜ—‘˜W q™[¨&´%·/±/À&Ó2IÓJˆØ˜F‘l×&Ñ&¤u§}¡}Ó5ˆ
ð ×Ñ×0Ñ0×:Ñ:¸gÒEÜŸ™ 3¬¯
©
´5·:±:¸j×>NÑ>NÔSX×S^ÑS^Ô_d×_lÑ_lÓSm×SrÑSrÐ>NÓ>sÓ3tÓ(uÓvˆJØ#Ÿ™Ô'7Ó'9Ó:ˆJØ˜:Ÿ=™=¬¯©Ó7Ñ7ˆFà!×+Ñ+¨BÓ/×9Ñ9¸"Ó=ÐØÐ,Ñ,ˆÜ—K‘K ¬H¼(ÔC×FÑFÄzÓRˆ	Ø×%Ñ% nÓ5ˆ	ØCFÇ<Á<ÐPYÔCZ�C—J‘J˜s AÓ& qÑ)Ð,?Ò?Ð`cÐfrÑ`rˆ	Ø�Y 	¨:Ð6Ð6r!   c                óª   — i }|j                  «       D ]=  \  }}t        |t        «      r|d   n|}|j                  | j	                  ||«      «       Œ? |S )Nr   )ÚitemsrD  ÚlistÚupdaterš  )r—   Ú
input_dictÚkwargsÚresultr‡  rr  r“   s          r"   ÚconvertzFp8Quantize.convert¦  sY   € ð +-ˆØ$×*Ñ*Ó,ò 	;‰JˆC�Ü!+¨E´4Ô!8�U˜1’X¸eˆFØ�M‰M˜$×,Ñ,¨S°&Ó9Õ:ð	;ð ˆr!   c                ó,   — t        | j                  «      S rD   )ÚFp8Dequantizerl  ©r—   s    r"   Ú
reverse_opzFp8Quantize.reverse_op°  s   € ä˜T×.Ñ.Ó/Ð/r!   N)rr  r«   r¬   ztuple[int, int])r‡  r©   rr  r«   r¬   údict[str, torch.Tensor])rŸ  r«   r¬   r§  ©r¬   r	   )
r(   r3   r4   r5   r�   rs  rš  r¢  Úpropertyr¦  r    r!   r"   ri  ri  k  s0   „ ñò)ó	!ó&7óPð ò0ó ñ0r!   ri  c                  óz   — e Zd ZdZd„ Zdd„ZdZdd„Z	 d	 	 	 	 	 	 	 dd„Zdd„Z		 	 d	 	 	 	 	 	 	 dd	„Z
edd
„«       Zy)r¤  u¬  Dequantize FP8 weights using their per-block ``weight_scale_inv``.

    Designed to run as the *first* op in any :class:`WeightConverter` chain when
    loading with ``dequantize=True`` â€” :meth:`update_weight_conversions` on the
    FP8 quantizer attaches it to each existing model-specific converter so that
    per-expert (weight, scale) pairs are folded into full-precision tensors before
    the chain's merge / concat ops collapse the per-expert structure.

    Pattern semantics
        Input ``input_dict`` carries one entry per source pattern; each value is a
        list of tensors (one per ``*`` match). For every weight pattern that has a
        sibling ``*.weight_scale_inv`` pattern in the dict, this op pairs them up by
        index, dequantizes per-pair, and emits the dequantized list under the
        original *weight* key. Scale entries are dropped from the output so the
        remaining ops only see weights.
    c                ó   — || _         y rD   rk  rm  s     r"   r�   zFp8Dequantize.__init__Ç  rn  r!   c                óª   — |j                  d«      }|r|d d n|}|j                  d«      r|d t        d«        dz   }n|dk(  rd}n|dz   }|r|dz   S |S )Nú$r·   rw  ry  ra   rn   rz  )rX  rJ  )r—   Úweight_patternÚanchoredÚbaseÚscales        r"   Ú_scale_pattern_forz Fp8Dequantize._scale_pattern_forÊ  so   € à!×*Ñ*¨3Ó/ˆÙ&.ˆ~˜c˜rÑ"°NˆØ�=‰=˜Ô#ØÐ*œC 	›N˜?Ð+Ð.AÑA‰EØ�XÒØ&‰Eà˜<Ñ'ˆEÙ&ˆu�s‰{Ð1¨EÐ1r!   )rÏ   g      à?r‹   g      ø?rv  g      @g      @g      @g       €g      à¿g      ð¿g      ø¿g       Àg      Àg      Àg      Àc                óÂ  — t        j                  | j                  t         j                  |j                  ¬«      }|j                  «       j                  t         j                  «      }|dz  j                  «       }|dz	  dz  j                  «       }t        j                  ||   ||   gd¬«      } |j                  g |j                  dd ¢d|j                  d   z  ‘­Ž S )uR   Two ``e2m1`` FP4 values per byte â†’ float32 tensor twice as wide on the last dim.)rO   r{   é   é   r·   rÌ   Nr   )r   r“   Ú_FP4_E2M1_LUTr”   r{   Ú
contiguousr¼   Úuint8ÚlongÚstackr¾   r»   )r—   ÚpackedÚlutÚu8ÚlowÚhighÚunpackeds          r"   Ú_unpack_fp4zFp8Dequantize._unpack_fp4Û  s¶   € ä�l‰l˜4×-Ñ-´U·]±]È6Ï=É=ÔYˆØ×ÑÓ ×%Ñ%¤e§k¡kÓ2ˆØ�C‰x�o‰oÓˆØ�q‘˜C‘×%Ñ%Ó'ˆÜ—;‘;  C¡¨#¨d©)Ð4¸"Ô=ˆØˆx×ÑÐI §¡¨c¨rÐ!2ÐI°A¸¿¹ÀRÑ8HÑ4HÒIÐIr!   Nc                ó   — t        t        dd «      }|j                  t        j                  k(  s|�!|j                  |k(  r| j	                  |«      }n|j                  t        j                  «      }|j                  dd  \  }}	 |j                  dd  \  }}	||z  s||	z  rt        d|› d|› d|› d|	› d�	«      ‚||z  }
||	z  }|€E|j                  j                  r|j                  «       dk\  r|j                  nt        j                  }|j                  t        j                  k(  r1|j                  t        j                  «      d	z
  j                  «       }n|j                  t        j                  «      }|j                  }|j                  d
||
|	|«      }|j                  d
||	«      j!                  d
«      j!                  d«      }||z  j                  |«      j                  |«      S # t        $ r	 d\  }}	Y �Œfw xY w)NÚfloat4_e2m1fn_x2r¶   )r   r   zWeight shape (r<   z) not divisible by scale grid (z).r   g     À_@r·   )r%   r   rO   r~   rÁ  rÜ   r”   r»   Ú	ExceptionÚ
ValueErrorrU   rž   Úbfloat16r¸  Úexp2r¾   rÒ   )r—   r˜  r”  rg   Ú	fp4_dtypeÚquantized_fp32rŠ  r‹  Ú
scale_rowsÚ
scale_colsrˆ  r‰  Ús_fp32r�  ÚqÚss                   r"   Ú_dequantize_onezFp8Dequantize._dequantize_oneä  sß  € ô
 œEÐ#5°tÓ<ˆ	Ø�?‰?œeŸj™jÒ(¨YÐ-BÀyÇÁÐZcÒGcØ!×-Ñ-¨iÓ8‰Nà&Ÿ\™\¬%¯-©-Ó8ˆNØ#×)Ñ)¨"¨#Ð.‰
ˆˆdð	*Ø%+§\¡\°"°#Ð%6Ñ"ˆJ˜
ð �*Ò  zÒ 1ÜØ    b¨¨Ð.MÈjÈ\ÐY[Ð\fÐ[gÐgiÐjóð ð ˜*Ñ$ˆØ˜*Ñ$ˆð
 Ðà &§¡× >Ò >À6×CVÑCVÓCXÐ\]ÒC]�—’Ôch×cqÑcqð ð �<‰<œ5Ÿ;™;Ò&Ø—i‘i¤§¡Ó.°Ñ6×<Ñ<Ó>‰Fà—Y‘YœuŸ}™}Ó-ˆFØ'×-Ñ-ˆØ×"Ñ" 2 z°7¸JÈÓPˆØ�N‰N˜2˜z¨:Ó6×@Ñ@ÀÓD×NÑNÈqÓQˆØ�A‘�z‰z˜,Ó'×/Ñ/°Ó?Ð?øô7 ò 	*à%)Ñ"ˆJ›
ð	*ús   ÂG+ Ç+G=Ç<G=c                ó^   — |�|€y t        ||«      \  }}t        ||d «      }t        |dd «      S )NrO   )r
   r%   )r—   rK  Úfull_layer_namerd  Útensor_namerO  s         r"   Ú_get_target_dtypezFp8Dequantize._get_target_dtype  s=   € Øˆ=˜OÐ3ØÜ2°5¸/ÓJÑˆ�Ü˜ ¨TÓ2ˆÜ�u˜g tÓ,Ð,r!   c                ó´  — | j                  ||«      }d|v r\|�|nd}|d   }t        |t        «      r|d   n|}d|v r2|d   }t        |t        «      r|d   n|}|| j                  |||¬«      iS ||iS i }	|j	                  «       D ]Ê  \  }
}d|
v sd|
v rŒ| j                  |
«      }||vr||	|
<   Œ*t        |t        «      r|n|g}||   }t        |t        «      r|n|g}t        |«      t        |«      k7  r't        d|
› dt        |«      › d	t        |«      › d
�«      ‚t        ||«      D ��cg c]  \  }}| j                  |||¬«      ‘Œ c}}|	|
<   ŒÌ |	S c c}}w )Nzweight$ra   r   rn   )rg   rk   z/Fp8Dequantize: weight/scale count mismatch for z (z weights vs z	 scales).)	rÓ  rD  r�  rÏ  rœ  r²  rJ  rÅ  Úzip)r—   rŸ  rÑ  rK  r   rg   Ú
target_keyr˜  r”  r¡  r‡  rr  r™  ÚweightsrÈ   rÎ  s                   r"   r¢  zFp8Dequantize.convert  s´  € ð ×-Ñ-¨e°_ÓEˆð
 ˜
Ñ"ð -<Ð,G™ÈXˆJØ" 9Ñ-ˆIÜ(2°9¼dÔ(C˜	 !šÈˆIØ! ZÑ/Ø#Ð$6Ñ7�Ü&0°¼Ô&>˜ šÀF�Ø" D×$8Ñ$8¸ÀFÐYeÐ$8Ó$fÐgÐgØ 	Ð*Ð*ð @BˆØ$×*Ñ*Ó,ò 	s‰JˆC�Ø! SÑ(Ð,>À#Ñ,EØØ×/Ñ/°Ó4ˆIØ 
Ñ*à#��s‘ØÜ)¨%´Ô6‘e¸U¸GˆGØ 	Ñ*ˆFÜ)¨&´$Ô7‘V¸f¸XˆFÜ�7‹|œs 6›{Ò*Ü ØEÀcÀUð KÜ˜G›�~ \´#°f³+°¸iðIóð ô ^aÐahÐjpÓ]q×rÑUYÐUVÐXY˜4×/Ñ/°°1À<Ð/ÕPÓrˆF�3ŠKð!	sð" ˆùó ss   Ä+Ec                ó,   — t        | j                  «      S rD   )ri  rl  r¥  s    r"   r¦  zFp8Dequantize.reverse_opF  s   € ô
 ˜4×,Ñ,Ó-Ð-r!   )r®  r©   r¬   r©   )r»  r«   r¬   r«   rD   )r˜  r«   r”  r«   rg   útorch.dtype | Noner¬   r«   )rK  útorch.nn.Module | NonerÑ  ú
str | Noner¬   rÙ  )NN)rŸ  ú,dict[str, list[torch.Tensor] | torch.Tensor]rÑ  rÛ  rK  rÚ  r¬   rÜ  r¨  )r(   r3   r4   r5   r�   r²  r¶  rÁ  rÏ  rÓ  r¢  r©  r¦  r    r!   r"   r¤  r¤  µ  sž   „ ñò")ó
2ð m€MóJð aeð+@Ø%ð+@Ø/;ð+@ØK]ð+@à	ó+@óZ-ð '+Ø(,ð	,à@ð,ð $ð,ð &ð	,ð 
6ó,ð\ ò.ó ñ.r!   r¤  )r¬   útorch.dtype)r¬   r/   )r¬   ÚNone)rJ   r§   rK   r§   r¬   r§   )r   NNr   )rW   r§   rX   r§   rY   r§   rZ   rÝ  r[   rÝ  r\   r§   r]   ú
int | Noner^   rß  r_   r§   r¬   z!tuple[nn.Parameter, nn.Parameter])NNNN)rm   r«   ra   r«   rn   r«   ro   úlist[int] | Nonerp   r;  rk   r;  rg   rÙ  r¬   r«   )NNNNT)rm   r«   ra   r«   rn   r«   ro   rà  rp   r;  rk   r;  rg   rÙ  r„   rª   r¬   r«   )
r—   ztorch.nn.Modulerß   r«   rà   r«   rá   r«   r¬   r«   )rK  z	nn.Moduler¬   rÞ  )NNF)r_  zlist[str] | None)LÚ
__future__r   Ú	functoolsr   Úcollections.abcr   Údataclassesr   r   Útorch.nnrS   r   rŸ   Úactivationsr   Úcore_model_loadingr	   Úquantizers.quantizers_utilsr
   r   Úutilsr   Úutils.deprecationr   Úutils.import_utilsr   r   r   r   r@  r   r   r   Úhub_kernelsr   Úmoer   r   Útensor_parallelr   Ú
get_loggerr(   r‚   Úfloat8_e4m3fnr’   rƒ  rñ   r…  rV   r  Úcacher#   r,   r/   rB   Ú_dynamoÚallow_in_graphrF   rH   rL   rf   rs   r‡   r\  r‰   r°   rí   rü   ÚModulerþ   r=  rZ  rP  rg  ri  r¤  r    r!   r"   ú<module>rõ     s�  ðõ #ã Û 	Ý $Ý !ã Ý Ý $å  Ý .ß UÝ Ý /÷ó ÷ñ õ
 *ß =Ý %ð 
ˆ×	Ñ	˜HÓ	%€ð × Ñ €
Øˆ5�;‰;�zÓ"×&Ñ&€Øˆ5�;‰;�zÓ"×&Ñ&€ð ‡�ò ó ð òGñ �$Ô÷ð ó ðð ‡�ò.ó ð.ðb ‡�×Ñòó ðó
*óð Ø Ø ØðØðàðð ðð ð	ð
 ðð ðð ðð ðð ðð 'óñ8 �¨Ô1ð
 $(Ø $Ø,0Ø'+ðØðàðð #ðð !ð	ð
 ðð *ðð %ðð òó 2ðñ: �¨Ô1ð
 $(Ø $Ø,0Ø'+ØðDgØðDgàðDgð #ðDgð !ð	Dgð
 ðDgð *ðDgð %ðDgð ðDgð òDgó 2ðDgôN;
�—	‘	ô ;
ô|D�yô DðNG7Ø
ðG7àðG7ð ðG7ð  ð	G7ð
 óG7ðT[7Ø
ð[7àð[7ð ð[7ð  ð	[7ð
 ó[7ô|b
�—‘ô b
ôJÐ*ô ñ 0Ó1Ð óð> ejðPØ#3óPôfG0�-ô G0ôTV.�Mõ V.r!   