Ë
    ÜÓ^jp6  ã                   ó¬   — d dl mZ ddlmZmZmZmZ ddlmZ ddl	m
Z
  e«       rd dlZerddlmZ dd	lmZ  ej                   e«      Z G d
„ de«      Zy)é    )ÚTYPE_CHECKINGé   )Úis_accelerate_availableÚis_torch_availableÚis_torch_xpu_availableÚloggingé   )ÚHfQuantizer)Úget_module_from_nameN)ÚPreTrainedModel)ÚFineGrainedFP8Configc                   óÌ   ‡ — e Zd ZU dZdZded<   ˆ fd„Zd„ Zddd	ed
e	fd„Z
ddd	eddd
efˆ fd„Zdd„Z	 	 dd„Zd„ Zd„ Zd„ Zed
e	fd„«       Zed
e	fd„«       Zd„ Zd„ Zd„ Zˆ xZS )ÚFineGrainedFP8HfQuantizerz†
    FP8 quantization implementation supporting both standard and MoE models.
    Supports both e4m3fn formats based on platform.
    Fr   Úquantization_configc                 ó&   •— t        ‰| �  |fi |¤Ž y )N)ÚsuperÚ__init__)Úselfr   ÚkwargsÚ	__class__s      €ú|/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/quantizers/quantizer_finegrained_fp8.pyr   z"FineGrainedFP8HfQuantizer.__init__   s   ø€ Ü‰ÑÐ,Ñ7°Ó7ó    c                 ó.  — t        «       st        d«      ‚| j                  j                  ry t        j
                  j                  «       sHt        «       s>| j                  r't        j                  d«       d| j                  _        y t        d«      ‚t        j
                  j                  «       r`t        j
                  j                  «       }|\  }}|dk  s
|dk(  r3|dk  r.t        j                  d|› d|› d	�«       d| j                  _        y |j                  d
«      }|€t        j                  d«       y t        |t        «      rL| j                  s?t!        |«      dkD  r0d|j#                  «       v sd|j#                  «       v rt%        d«      ‚y y y y )NzMLoading an FP8 quantized model requires accelerate (`pip install accelerate`)z„Using FP8 quantized models requires a GPU or XPU, we will default to dequantizing the model to bf16 since no GPU or XPU is availableTzANo GPU or XPU found. A GPU or XPU is needed for FP8 quantization.é   é	   ziFP8 quantized models is only supported on GPUs with compute capability >= 8.9 (e.g 4090/H100), actual = `ú.zƒ`. We will default to dequantizing the model to bf16. Feel free to use a different quantization method like bitsandbytes or torchaoÚ
device_mapz×You have loaded an FP8 model on CPU and have a CUDA or XPU device available, make sure to set your model on a GPU or XPU device in order to run your model. To remove this warning, pass device_map = 'cuda' or 'xpu'. r	   ÚcpuÚdiskzìYou are attempting to load an FP8 model with a device_map that contains a cpu/disk device.This is not supported when the model is quantized on the fly. Please use a quantized checkpoint or remove the cpu/disk device from the device_map.)r   ÚImportErrorr   Ú
dequantizeÚtorchÚcudaÚis_availabler   Úpre_quantizedÚloggerÚwarning_onceÚRuntimeErrorÚget_device_capabilityÚgetÚ
isinstanceÚdictÚlenÚvaluesÚ
ValueError)r   Úargsr   Úcompute_capabilityÚmajorÚminorr   s          r   Úvalidate_environmentz.FineGrainedFP8HfQuantizer.validate_environment   s…  € Ü&Ô(ÜÐmÓnÐnà×#Ñ#×.Ò.Øä�z‰z×&Ñ&Ô(Ô1GÔ1IØ×!Ò!Ü×#Ñ#ð [ôð 7;�×(Ñ(Ô3Øä"Ð#fÓgÐgä�:‰:×"Ñ"Ô$Ü!&§¡×!AÑ!AÓ!CÐØ-‰LˆE�5Ø˜’	˜u¨šz¨e°aªiÜ×#Ñ#ð#Ø#( '¨¨5¨'ð 2Zð[ôð
 7;�×(Ñ(Ô3Øà—Z‘Z Ó-ˆ
ØÐÜ×Ñð6õô
 ˜
¤DÔ)à×&Ò&Ü˜
“O aÒ'Ø˜j×/Ñ/Ó1Ñ1°V¸z×?PÑ?PÓ?RÑ5Rä ðkóð ð 6Sð (ð 'ð *r   Úmodelr   Ú
param_nameÚreturnc                 ót   — ddl m}m} t        ||«      \  }}t	        |||f«      r| j
                  s|dk(  ryyy)Nr   )Ú
FP8ExpertsÚ	FP8LinearÚbiasFT)Úintegrations.finegrained_fp8r9   r:   r   r+   r%   )r   r5   r6   r   r9   r:   ÚmoduleÚtensor_names           r   Úparam_needs_quantizationz2FineGrainedFP8HfQuantizer.param_needs_quantizationN   s>   € çHä2°5¸*ÓEÑˆ�Ü�f˜y¨*Ð5Ô6Ø×!Ò! [°FÒ%:ØàØr   Úparamztorch.Tensorc                 óL   •— | j                  ||«      ryt        ‰| �	  |||«      S )z4Return the element size (in bytes) for `param_name`.r	   )r?   r   Úparam_element_size)r   r5   r6   r@   r   s       €r   rB   z,FineGrainedFP8HfQuantizer.param_element_sizeZ   s*   ø€ à×(Ñ(¨°
Ô;àÜ‰wÑ)¨%°¸UÓCÐCr   c                 óâ   — | j                   j                  }|syddlm}  ||«      }g }|D ]0  }|}|D ]  }|j	                  |«      \  }}	Œ |j                  |«       Œ2 || j                   _        y)u.  Rewrite the skip-list to the model's own module tree.
        For models that were already released, if they have a list of modules to not quantize
        we need to apply the weight renaming / weight conversion opÃ©rations to get the actual
        layer name of the model in `transformers`.
        Nr   )Úget_model_conversion_mapping)r   Úmodules_to_not_convertÚconversion_mappingrD   Úrename_source_keyÚappend)
r   r5   ÚskiprD   Ú	renamingsÚremappedÚnameÚrenamedÚrenameÚ_s
             r   Ú!_normalize_modules_to_not_convertz;FineGrainedFP8HfQuantizer._normalize_modules_to_not_converta   s�   € ð ×'Ñ'×>Ñ>ˆÙØåEá0°Ó7ˆ	ØˆØò 	%ˆDØˆGØ#ò ?�Ø#×5Ñ5°gÓ>‘
�™ð?à�O‰O˜GÕ$ð		%ð
 ;Cˆ× Ñ Õ7r   c                 óò   — ddl m} | j                  |«       | j                  || j                  j
                  |j                  «      | _         ||| j
                  | j                  | j                  ¬«      }y )Nr   )Úreplace_with_fp8_linear)rE   r   r%   )r<   rR   rP   Úget_modules_to_not_convertr   rE   Ú_keep_in_fp32_modulesr%   )r   r5   r   rR   s       r   Ú$_process_model_before_weight_loadingz>FineGrainedFP8HfQuantizer._process_model_before_weight_loadingv   sl   € õ
 	Kà×.Ñ.¨uÔ5Ø&*×&EÑ&EØ�4×+Ñ+×BÑBÀE×D_ÑD_ó'
ˆÔ#ñ (ØØ#'×#>Ñ#>Ø $× 8Ñ 8Ø×,Ñ,ô	
‰r   c           
      ó  — | j                   j                  dk(  rÛddlm}  |«       }|j	                  «       D ��cg c]5  \  }}|j                  d«      r|j                  t        j                  k(  r|‘Œ7 }}}|D ]y  }|j                  d«      \  }}	}
|j                  |«      }t        ||
«      }t        ||
t        j                  j                  |j                  j!                  |«      d¬«      «       Œ{ ddlm}  ||«       |S c c}}w )	NÚue8m0r   )Ú_get_ue8m0_dtypeÚ
_scale_invr   F)Úrequires_grad)Ú!_disable_deepgemm_on_multi_device)r   Ú	scale_fmtr<   rX   Únamed_parametersÚendswithÚdtyper"   Úfloat32Ú
rpartitionÚget_submoduleÚgetattrÚsetattrÚnnÚ	ParameterÚdataÚtor[   )r   r5   r   rX   rW   rL   r@   Úfloat32_scalesÚmodule_namerO   Úattrr=   Úscaler[   s                 r   Ú#_process_model_after_weight_loadingz=FineGrainedFP8HfQuantizer._process_model_after_weight_loading‰   sô   € ð
 ×#Ñ#×-Ñ-°Ò8ÝGá$Ó&ˆEð $)×#9Ñ#9Ó#;÷á�D˜%Ø—=‘= Ô.°5·;±;Ä%Ç-Á-Ò3Oò ðˆNñ ð
 'ò e�Ø'+§¡°sÓ';Ñ$�˜Q Ø×,Ñ,¨[Ó9�Ü ¨Ó-�Ü˜ ¤e§h¡h×&8Ñ&8¸¿¹¿¹ÀuÓ9MÐ]bÐ&8Ó&cÕdð	eõ 	Uá)¨%Ô0Øˆùó!s   º:Dc                 óŽ  — d|j                   j                  v rdddddddddddddddœ}||_        ddlm} t        |dd «      }|j                  j                  |«      }|r^dD ]Y  }t        ||d «      xs i }|j                  «       D ��	ci c]  \  }}	||j                  |	|	«      “Œ }
}}	|
|k7  sŒMt        |||
«       Œ[ |S c c}	}w )	NÚQwen3ÚcolwiseÚrowwise)z layers.*.self_attn.q_proj.weightz*layers.*.self_attn.q_proj.weight_scale_invz layers.*.self_attn.k_proj.weightz*layers.*.self_attn.k_proj.weight_scale_invz layers.*.self_attn.v_proj.weightz*layers.*.self_attn.v_proj.weight_scale_invz layers.*.self_attn.o_proj.weightz*layers.*.self_attn.o_proj.weight_scale_invzlayers.*.mlp.gate_proj.weightz'layers.*.mlp.gate_proj.weight_scale_invzlayers.*.mlp.up_proj.weightz%layers.*.mlp.up_proj.weight_scale_invzlayers.*.mlp.down_proj.weightz'layers.*.mlp.down_proj.weight_scale_invr   )r9   Ú_experts_implementation)Úbase_model_tp_planÚbase_model_ep_plan)
r   Ú__name__rs   r<   r9   rc   Ú_impl_tp_layer_overridesr*   Úitemsrd   )r   ÚconfigÚ	text_planr9   ÚimplÚlayer_overridesÚ	plan_attrÚ	base_planÚkÚvÚupdated_plans              r   Úupdate_tp_planz(FineGrainedFP8HfQuantizer.update_tp_plan¤   sñ   € Ø�f×&Ñ&×/Ñ/Ñ/à4=Ø>GØ4=Ø>GØ4=Ø>GØ4=Ø>GØ1:Ø;DØ/8Ø9BØ1:Ø;DñˆIð" )2ˆFÔ%õ 	>ä�vÐ8¸$Ó?ˆØ$×=Ñ=×AÑAÀ$ÓGˆÙØIò =�	Ü# F¨I°tÓ<ÒBÀ�	ØIRÏÉÓIZ×[ÁÀÀA  ?×#6Ñ#6°q¸!Ó#<Ñ <Ð[�Ñ[Ø 9Ó,Ü˜F I¨|Õ<ð	=ð ˆùó	  \s   Â
Cc                  ó   — y©NT© ©r   s    r   Úis_serializablez)FineGrainedFP8HfQuantizer.is_serializableÊ   s   € Ør   c                  ó   — y)NFr„   r…   s    r   Úis_trainablez&FineGrainedFP8HfQuantizer.is_trainableÍ   s   € àr   c                  ó   — yrƒ   r„   r…   s    r   Úis_compileablez(FineGrainedFP8HfQuantizer.is_compileableÑ   s   € àr   c                 ó   — ddl m}  || «      S )Nr   )ÚFp8Quantize)r<   rŒ   )r   rŒ   s     r   Úget_quantize_opsz*FineGrainedFP8HfQuantizer.get_quantize_opsÕ   s   € Ý>á˜4Ó Ð r   c                 óŒ   — ddl m} ddlm} | j                  r+| j
                  j                  r |g d¢d || «      g¬«      gS g S )Nr   )ÚWeightConverter©ÚFp8Dequantize)zweight$Úweight_scale_invÚactivation_scaleÚweight©Úsource_patternsÚtarget_patternsÚ
operations)Úcore_model_loadingr�   r<   r‘   r%   r   r!   )r   r�   r‘   s      r   Úget_weight_conversionsz0FineGrainedFP8HfQuantizer.get_weight_conversionsÚ   sK   € Ý8Ý@à×Ò $×":Ñ":×"EÒ"Eñ  Ú$WØ$,Ù -¨dÓ 3Ð4ôðð ð ˆ	r   c           	      ó  — ddl m}m} ddlm}  |dd¬«      }|gt        |«      z   }| j                  r| j                  j                  s|| j                  «       z   S g }|D ]ö  }t        ||«      s|j                  |«       Œ!|j                  D �cg c]  }|j                  d«      sŒ|‘Œ }	}|	r˜|	D �cg c]  }|dz   ‘Œ	 }
}|	D �cg c]  }|d	t        d«        d
z   ‘Œ }}|j                  D �cg c]  }|j                  d«      rŒ|‘Œ }}|
|z   |z   } || «      gt        |j                  «      z   } |||j                   |¬«      }|j                  |«       Œø |j#                  | j                  «       «       |S c c}w c c}w c c}w c c}w )uÕ  When loading with ``dequantize=True``, attach an :class:`Fp8Dequantize` op to
        every existing :class:`WeightConverter` so that per-block scales are folded into
        the weight *before* any later merge/concat ops collapse the per-expert structure.

        For each model-supplied converter that has a ``.weight`` source, we:
          1. anchor the existing weight patterns with ``$`` so they don't accidentally
             also match the ``.weight_scale_inv`` keys (the regex is searched, so the
             unanchored prefix would match both, sending scales to the wrong bucket);
          2. add anchored ``*.weight_scale_inv`` sources next to each weight pattern so
             the loader collects scale tensors alongside the weight tensors into the
             *same* converter bucket (both keys rewrite to the same target);
          3. prepend a fresh :class:`Fp8Dequantize` op so dequant runs first, before
             any merge/concat collapses the per-expert structure.

        The generic ``weight$ + weight_scale_inv â†’ weight`` converter from
        :meth:`get_weight_conversions` is still appended at the end as a fallback for
        plain ``nn.Linear`` weights with no model-specific converter.
        r   )r�   ÚWeightRenamingr�   z^(.+)\.scale$z\1.weight_scale_inv)r–   r—   z.weightú$Nz.weight_scale_inv$r•   )r™   r�   rœ   r<   r‘   Úlistr%   r   r!   rš   r+   rH   r–   r^   r-   r˜   Ú_original_target_patternsÚextend)r   Úweight_conversionsr�   rœ   r‘   Úscale_renameÚupdatedÚconvÚpÚweight_sourcesÚanchored_weightÚscale_sourcesÚotherÚnew_sourcesÚnew_opss                  r   Úupdate_weight_conversionsz3FineGrainedFP8HfQuantizer.update_weight_conversionsê   sŽ  € ÷& 	IÝ@ñ &Ð6FÐXnÔoˆØ*˜^¬dÐ3EÓ.FÑFÐà×"Ò" t×'?Ñ'?×'JÒ'JØ%¨×(CÑ(CÓ(EÑEÐEàˆØ&ò 	!ˆDô ˜d OÔ4Ø—‘˜tÔ$ØØ)-×)=Ñ)=ÖW AÀÇÁÈIÕAVšaÐWˆNÐWÙØ4BÖ"C¨q 1 s£7Ð"C�Ð"CØVdÖ eÐQR Ð#4¤c¨)£n _Ð!5Ð8LÓ!LÐ e�Ð eØ$(×$8Ñ$8ÖV˜qÀÇ
Á
È9Õ@UšÐV�ÐVØ-°Ñ=ÀÑE�Ù(¨Ó.Ð/´$°t·±Ó2GÑG�Ù&Ø$/Ø$(×$BÑ$BØ&ô�ð
 �N‰N˜4Õ ð%	!ð( 	�‰�t×2Ñ2Ó4Ô5Øˆùò Xùâ"CùÚ eùÚVs$   ÂE;Â'E;Â3F ÃFÃ.F
ÄF
)r5   r   )ru   Ú
__module__Ú__qualname__Ú__doc__Úrequires_calibrationÚ__annotations__r   r4   ÚstrÚboolr?   ÚfloatrB   rP   rU   rm   r�   r†   Úpropertyrˆ   rŠ   r�   rš   r¬   Ú__classcell__)r   s   @r   r   r      sÎ   ø… ñð
 !ÐØ/Ó/ô8ò.ð`
Ð.?ð 
ÈSð 
Ð_có 
ðDÐ(9ð DÀsð DÐSað DÐfkõ DóCð*
à ó
ò&ò6$òLð ð˜dò ó ðð ð ò ó ðò!ò
ö 7r   r   )Útypingr   Úutilsr   r   r   r   Úbaser
   Úquantizers_utilsr   r"   Úmodeling_utilsr   Úutils.quantization_configr   Ú
get_loggerru   r&   r   r„   r   r   ú<module>r¾      sH   ðÝ  ç `Ó `Ý Ý 2ñ ÔÛáÝ0Ý@à	ˆ×	Ñ	˜HÓ	%€ôO õ Or   