Ë
    ÜÓ^jb&  ã                   ó¸   — d dl mZ ddlmZ erddlmZ ddlmZ ddlm	Z	m
Z
mZmZmZmZmZ ddlmZ  e«       rd d	lZ ej&                  e«      Z G d
„ de«      Zy	)é    )ÚTYPE_CHECKINGé   )ÚHfQuantizeré   )ÚPreTrainedModel)ÚFbgemmFp8Config)Úis_accelerate_availableÚis_fbgemm_gpu_availableÚis_kernels_availableÚis_torch_availableÚis_torch_cuda_availableÚis_torch_xpu_availableÚlogging)Úget_module_from_nameNc                   óª   ‡ — e Zd ZU dZdZded<   ˆ fd„Zd„ Zdd„Zd	d
de	de
fd„Zd	d
de	dddefˆ fd„Z	 	 dd„Zd„ Zd„ Zd„ Zede
fd„«       Zd„ Zˆ xZS )ÚFbgemmFp8HfQuantizerz/
    FP8 quantization using fbgemm kernels
    Fr   Úquantization_configc                 ó&   •— t        ‰| �  |fi |¤Ž y )N)ÚsuperÚ__init__)Úselfr   ÚkwargsÚ	__class__s      €úw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/quantizers/quantizer_fbgemm_fp8.pyr   zFbgemmFp8HfQuantizer.__init__1   s   ø€ Ü‰ÑÐ,Ñ7°Ó7ó    c                 óN  — t        «       st        «       st        d«      ‚t        «       rt        «       st        d«      ‚t        «       rt	        «       st        d«      ‚t        «       st        d«      ‚t        «       r3t        j                  j                  «       }|\  }}|dk  rt        d«      ‚|j                  d«      }|€t        j                  d«       y t        |t        «      r=| j                  s0d	|j!                  «       v sd
|j!                  «       v rt        d«      ‚y y y )Nz3Using fbgemm fp8 quantization requires a GPU or XPUz@Using FP8 fbgemm on XPU requires kernels (`pip install kernels`)züLoading an FP8 fbgemm quantized model on CUDA requires fbgemm-gpu libraryPlease install the latest version of fbgemm-gpu library by following : https://pytorch.org/FBGEMM/fbgemm_gpu-development/InstallationInstructions.html#fbgemm-gpu-install-librarieszWLoading an FP8 quantized model requires accelerate (`pip install --upgrade accelerate`)é	   zXFP8 quantized models is only supported on GPUs with compute capability >= 9.0 (e.g H100)Ú
device_mapzÛYou have loaded an FP8 model on CPU and have a CUDA/XPU device available, make sure to set your model on a GPU/XPU device in order to run your model. To remove this warning, pass device_map = 'cuda' or 'xpu' or 'auto'. ÚcpuÚdiskzòYou are attempting to load an FP8 model with a device_map that contains a CPU or disk device.This is not supported when the model is quantized on the fly. Please use a quantized checkpoint or remove the CPU or disk device from the device_map.)r   r   ÚImportErrorr   r
   r	   ÚtorchÚcudaÚget_device_capabilityÚ
ValueErrorÚgetÚloggerÚwarning_onceÚ
isinstanceÚdictÚpre_quantizedÚvalues)r   Úargsr   Úcompute_capabilityÚmajorÚ_r   s          r   Úvalidate_environmentz)FbgemmFp8HfQuantizer.validate_environment4   s&  € Ü&Ô(Ô1GÔ1IÜÐSÓTÐTÜ!Ô#Ô,@Ô,BÜÐ`ÓaÐaÜ"Ô$Ô-DÔ-FÜðFóð ô 'Ô(ÜØióð ô #Ô$Ü!&§¡×!AÑ!AÓ!CÐØ)‰HˆE�1Ø�qŠyÜ Ønóð ð —Z‘Z Ó-ˆ
ØÐÜ×ÑðSõô ˜
¤DÔ)Ø×%Ò%¨5°J×4EÑ4EÓ4GÑ+GÈ6ÐU_×UfÑUfÓUhÑKhÜ ðnóð ð LiÐ%ð *r   Úreturnc                 ó~   — |t         j                  k7  r)t        j                  d|› d�«       t         j                  }|S )NzSetting dtype to zP, but only bfloat16 is supported right now. Overwriting torch_dtype to bfloat16.)r"   Úbfloat16r'   r(   )r   Údtypes     r   Úupdate_dtypez!FbgemmFp8HfQuantizer.update_dtypeX   s9   € Ø”E—N‘NÒ"Ü×ÑØ# E 7Ð*zÐ{ôô —N‘NˆEØˆr   Úmodelr   Ú
param_namec                 ó®   — ddl m}m} t        ||«      \  }}t	        ||«      r| j
                  s|dk(  ryyt	        ||«      r| j
                  s|dk(  ryyy)Nr   ©ÚFbgemmFp8LinearÚFbgemmFp8Llama4TextExpertsÚbiasFT)Úintegrationsr;   r<   r   r)   r+   )r   r7   r8   r   r;   r<   ÚmoduleÚtensor_names           r   Úparam_needs_quantizationz-FbgemmFp8HfQuantizer.param_needs_quantization`   sZ   € ßNä2°5¸*ÓEÑˆ�ä�f˜oÔ.Ø×!Ò! [°FÒ%:ØàÜ�fÐ8Ô9Ø×!Ò! [°FÒ%:ØàØr   Úparamztorch.Tensorc                 óL   •— | j                  ||«      ryt        ‰| �	  |||«      S )z4Return the element size (in bytes) for `param_name`.r   )rA   r   Úparam_element_size)r   r7   r8   rB   r   s       €r   rD   z'FbgemmFp8HfQuantizer.param_element_sizeq   s*   ø€ à×(Ñ(¨°
Ô;àÜ‰wÑ)¨%°¸UÓCÐCr   c                 óæ   — ddl m} | j                  || j                  j                  |j
                  «      | _         ||| j                  | j                  | j                  |j                  ¬«      }y )Nr   )Úreplace_with_fbgemm_fp8_linear)Úmodules_to_not_convertr   r+   Útp_plan)r>   rF   Úget_modules_to_not_convertr   rG   Ú_keep_in_fp32_modulesr+   Ú_tp_plan)r   r7   r   rF   s       r   Ú$_process_model_before_weight_loadingz9FbgemmFp8HfQuantizer._process_model_before_weight_loadingx   se   € õ
 	Bà&*×&EÑ&EØ�4×+Ñ+×BÑBÀE×D_ÑD_ó'
ˆÔ#ñ /ØØ#'×#>Ñ#>Ø $× 8Ñ 8Ø×,Ñ,Ø—N‘Nô
‰r   c                 óÖ   — ddl m}m} |j                  «       D ]M  }t	        |||f«      sŒt        |d«      sŒ|j                  j                  | j                  j                  «       ŒO |S )zá
        Force update the input scale upper bound after weight loading and device dispatch are complete.
        This resolves issues where persistent buffers are zeroed out or overwritten during the loading process.
        r   r:   Úinput_scale_ub)
Úintegrations.fbgemm_fp8r;   r<   Úmodulesr)   ÚhasattrrN   Úfill_r   Úactivation_scale_ub)r   r7   r   r;   r<   Úms         r   Ú#_process_model_after_weight_loadingz8FbgemmFp8HfQuantizer._process_model_after_weight_loading‹   sa   € ÷
 	Zà—‘“ò 	YˆAÜ˜!˜oÐ/IÐJÕKÜ˜1Ð.Õ/à×$Ñ$×*Ñ*¨4×+CÑ+C×+WÑ+WÕXð		Yð
 ˆr   c                 ó  — d|j                   j                  v rni dd“dd“dd“dd“dd“dd“d	d
“dd“dd“dd“dd“dd“dd“dd“dd
“dd“dd“ddd
ddddœ¥}|j                  «       �||j                  «       _        |S ||_        |S |S )NÚLlama4z layers.*.self_attn.q_proj.weightÚcolwisez&layers.*.self_attn.q_proj.weight_scalez layers.*.self_attn.k_proj.weightz&layers.*.self_attn.k_proj.weight_scalez layers.*.self_attn.v_proj.weightz&layers.*.self_attn.v_proj.weight_scalez layers.*.self_attn.o_proj.weightÚrowwisezlayers.*.input_layernorm.weightÚsequence_parallelz(layers.*.post_attention_layernorm.weightznorm.weightz4layers.*.feed_forward.shared_expert.gate_proj.weightz:layers.*.feed_forward.shared_expert.gate_proj.weight_scalez2layers.*.feed_forward.shared_expert.up_proj.weightz8layers.*.feed_forward.shared_expert.up_proj.weight_scalez4layers.*.feed_forward.shared_expert.down_proj.weightz0layers.*.feed_forward.experts.*.gate_proj.weightz6layers.*.feed_forward.experts.*.gate_proj.weight_scaleÚpacked_rowwise)z.layers.*.feed_forward.experts.*.up_proj.weightz4layers.*.feed_forward.experts.*.up_proj.weight_scalez0layers.*.feed_forward.experts.*.down_proj.weightz*layers.*.feed_forward.experts.gate_up_projz0layers.*.feed_forward.experts.gate_up_proj_scalez'layers.*.feed_forward.experts.down_proj)r   Ú__name__Úget_text_configÚbase_model_tp_plan)r   ÚconfigÚ	text_plans      r   Úupdate_tp_planz#FbgemmFp8HfQuantizer.update_tp_plan™   sP  € Ø�v×'Ñ'×0Ñ0Ñ0ð!ð 3°Ið	!ð
 9¸)ð!ð 3°Ið!ð 9¸)ð!ð 3°Ið!ð 9¸)ð!ð 3°Ið!ð 2Ð3Fð!ð ;Ð<Oð!ð Ð2ð!ð$ GÈ	ð%!ð& MÈið'!ð( EÀið)!ð* KÈIð+!ð, GÈ	ð-!ð. CÀIð/!ð0 IÈ)ð1!ð2 CLØHQØDMð ?OØDTØ;DòA!ˆIðD ×%Ñ%Ó'Ð3Ø>G�×&Ñ&Ó(Ô;ð ˆMð -6�Ô)ØˆMàˆr   c                  ó   — y)NT© ©r   s    r   Úis_serializablez$FbgemmFp8HfQuantizer.is_serializableÅ   s   € Ør   c                  ó   — y)NFrc   rd   s    r   Úis_trainablez!FbgemmFp8HfQuantizer.is_trainableÈ   s   € àr   c                 ó   — ddl m}  || «      S )Nr   )ÚFbgemmFp8Quantize)rO   ri   )r   ri   s     r   Úget_quantize_opsz%FbgemmFp8HfQuantizer.get_quantize_opsÌ   s   € Ý?á  Ó&Ð&r   )r5   útorch.dtyper2   rk   )r7   r   )r\   Ú
__module__Ú__qualname__Ú__doc__Úrequires_calibrationÚ__annotations__r   r1   r6   ÚstrÚboolrA   ÚfloatrD   rL   rU   ra   re   Úpropertyrg   rj   Ú__classcell__)r   s   @r   r   r   )   s¨   ø… ñð !ÐØ*Ó*ô8ò"óHðÐ.?ð ÈSð Ð_có ð"DÐ(9ð DÀsð DÐSað DÐfkõ Dð
à ó
ò&ò*òXð ð˜dò ó ðö'r   r   )Útypingr   Úbaser   Úmodeling_utilsr   Úutils.quantization_configr   Úutilsr	   r
   r   r   r   r   r   Úquantizers_utilsr   r"   Ú
get_loggerr\   r'   r   rc   r   r   ú<module>r}      sW   ðõ !å ñ Ý0Ý;÷÷ ñ õ 3ñ ÔÛà	ˆ×	Ñ	˜HÓ	%€ôf'˜;õ f'r   