
    ^jp6                         d dl mZ ddlmZmZmZmZ ddlmZ ddl	m
Z
  e       rd dlZerddlmZ dd	lmZ  ej                   e      Z G d
 de      Zy)    )TYPE_CHECKING   )is_accelerate_availableis_torch_availableis_torch_xpu_availablelogging   )HfQuantizer)get_module_from_nameN)PreTrainedModel)FineGrainedFP8Configc                        e Zd ZU dZdZded<    fdZd Zddd	ed
e	fdZ
ddd	eddd
ef fdZddZ	 	 ddZd Zd Zd Zed
e	fd       Zed
e	fd       Zd Zd Zd Z xZS )FineGrainedFP8HfQuantizerz
    FP8 quantization implementation supporting both standard and MoE models.
    Supports both e4m3fn formats based on platform.
    Fr   quantization_configc                 &    t        |   |fi | y )N)super__init__)selfr   kwargs	__class__s      |/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/quantizers/quantizer_finegrained_fp8.pyr   z"FineGrainedFP8HfQuantizer.__init__   s    ,77    c                 .   t               st        d      | j                  j                  ry t        j
                  j                         sHt               s>| j                  r't        j                  d       d| j                  _        y t        d      t        j
                  j                         r`t        j
                  j                         }|\  }}|dk  s
|dk(  r3|dk  r.t        j                  d| d| d	       d| j                  _        y |j                  d
      }|t        j                  d       y t        |t              rL| j                  s?t!        |      dkD  r0d|j#                         v sd|j#                         v rt%        d      y y y y )NzMLoading an FP8 quantized model requires accelerate (`pip install accelerate`)zUsing FP8 quantized models requires a GPU or XPU, we will default to dequantizing the model to bf16 since no GPU or XPU is availableTzANo GPU or XPU found. A GPU or XPU is needed for FP8 quantization.   	   ziFP8 quantized models is only supported on GPUs with compute capability >= 8.9 (e.g 4090/H100), actual = `.z`. We will default to dequantizing the model to bf16. Feel free to use a different quantization method like bitsandbytes or torchao
device_mapzYou have loaded an FP8 model on CPU and have a CUDA or XPU device available, make sure to set your model on a GPU or XPU device in order to run your model. To remove this warning, pass device_map = 'cuda' or 'xpu'. r	   cpudiskzYou are attempting to load an FP8 model with a device_map that contains a cpu/disk device.This is not supported when the model is quantized on the fly. Please use a quantized checkpoint or remove the cpu/disk device from the device_map.)r   ImportErrorr   
dequantizetorchcudais_availabler   pre_quantizedloggerwarning_onceRuntimeErrorget_device_capabilityget
isinstancedictlenvalues
ValueError)r   argsr   compute_capabilitymajorminorr   s          r   validate_environmentz.FineGrainedFP8HfQuantizer.validate_environment   s   &(mnn##..zz&&(1G1I!!## [ 7;((3"#fgg::""$!&!A!A!C-LE5	uzeai####('5' 2Z[
 7;((3ZZ-
6
 
D)&&
Oa'j//11Vz?P?P?R5R k  6S ( ' *r   modelr   
param_namereturnc                 t    ddl m}m} t        ||      \  }}t	        |||f      r| j
                  s|dk(  ryyy)Nr   )
FP8Experts	FP8LinearbiasFT)integrations.finegrained_fp8r9   r:   r   r+   r%   )r   r5   r6   r   r9   r:   moduletensor_names           r   param_needs_quantizationz2FineGrainedFP8HfQuantizer.param_needs_quantizationN   s>    H25*Efy*56!![F%:r   paramztorch.Tensorc                 L    | j                  ||      ryt        | 	  |||      S )z4Return the element size (in bytes) for `param_name`.r	   )r?   r   param_element_size)r   r5   r6   r@   r   s       r   rB   z,FineGrainedFP8HfQuantizer.param_element_sizeZ   s*    ((
;w)%UCCr   c                     | j                   j                  }|syddlm}  ||      }g }|D ]0  }|}|D ]  }|j	                  |      \  }}	 |j                  |       2 || j                   _        y)u.  Rewrite the skip-list to the model's own module tree.
        For models that were already released, if they have a list of modules to not quantize
        we need to apply the weight renaming / weight conversion opérations to get the actual
        layer name of the model in `transformers`.
        Nr   )get_model_conversion_mapping)r   modules_to_not_convertconversion_mappingrD   rename_source_keyappend)
r   r5   skiprD   	renamingsremappednamerenamedrename_s
             r   !_normalize_modules_to_not_convertz;FineGrainedFP8HfQuantizer._normalize_modules_to_not_converta   s     ''>>E07	 	%DG# ?#55g>
?OOG$		%
 ;C  7r   c                     ddl m} | j                  |       | j                  || j                  j
                  |j                        | _         ||| j
                  | j                  | j                        }y )Nr   )replace_with_fp8_linear)rE   r   r%   )r<   rR   rP   get_modules_to_not_convertr   rE   _keep_in_fp32_modulesr%   )r   r5   r   rR   s       r   $_process_model_before_weight_loadingz>FineGrainedFP8HfQuantizer._process_model_before_weight_loadingv   sl    
 	K..u5&*&E&E4++BBED_D_'
# (#'#>#> $ 8 8,,	
r   c           
         | j                   j                  dk(  rddlm}  |       }|j	                         D cg c]5  \  }}|j                  d      r|j                  t        j                  k(  r|7 }}}|D ]y  }|j                  d      \  }}	}
|j                  |      }t        ||
      }t        ||
t        j                  j                  |j                  j!                  |      d             { ddlm}  ||       |S c c}}w )	Nue8m0r   )_get_ue8m0_dtype
_scale_invr   F)requires_grad)!_disable_deepgemm_on_multi_device)r   	scale_fmtr<   rX   named_parametersendswithdtyper"   float32
rpartitionget_submodulegetattrsetattrnn	Parameterdatator[   )r   r5   r   rX   rW   rL   r@   float32_scalesmodule_namerO   attrr=   scaler[   s                 r   #_process_model_after_weight_loadingz=FineGrainedFP8HfQuantizer._process_model_after_weight_loading   s    
 ##--8G$&E $)#9#9#;D%==.5;;%--3O N 
 ' e'+s';$Q,,[9-ehh&8&8u9M]b&8&cd	e 	U)%0!s   :Dc                    d|j                   j                  v rddddddddddddddd}||_        ddlm} t        |dd       }|j                  j                  |      }|r^dD ]Y  }t        ||d       xs i }|j                         D 	ci c]  \  }}	||j                  |	|	       }
}}	|
|k7  sMt        |||
       [ |S c c}	}w )	NQwen3colwiserowwise)z layers.*.self_attn.q_proj.weightz*layers.*.self_attn.q_proj.weight_scale_invz layers.*.self_attn.k_proj.weightz*layers.*.self_attn.k_proj.weight_scale_invz layers.*.self_attn.v_proj.weightz*layers.*.self_attn.v_proj.weight_scale_invz layers.*.self_attn.o_proj.weightz*layers.*.self_attn.o_proj.weight_scale_invzlayers.*.mlp.gate_proj.weightz'layers.*.mlp.gate_proj.weight_scale_invzlayers.*.mlp.up_proj.weightz%layers.*.mlp.up_proj.weight_scale_invzlayers.*.mlp.down_proj.weightz'layers.*.mlp.down_proj.weight_scale_invr   )r9   _experts_implementation)base_model_tp_planbase_model_ep_plan)
r   __name__rs   r<   r9   rc   _impl_tp_layer_overridesr*   itemsrd   )r   config	text_planr9   impllayer_overrides	plan_attr	base_plankvupdated_plans              r   update_tp_planz(FineGrainedFP8HfQuantizer.update_tp_plan   s    f&&///4=>G4=>G4=>G4=>G1:;D/89B1:;DI" )2F% 	>v8$?$==AA$GI =	#FIt<B	IRIZ[A?#6#6q!#< <[[9,FI|<	= 	  \s   
Cc                      yNT r   s    r   is_serializablez)FineGrainedFP8HfQuantizer.is_serializable   s    r   c                      y)NFr   r   s    r   is_trainablez&FineGrainedFP8HfQuantizer.is_trainable   s    r   c                      yr   r   r   s    r   is_compileablez(FineGrainedFP8HfQuantizer.is_compileable   s    r   c                     ddl m}  ||       S )Nr   )Fp8Quantize)r<   r   )r   r   s     r   get_quantize_opsz*FineGrainedFP8HfQuantizer.get_quantize_ops   s    >4  r   c                     ddl m} ddlm} | j                  r+| j
                  j                  r |g dd ||       g      gS g S )Nr   )WeightConverterFp8Dequantize)zweight$weight_scale_invactivation_scaleweightsource_patternstarget_patterns
operations)core_model_loadingr   r<   r   r%   r   r!   )r   r   r   s      r   get_weight_conversionsz0FineGrainedFP8HfQuantizer.get_weight_conversions   sK    8@$":":"E"E  $W$, -d 34  	r   c           	         ddl m}m} ddlm}  |dd      }|gt        |      z   }| j                  r| j                  j                  s|| j                         z   S g }|D ]  }t        ||      s|j                  |       !|j                  D cg c]  }|j                  d      s| }	}|	r|	D cg c]  }|dz   	 }
}|	D cg c]  }|d	t        d        d
z    }}|j                  D cg c]  }|j                  d      r| }}|
|z   |z   } ||       gt        |j                        z   } |||j                   |      }|j                  |        |j#                  | j                                |S c c}w c c}w c c}w c c}w )u  When loading with ``dequantize=True``, attach an :class:`Fp8Dequantize` op to
        every existing :class:`WeightConverter` so that per-block scales are folded into
        the weight *before* any later merge/concat ops collapse the per-expert structure.

        For each model-supplied converter that has a ``.weight`` source, we:
          1. anchor the existing weight patterns with ``$`` so they don't accidentally
             also match the ``.weight_scale_inv`` keys (the regex is searched, so the
             unanchored prefix would match both, sending scales to the wrong bucket);
          2. add anchored ``*.weight_scale_inv`` sources next to each weight pattern so
             the loader collects scale tensors alongside the weight tensors into the
             *same* converter bucket (both keys rewrite to the same target);
          3. prepend a fresh :class:`Fp8Dequantize` op so dequant runs first, before
             any merge/concat collapses the per-expert structure.

        The generic ``weight$ + weight_scale_inv → weight`` converter from
        :meth:`get_weight_conversions` is still appended at the end as a fallback for
        plain ``nn.Linear`` weights with no model-specific converter.
        r   )r   WeightRenamingr   z^(.+)\.scale$z\1.weight_scale_inv)r   r   z.weight$Nz.weight_scale_inv$r   )r   r   r   r<   r   listr%   r   r!   r   r+   rH   r   r^   r-   r   _original_target_patternsextend)r   weight_conversionsr   r   r   scale_renameupdatedconvpweight_sourcesanchored_weightscale_sourcesothernew_sourcesnew_opss                  r   update_weight_conversionsz3FineGrainedFP8HfQuantizer.update_weight_conversions   s   & 	I@ &6FXno*^d3E.FF""t'?'?'J'J%(C(C(EEE& 	!D dO4t$)-)=)=WAIAVaWNW4B"Cq1s7"C"CVd eQR#4c)n_!58L!L e e$($8$8Vq

9@UVV-=E(./$t2GG&$/$($B$B&
 NN4 %	!( 	t2245 X"C eVs$   E;'E;3F F.F
F
)r5   r   )ru   
__module____qualname____doc__requires_calibration__annotations__r   r4   strboolr?   floatrB   rP   rU   rm   r   r   propertyr   r   r   r   r   __classcell__)r   s   @r   r   r      s    
 !//8.`
.? 
S 
_c 
D(9 Ds DSa Dfk DC*
 
&6$L d     !
 7r   r   )typingr   utilsr   r   r   r   baser
   quantizers_utilsr   r"   modeling_utilsr   utils.quantization_configr   
get_loggerru   r&   r   r   r   r   <module>r      sH      ` `  2 0@			H	%O Or   