^j7ddlmZddlmZddlmZddlmZmZm Z ddl m Z m Z m Z mZmZe r ddlZddlmZe rdd lmZeZe resddlZej.eZGd d eZGd d ej"j6ZGddej:ZeddZ dde e!dzfdZ"y)) lru_cache)ACT2FN) ConversionOps)get_module_from_name on_deviceshould_convert_module)is_accelerate_availableis_fbgemm_gpu_availableis_torch_availableis_torch_xpu_availableloggingN)nn)init_empty_weightsc eZdZdZ ddeeejeejzfdejjdzdeeejffdZ y)FbgemmFp8Quantizec||_yN) hf_quantizer)selfrs o/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/integrations/fbgemm_fp8.py__init__zFbgemmFp8Quantize.__init__-s (N input_dictmodelreturnc <t|jd\}}|d}ddlm}t ||\}}t ||r.|dk(r|j dd} | j} | jd| d} t| 5t| \} } ddd j| }|j dd} j| dd| d}n|dk(r|j dd} | j} | jd| d} t| 5t| \} } ddd j| }|j dd} j| d| dd}n^t|5t|\}}dddtjjj|jdd}|tjj|diS#1swYbxYw#1swYxYw#1swYxYw) Nrr)FbgemmFp8Llama4TextExperts gate_up_proj down_proj_scale)tupleitems integrationsrr isinstance transposeshapereshaperquantize_fp8_per_rowtorchr Parameterview)rrrkwargs target_keyvaluermodule tensor_nametransposed_paramoriginal_shapeflattened_paramnew_value_flatweight_scale_flat new_value weight_scales rconvertzFbgemmFp8Quantize.convert0s8 "*"2"2"45a8 Ea=25*E f8 9n,$)??1a#8 "2!7!7"2":":2~b?Q"R/^8L_8]5N$5^+22>B %//15 0889JA~^_O`a  +$)??1a#8 "2!7!7"2":":2~b?Q"R/^8L_8]5N$5^+22>B %//15 0889JN[\L]_`a 5! F*>u*E' < F 88--l.?.? @R@RST@UWX.YZLEHH..y9j\;PR^__;^^"^^ F Fs$ G9!HH9HHHr) __name__ __module__ __qualname__rdictstrr,TensorlistrModuler;rrrr,sh) )-5`ellT%,,-???@5`xx%5` c5<< 5`rrc<eZdZejffd ZdZxZS)FbgemmFp8Linearcrt||||||_||_tj j t j||f||_tj j t j|dftj|_ |jdt jdgtjd|rRtj j t j|jtj|_ yd|_ y)Ndtyper input_scale_ubF persistent)superr in_features out_featuresr,rr-zerosweightfloat32r:register_bufferfloatbias)rrNrOrUrI __class__s rrzFbgemmFp8Linear.__init__is lD9&(hh((lK5PX])^_ !HH..u{{L!;LTYTaTa/bc -u{{A3ekk/R_de **5;;8I8IRWR_R_+`aDIDIrcg|jddd}t|5t|jd|jdj |j \}}ddd|j jtj}tretj|jjjd|j|j|j }nUtj"j$j'|j|d}|j ||j zn|}|j)|}~~|S#1swYxYw)Nr!)scale_ub)scale_ascale_b out_dtyperUTuse_fast_accum)r)rr+r. contiguousrJr:tor,rR_is_torch_xpu_available _scaled_mmrQt unsqueezerIrUopsfbgemmf8f8bf16_rowwiser*)rx output_shape x_quantizedx_scaleweight_scale_float32outputs rforwardzFbgemmFp8Linear.forwardws<*"*r* q\ #7r1772;'224t?R?R$ K  $0033EMMB "%% ))"-,..0''YY FYY%%66T[['3GX\7F,099+@Vdii'fF -  -  s AE,,E6)r<r=r>r, float8_e4m3fnrrm __classcell__rVs@rrFrFhs>C>Q>Q rrFc<eZdZejffd ZdZxZS)rcnt||j|_|j|_|j |_|j|_t|j|_ tjjtj|j|j d|j zftj|_tjjtj|jd|j dzftj |_tjjtj|j|j |j ftj|_tjjtj|j|j dftj |_|j)dtjdgtj*dy)NrrHr rJFrK)rMrnum_local_experts num_expertsintermediate_size hidden_size expert_dimr hidden_actact_fnr,rr-rPrnrrRgate_up_proj_scaler"down_proj_scalerSrT)rconfigrIrVs rrz#FbgemmFp8Llama4TextExperts.__init__s !33!'!9!9!--00V../ !HH.. KK))4+;+;Q=PQY^YlYl m #((("4"4 KK))1doo.AB%-- X# ++ KK))4??DPQVVWY[\]hhjlln+11 %%$%%a(221a89K9LMXXZ\\^(22266q9!<=O=PQVVWY[\]hhjlln+11 yy''88$%%a(221a89L:LMXXZ .q1!45H6HINNrSTU``b#' 9YY%%66$%%a(221a89K9LMXXZ .q1!45G5HINNrSTU``b#' 7T[[..I9% x7KIWacgcvcv7w4#_ x'+&:&:&=&=emm&L #& % 0 0'NN1%//15@@B+55b93A6;;BBMMOQQS+11 ! !& 0 0 A A'NN1%//15@@B#+A.33B:EEG#' !B! +KNEB +F"nn]%9%9: D$4$455  P x xsS;"T;T T )r<r=r>r,rRrrmrorps@rrrs%*]]f0R6rrr )maxsizectrddlm}|ddjStj j jS)Nr  get_kernelzkernels-community/fp8-fbgemm)version)r` hub_kernelsrr+r,rdrers rget_quantize_fp8_per_rowrs2+8!DYYY 99   0 00rmodules_to_not_convertcttad}|rinddi}|jD]\}}t||sd} t d5|j j dk(r;t|jd|j} t| xs |j} nYt|tjr?t|j|j|j dufi|} | j#dddd| |j%|| d}|st&j)d|S#1swY;xYw) a A helper function to replace all `torch.nn.Linear` modules by `FbgemmFp8Linear` modules. This will enable running your models using high performance fp8 kernel from FBGEMM library. Parameters: model (`torch.nn.Module`): Input model or `torch.nn.Module` as the function is run recursively. modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`): Names of the modules to not convert. In practice we keep the `lm_head` in full precision for numerical stability reasons. quantization_config (`FbgemmFp8Config`): The quantization config object that contains the quantization parameters. pre_quantized (`book`, defaults to `False`): Whether the model is pre-quantized or not FrINT)include_buffersLlama4TextExperts text_configzYou are loading your model using FP8 quantization but no linear modules were found in your model. Please double check your model architecture, or submit an issue on github if you think this is a bug.)rr+ named_modulesr rrVr<getattrr|rr'rLinearrFrNrOrUrequires_grad_ set_submoduleloggerwarning) rrquantization_config pre_quantizedtp_planhas_been_replaced module_kwargs module_namer2 new_modulers rreplace_with_fbgemm_fp8_linearr s;$45'Bgt_M$224! V$[2HI   5 1((,?? &ellM5<<P 7 8Su||T FBII.,&&''KKt+$  ))%0 1"    K4 5!8   L= 1 1s B.D..D7 )NNFN)# functoolsr activationsrcore_model_loadingrquantizers.quantizers_utilsrrr utilsr r r r rr,r acceleraterr`fbgemm_gpu.experimental.gen_ai fbgemm_gpu get_loggerr<rrrrFrCrrrBr@rrDrrrs .``-02%<)   H %9` 9`x+ehhoo+\k6k6\ 111tx:#'9t#3:r