^jddlZddlmZddlZddlmZddlmZmZddl m Z m Z ddl m Z ddlmZmZdd lmZdd lmZmZmZdd lmZmZdd lmZej6eZeej<d dZeej@jBjDddejFZ$ej@jBZ!ej@jJZ%ej@jLZ&ddZ'ddZ(y)N)Any)mm_args)configlowering)CppGemmTemplateCppWoqInt4GemmTemplate)create_epilogue_with_attr)expandregister_lowering)WeightInt4PackMatmul)autotune_select_algorithmExternKernelChoicerealize_inputs)use_aten_gemm_kernelsuse_cpp_gemm_template)Vzat::_weight_int8pack_mmF)has_out_variantz*at::native::_weight_int4pack_mm_cpu_tensor)rkernel_creatorcZtjtjtj tj gtjtjtjtj tjtj yN)radd_needs_realized_inputs quantized max_pool2d _quantized$wrapped_fbgemm_pack_gemm_matrix_fp16!wrapped_fbgemm_linear_fp16_weight make_fallbackn/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/quantized_lowerings.pyregister_quantized_opsr"'sn &&   ; ;  8 8  9//0 :JJK :GGHr c ttjddddtjdtjdtjdt dt f d}ttj ddddtjdtjd td tjdt dt f d }tjtjtjtjy) N)type_promotion_kind)layoutinputweightscaler%returnct||d\}}}}}|jtjtjtj fvr!|jtj k(sJ}trtj||f|gng}dtjdtffd } t|||drtj||||gd| td|||g|\} }| S) NT)r%mat2_transposedbufr)c Zt|dttjS)Nmul)other)r rr size)r,r%r(s r! _mul_epiloguez?register_woq_mm_ops..int8pack_mm.._mul_epilogueOs',U.v{{1K"L r )r+)trans_wepilogue_creator_weight_int8pack_mm)r get_dtypetorchbfloat16float16floatint8raten__weight_int8pack_mmbindTensorrrr add_choicesr) r&r'r(r%_mat1mat2 aten_layoutchoicesr1nodes `` r! int8pack_mmz(register_woq_mm_ops..int8pack_mm5s '. 6&$' #1at NN  L L EJJ.  / %&& * *D$+> L M  u||   !dD$ O  ' 'tU#!.  , !7T4,? a r qGroupSizeqScaleAndZeroscVt|||dd\}}}}}}|jtjtjtj fvr!|jtj k(sJtjjtj|tjd}|} trtj||||f| gng} tj stj"rMt%| ||dd|r<|j'j)rt*|j-| | ||||gdtj.j0j2dtj4fd} | d d } t7d | ||||g| | \} }| S) NT)r% use_4x2_dimr+)dtype)name)r+ is_woq_int4 q_group_sizexr)c|jjsJ|j}|j}t j dd|tj |S)Nr)rJdevice) get_layout is_contiguousget_size get_devicer6randintuint8)rNshaperQs r!get_example_weightzHregister_woq_mm_ops..int4pack_mm_cpu..get_example_weightsK<<>//1 11JJLE\\^F==Cekk&Q Qr cXtjj|jSr)rgraph constantsget_name)rNs r!z>register_woq_mm_ops..int4pack_mm_cpu..s**1::<8r )r_weight_int4pack_mm_for_cpu) input_gen_fns)rr5r6r7r8r9rWrr[add_tensor_constanttensorint64raten__weight_int4pack_mm_cpur<r max_autotunemax_autotune_gemmrrRrSr r> _inductorirIRNoder=r)r&r'rFrGr%r?r@rA group_sizerBrCrYrarDs r!int4pack_mm_cpuz,register_woq_mm_ops..int4pack_mm_cpubs'. 6&dD' #1at NN  L L EKK/  0WW00 LL5;; 7d1  %& -114^rs 4N8/+ @g!- 8% 2 II000'..   II   YY ! !  yy~~ Iw=r