
    ^jU.                        d dl Z d dlmZ d dlZd dlmZ d dlmZ d dlm	Z	 ddl
mZmZmZ ddlmZ dd	lmZmZmZmZ dd
lmZ ddlmZmZmZmZ ddlmZmZm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z)m*Z*m+Z+m,Z, erddlm-Z- ddlm.Z.  e j^                  e0      Z1ejL                  jd                  Z2ed        Z3 ede3 e	d      d      Z4 eejj                  de2jj                  jl                        Z7 eejj                  ejp                  js                         rdndde2jj                  jt                        Z; eejx                  de2jx                  jl                        Z=dZ>ej~                  ej                  ej                  ej                  fZC ej                  e2jj                        d#ddd        ZE ej                  e2jx                        dddd!d"       ZFy)$    N)TYPE_CHECKING)counters)CKGemmTemplate)load_kernel_template   )configirlowering)MMKernelInputs)	loweringsmake_pointwisemake_reductiontransform_argsget_max_y_grid)autotune_select_algorithmExternKernelChoiceSymbolicGridFnTritonTemplate)_use_cutlass_for_opuse_aten_gemm_kernelsuse_ck_gemm_templateuse_cpp_bmm_templateuse_cutlass_templateuse_nv_universal_gemm_templateuse_triton_template)opsV   )_is_static_problemis_batch_stride_largest_or_zeromm_argsuse_native_matmul)ChoiceCaller)KernelTemplatec                     |||d          |||d         z  }t               } | || |      d      } || |      }	||	|fS )NBLOCK_MBLOCK_Nr   r   )
bmnmetacdivmaxtiles
max_y_gridgrid_zgrid_ys
             e/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/kernel/bmm.pybmm_gridr4   .   sW    DO$tAtI'??E  !Ja$a(F!V_F66""    bmm
triton_bmmT)namegridsource"cache_codegen_enabled_for_templatezat::bmm_out)op_overloadzat::_bmm_out_dtype_xpuzat::_bmm_out_dtype_cuda	bmm_dtype)r8   r<   zat::baddbmm_out    layoutc          
      0   $% t         j                  j                  % j                         } j	                         j
                  }%fd}%fd}||dv r||j	                         j
                  k(  r||j                         k(  r|t        v r | j                         d         r ||j                         d         ru | j                         d   t              rWt        j                   d       t        j                  |d      }t        j                  t        j                   |      d      S t        d	  |fD              r j                         d   dk(  s|j                         d   dk(  rWt        j                   d       t        j                  |d      }t        j                  t        j                   |      d      S d
 }d $$fd}	 |       r0t         j                  j                  j                  d   }
 |	 |
        ||      r0t         j                  j                  j                  d   } |	||      }t!         |      rt#        t$        j                      d       t#        t$        j                     |d      }t'         |gi ddd      \  }}t(        j*                  j,                  rO j.                  t0        j2                  t0        j4                  fv r# fd}|D cg c]  } t7        |      |       }} t7        t8        j:                        | } t=        d      |d      }|S t?         |||      \  }}}} }d}tA         |g|      } j                         d   }tB        d   d| d| d| d| xx   dz  cc<   tD        jG                  d|||| j                         |j                         |       tH        }i }|r- j	                         j
                  dv sJ d       tJ        }d|i}g }g }i }tM               r |jO                  |       |||jP                  <   tS        |d      r|jO                  tT               |jW                  t         jX                  j[                  ||||             t]        |      \  }}t_         ||      }|rB|r@ta        ||||      r2tc        |      r'ddl2m3}  | ji                  |||jk                                tm        | |      r'ddl7m8}! |!js                  |||jk                                tu        ||||      r%tw        jx                  |||jk                                |r t{        |||| |      rdd l>m?}"  |"|||       t        |||jk                         |      \  }#}|#S c c}w )!z`
    Lowering for autotuning aten.bmm with different backends (Aten, Triton, CUTLASS, etc.)
    c                 0    j                  | d      dk(  S )Nr   )fallbackr   )optimization_hint)dimsizevarss    r3   dim_is_one_or_hintz%tuned_bmm.<locals>.dim_is_one_or_hintd   s      ))#):a??r5   c                 *    j                  | |       S N)statically_known_gt)rE   	thresholdrF   s     r3   dim_is_not_known_gtz&tuned_bmm.<locals>.dim_is_not_known_gti   s     //Y???r5   N)cudaxpur   r   )axisc              3   V   K   | ]!  }|j                         j                  d k(   # yw)cpuN)
get_devicetype).0xs     r3   	<genexpr>ztuned_bmm.<locals>.<genexpr>~   s!     
>A1<<>%'
>s   ')c                     t        j                  |       syt        j                  | d      \  }}t        |t         j                        S )NTF)freeze)r	   is_storage_and_layoutas_storage_and_layout
isinstanceFlexibleLayout)t_r@   s      r3   is_valid_to_require_contiguousz1tuned_bmm.<locals>.is_valid_to_require_contiguous   s<    ++A.005AIAvfb&7&788r5   c                     |d   dk(  xr | d   dk(  xs |d   | d   k\  xs |d   dk(  xr | d   dk(  xs |d   | d   k\  S )NrO   r    )sizesstridess     r3    is_preferred_layout_as_bmm_inputz3tuned_bmm.<locals>.is_preferred_layout_as_bmm_input   sf     q QeBi1n&PuRy8PU"+"Sb	Q(R'"+r:RUr5   c                     |j                   d   j                         }|j                   d   j                         } ||      st        j                  j                  |       } | S )Nval)r,   sizestrider	   ExternKernelrequire_contiguous)r^   meta_trd   re   rf   s       r3   may_require_contiguousz)tuned_bmm.<locals>.may_require_contiguous   sT    KK&++-Ekk%(//1G3E7COO66q9Hr5   r   TF)argskwargs	broadcasttype_promotion_kindconvert_input_to_boolc                 H    t        j                  | j                  d      S )NF)use_compute_types)r   to_dtypedtype)rV   mat1s    r3   	_to_dtypeztuned_bmm.<locals>._to_dtype   s    ||AtzzUKKr5   dot)r@   	out_dtyper6   )r{   aten_mm_infoz	aten.bmm_r_   zZTuned aten.bmm: batch=%s, m=%s, n=%s, k=%s, mat1_dtype=%s, mat2_dtype=%s, output_layout=%sz+out_dtype is only supported for CUDA or XPUr{   check_max_autotune)kwarg_overrides)CUTLASS3xGemmTemplate)CppBmmTemplate)add_nv_universal_gemm_choices)Ar   graphrF   	get_dtyperS   rT   _BMM_DOT_DECOMPOSE_DTYPESget_size_BMM_DOT_K_DECOMPOSE_THRESHOLDL	unsqueezesum_mulallcurrent_nodero   r#   r   atenr   inductor_configtritoncodegen_upcast_to_fp32rw   torchfloat16bfloat16r   r   rz   r   r"   r   r   loginfoaten_bmmaten_bmm_dtyper   appenduidr   bmm_templateextendchoicesget_template_configsr    r!   r   r   codegen.cutlass.gemm_templater   add_cutlass_gemm_choicesnodesr   codegen.cpp_bmm_templater   add_choicesr   r   add_ck_gemm_choicesr   codegen.nv_universal_gemmr   r   )&rx   mat2r{   r@   rw   device_typerG   rL   r`   rn   	meta_mat1	meta_mat2ro   rp   ry   rV   mul_pointwisedot_reductionr*   r+   kr8   kernel_inputs
batch_sizeaten_handleraten_extra_kwargsr   templates_to_user   r_   
is_nonzerobatch_stride_largest_or_zeror   r   r   noderf   rF   s&   `                                   @@r3   	tuned_bmmr   [   s+   
 wwHNNE//#((K@
@ 	?*4??,111T^^%%..t}}q12t}}q12 24RS {{4${{4#vvaeeD$'a00

>$
>>==?1"dmmoa&8A&=;;tR(D;;tQ'D66!%%d+!44	9	U	 *$/,,11!4I)$	:D)$/,,11!4I)$	:Dt$(r2(q1% $"'
f !!88TZZMMNNL
 >

L ;??Q-N9-a0?D?/sww/6-u-mQ? #*d6Y#Aq!VT4 D #D$<9EM #J^yAaS!AaSABaGBHHd				 (0L %%8 	
9	
8 &()4"$G CEO-,=(()6e<- NN			&&+	 	' 	
 'v.MAz#B4v#V $ Aq1%I66V]002	
 FD$/=""!	
 FAq!,**7FM<O<O<QR4VQ1dDQM%gv}E'g}7J7J7LfUGD!K{ @s   3V)alphabetar@   c                   t        ||      r|dk(  rd}nt        t        j                     ||       }|dk(  rd}n8t        t        j                     |t        t        j                     ||            }t        t        j
                     ||      S t        ||| |      \  }}	}
}}}} t        | ||gt        ||            }|j                         d   }t        d   d| d| d|	 d|
 xx   dz  cc<   t        j                  d	|||	|
|j                         |j                         | j                         |	       d
}g }g }t               r|j                  t                t#        |d      r|j                  t$               |j'                  t(        j*                  j-                  |||             t/        |||j1                         |      \  }}|S )z_
    Lowering for autotuning aten.mm with different backends (Aten, Triton, CUTLASS, etc.)
    r   r?   )r   r   )scalarsr|   zaten.baddbmm_r_   r   zkTuned aten.baddbmm: batch_size=%s, m=%s, n=%s, k=%s, mat1_dtype=%s, mat2_dtype=%s, inp=%s, output_layout=%sbaddbmmFr}   )r#   r   r   r   r6   addr"   r   dictr   r   r   r   r   r   r   aten_baddbmmr   r   r   r   r   r   r   r   )inprx   r   r   r   r@   arg1arg2r*   r+   r   r   r   r8   r   r   r   r_   s                     r3   tuned_baddbmmr     s   
 t$19DTXX&tS1DA:DTXX&ui.A$.MND"4.. (/tT3v'N$Aq!VT4 #	dD4e$#?M
 #J^}ZL!AaS!EF!KFHHu			
 D"$G CE-6e<- NN			&&}6FM (g}7J7J7LfUGD!Kr5   rI   )Gloggingtypingr   r   torch._dynamo.utilsr   7torch._inductor.codegen.rocm.ck_universal_gemm_templater    torch._inductor.kernel.mm_commonr    r   r   r	   r
   r   r   r   r   r   r   r   runtime.runtime_utilsr   select_algorithmr   r   r   r   utilsr   r   r   r   r   r   r   virtualizedr   r   	mm_commonr    r!   r"   r#   r$   r%   	getLogger__name__r   r   r4   r   r6   outr   rN   _is_compiled	dtype_outr   r   r   r   r   r   float32float64r   register_loweringr   r   rc   r5   r3   <module>r      s       ( R A ; ; * P P 2    !  !1g!yy~~ # # 		-'+	 eiiDHHLLQ#	II %		 6 6 8>W	""	 "	MM$$,,2B2B "$ 	MM	NN	MM	MM	  TXXwD w wt T\\",-Ad 9 #9r5   