
    ^j                      d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZmZmZmZmZ d dlmZ d dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d d	lmZ d d
lm Z  d dl!m"Z"m#Z#m$Z$ d dl%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6m7Z7m8Z8 erd dl9m:Z: ddl.m;Z; ddl<m=Z= ddl>m?Z?m@Z@ ddlAmBZB ddlCmDZD ddlEmFZFmGZGmHZHmIZI ddl/mJZJmKZKmLZL ddlMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZX ddlYmZZZm[Z[m\Z\ ddl]m^Z^ dd l_m`Z`maZambZbmcZc dd!ldmeZemfZf dd"lgmhZhmiZimjZjmkZkmlZl erd d#l9mmZmmnZnmoZo d d$lpmqZq d d%lrmsZs  ej                  eu      Zvej                  j                  eud&      Zyej                  j                  eud'      Zzej                  j                  eud(      Z{ ec       j                  Z} e g d)      Z~dRdSd*Zej                    G d+ d,             Z G d- d.e      Z G d/ d0e      Z G d1 d2e      ZdTd3Z ed4e`e`5      Zej                    G d6 d7             Z G d8 d9e      Z G d: d;ebe   ee         Z ej                   d<=       G d> d?             Zej                    G d@ dA             Z ej                   d<=       G dB dC             Z ej                   d<=       G dD dE             Z ej                   d<=       G dF dG             Z G dH dIe=      Z G dJ dKe=      Z G dL dMeK      Z ej                   d<=       G dN dO             Z G dP dQe      Zy)U    )annotationsN)Counter)AnycastGeneric
NamedTupleTYPE_CHECKING)TypeVar)metrics)MultiTemplateBuffer)free_unbacked_symbols)immutable_dict)
OrderedSet)FloorDivIdentityModularIndexing)free_symbol_is_type
prefix_strsymbol_is_typeSymT   )counters   )configir	scheduler)prologue_preserves_zero_mask)	code_hashPyCodeCache)	MemoryDepStarDepWeakDep)CallableIRNode)WrapperHandler) convert_index_expr_to_value_expr!indexing_dtype_strength_reduction)CoordescTuner)DeviceProperties)
green_textlast_power_of_2next_power_of_2yellow_text)BaseSchedulerNodeBaseScheduling	WhyNoFuse)cache_property_on_selfdecompose_indexexpr_fits_within_32bitget_dtype_sizeIndentedBufferPlaceholderprefix_is_reductionsympy_index_symbolsympy_product
sympy_subsunique)ops
OpsWrapperV   )BlockPatternMatcher)CSEVariableindex_prevent_reorderingKernelPythonPrinter)MultiKernelSizeHintMultiKernel)DisableReductionEnableReductionNodeScheduleEntryNodeScheduleMarkerSIMDKernelFeatures)IterableIteratorSequenceTritonKernel)CoalesceVarAnalysis
perf_hintsschedulefusion)zyxr0_r1_c                j    t         j                  j                  j                  j                  }||S | S N)torch	_inductorr   triton	max_tiles)defaultr`   s     g/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/codegen/simd.pyget_max_tilesrc   h   s-    &&--77I!-9:7:    c                       e Zd ZdZej
                  j                  ej
                  j                  d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZee	dd              Z
d	dZee	d
d              Z xZS )IterationRangesa  
    Each range tree represents multiple sets of iteration indexing
    in a single tiled dimension in the output kernel.

    If you have two loops ranges one (4, 3, 2) and another (4, 6),
    then the range tree will be:
            4 (i0)
        3 (i1)  6 (i3)
        2 (i2)
    Where i0 is shared between both loops, but then the split into
    different indexing vars.  All loop ranges must iterate over
    the same number of elements.
    )divisorlengthc                   t         
|           || _        || _        || _        || _        || _        || _        || _        || _	        |	| _
        y r\   )super__init__namevar_list
var_rangesnumelprefixrg   rh   kernelroot)selfrl   rm   rn   ro   rp   rq   rg   rh   rr   	__class__s             rb   rk   zIterationRanges.__init__}   sO     		 $
	rd   c                ,    t        | j                        S r\   )r8   rp   rs   s    rb   is_reductionzIterationRanges.is_reduction   s     #4;;//rd   c                ,    t        | j                        S r\   r9   rl   rv   s    rb   symbolzIterationRanges.symbol       !$)),,rd   c                z    t        j                         D ci c]  \  }}||
 }}}|| j                     S c c}}w r\   )r   itemsrp   )rs   symtrp   prefix_to_symts       rb   r~   zIterationRanges.symt   s>     <F;K;K;MN<4&$,NNdkk** Os   7)rl   strrm   list[sympy.Symbol]rn   dict[sympy.Symbol, sympy.Expr]ro   
sympy.Exprrp   r   rq   
SIMDKernelrr   IterationRangesRootreturnNoner   boolr   sympy.Symbol)r   r   )__name__
__module____qualname____doc__sympySOnerk   propertyr2   rw   rz   r~   __classcell__rt   s   @rb   rf   rf   m   s    . ww{{ % 3	
    " 
0 0  0- +  +rd   rf   c                       e Zd ZdZ	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZddZddZddZddZddZ	ddZ
dd	Zdd
ZddZddZddZddZddZddZ	 	 	 	 d dZd!dZ	 	 	 	 d"dZ xZS )#r   z
    Root of a iteration range tree that represents a single
    tiled dimension in the output kernel. It contains multiple
    sets of iteration represented with IterationRangesEntry.
    c          	         |i }t         |   |g i ||||        || _        i | _        || _        |r| j
                  r|	J || _        || _        |	| _        |
| _	        y )N)rl   rm   rn   ro   rp   rq   rr   )
rj   rk   indexnodes	pid_cacherw   is_loop
tensor_dimgrid_dimhas_zdim)rs   rl   ro   rp   r   rq   r   r   r   r   r   rt   s              rb   rk   zIterationRangesRoot.__init__   s     I 	 	
 
=?
 *3
 t00X5EFF$  rd   c                <    d| j                   d| j                   dS )NzIterationRangesRoot(, z, ...))rl   ro   rv   s    rb   __repr__zIterationRangesRoot.__repr__   s    %dii]"TZZLGGrd   c                h    t        j                  | j                  j                          ddd      S )NBLOCKTintegerpositive)r   Symbolrp   upperrv   s    rb   
block_sizezIterationRangesRoot.block_size   s+    ||t{{0023594RVWWrd   c                4    t        | j                               S r\   )r   r   rv   s    rb   block_size_strz"IterationRangesRoot.block_size_str   s    4??$%%rd   c                L    t        j                  | j                   ddd      S )NoffsetT)r   nonnegative)r   r   rp   rv   s    rb   block_offsetz IterationRangesRoot.block_offset   s"     ||t{{m62DdSSrd   c                     | j                    dS )Nmask)rp   rv   s    rb   	mask_namezIterationRangesRoot.mask_name   s    ++d##rd   c                (    || j                         k(  S r\   )r   )rs   mask_vars     rb   	owns_maskzIterationRangesRoot.owns_mask   s    4>>+++rd   c                x    | j                   ydg|z  }| j                         || j                   <   t        |      S )N 1)r   r   tuple)rs   tensor_ndimshapes      rb   
mask_shapezIterationRangesRoot.mask_shape   s<    ??"#!%!4!4!6dooU|rd   c                     yNTr   rv   s    rb   supports_constant_maskz*IterationRangesRoot.supports_constant_mask       rd   c                     yNFr   rv   s    rb   has_custom_codegen_headerz-IterationRangesRoot.has_custom_codegen_header       rd   c                     yNr   r   rv   s    rb   named_constantsz#IterationRangesRoot.named_constants   s    rd   c                b    | j                   j                         D ]  }|j                           y r\   )r   valuescache_clear)rs   nodes     rb   r   zIterationRangesRoot.cache_clear   s*    JJ%%' 	D	rd   c                2    t        | j                   d      S )Nr   )r9   rp   rv   s    rb   	index_symzIterationRangesRoot.index_sym   s    !T[[M"788rd   c                   t         j                  j                  j                  ||z  | j                        rt        | j                         |      }nt        | j                         ||      }|| j                  vrt        | j                   t        t         j                  j                         ||||       }|t         j                  j                  |j                         <   | j                   j#                  |j                                || j$                  |j                         <   || j                  |<   | j                  |   S )zF
        Lookup a given RangeTreeEntry, creating it if needed
        )r?   graphsizevarsstatically_known_equalsro   r   r   r   r   IterationRangesEntryrp   nextrq   iter_vars_countrange_tree_nodesrz   rm   appendrn   )rs   rg   rh   exprr   s        rb   lookupzIterationRangesRoot.lookup  s     7733Gf4DdjjQDNN,g6D"4>>#3WfEDtzz!';;-QXX%=%= >?@D 8<AHH%%dkkm4MM  /-3DOODKKM*#DJJtzz$rd   c                j    | j                  t        j                  j                  | j                        S )zReturn the canonical entry for this root's unsplit logical index.

        This is ``lookup(1, numel)``, so the entry is interned in
        ``range_tree_nodes`` just like split entries.
        )r   r   r   r   ro   rv   s    rb   
full_rangezIterationRangesRoot.full_range  s!     {{577;;

33rd   c                    t         j                  j                  }g }t        |      D ](  }|j	                  | j                  ||             ||z  }* g t        |      S r\   )r   r   r   reversedr   r   )rs   lengthsrg   itervarsrh   s        rb   construct_entriesz%IterationRangesRoot.construct_entries!  s]     ''++w' 	'FOODKK89&G	' %(#$$rd   c                f    | j                  |      D cg c]  }|j                          c}S c c}w r\   )r   rz   )rs   r   es      rb   	constructzIterationRangesRoot.construct+  s'    $($:$:7$CDq
DDDs   .c           
     F  	
 dd|j                   D cg c]+  }t        j                  j                  j	                  |      - }}|D cg c]  }|s|j
                  | u s| }}|j                  fd       t        j                  j                  g 	g 
	
fd}|D ]v  }t        j                  j                  j                  |j                        s8 || j                  t        |j                                     |j                   ||       x t        j                  j                  j                  | j                         s, || j                  t        | j                                      g t#        	      g t#        
      fS c c}w c c}w )z,Figure out vars from this tree used in indexc                    t         j                  j                  j                  | j                        }t         j                  j                  j                  | j
                        dk(  }|| fS )a:  
            Gets the key for sorting nodes. When two nodes have the
            same divisor, the node with length as 1 should be handled
            first so the current divisor is not changed after multiplied
            node.length. Returns `not length_is_one_hint` for ascending
            sort.
            r@   )r?   r   r   optimization_hintrg   rh   )rX   divisor_hintlength_is_one_hints      rb   get_sort_keyz8IterationRangesRoot.vars_and_sizes.<locals>.get_sort_key3  sV     77++==aiiHL!"!1!1!C!CAHH!MQR!R &8"899rd   c                     |       S r\   r   )rX   r   s    rb   <lambda>z4IterationRangesRoot.vars_and_sizes.<locals>.<lambda>A  s    a rd   keyc                    j                  | j                                j                  | j                         | j                  z  y r\   )r   rz   rh   )r   rg   
index_varssizess    rb   addz/IterationRangesRoot.vars_and_sizes.<locals>.addF  s5    dkkm,LL%+Grd   )rX   r   r   ztuple[int, bool])free_symbolsr?   rq   r   getrr   sortr   r   r   r   r   r   rg   r   r   ro   r   )rs   r   sr   nr   r   rg   r   r   r   s          @@@@rb   vars_and_sizesz"IterationRangesRoot.vars_and_sizes.  sF   

	: <A;M;MNa**..q1NN!:qQ166T>::

0
1''++
	,  	D77##;;DLL'RDKK$,,)HIJ,,I	 ww77

GLGXdjj'%BCD&*%&(:(5/(:::/ O:s   0FFF$Fr\   )rl   r   ro   r   rp   r   r   intrq   r   r   dict[str, str] | Noner   r   r   
int | Noner   r   r   r   r   r   r   r   r   r   )r   r   r   r   )r   r   r   ztuple[str, ...]r   r   1tuple[tuple[sympy.Symbol, sympy.Expr, bool], ...]r   r   r   )rg   r   rh   r   r   r   )r   r   )r   list[sympy.Expr]r   zlist[IterationRangesEntry])r   r   r   r   )r   r   r   z+tuple[list[sympy.Symbol], list[sympy.Expr]])r   r   r   r   rk   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   s   @rb   r   r      s     ,0)!)! )! 	)!
 )! )! ))! )! )! )! )! 
)!VHX&T
$,9 .4%'%	#%E(;(;	4(;rd   r   c                  p     e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 d	 fdZd
dZddZddZd
dZddZddZ	ddZ
 xZS )r   c                $   t         |   ||j                  |z  |j                  |j                  |j
                  |||j                  |j                  	       || _         t        j                  d       | j                        | _        || _        y )N)	rl   ro   rm   rn   rp   rg   rh   rq   rr   )rj   rk   ro   rm   rn   rp   rq   rr   parent	functools	lru_cache_codegencodegenr   )rs   rl   rg   rh   r   r  rt   s         rb   rk   zIterationRangesEntry.__init__Z  s~     	,,'__((==== 	 
	
 0y**40?	rd   c                    d| j                    d| j                   d| j                   d| j                   d| j                   dS )NzIterationRangesEntry(r   ))rl   rg   rh   r   rn   rv   s    rb   r   zIterationRangesEntry.__repr__q  sH    &tyykDLL>DKK=PRSWS\S\R]]_`d`o`o_ppqrrrd   c                L    fd| _         d | j                   _        | _        y )Nc                      S r\   r   rl   s   rb   r   z/IterationRangesEntry.set_name.<locals>.<lambda>u  s    t rd   c                      y r\   r   r   rd   rb   r   z/IterationRangesEntry.set_name.<locals>.<lambda>v      rd   )r  r   rl   )rs   rl   s    `rb   set_namezIterationRangesEntry.set_namet  s    ##/ 	rd   c                8    | j                   j                          y r\   )r  r   rv   s    rb   r   z IterationRangesEntry.cache_cleary  s      "rd   c                X    t         j                  j                  |        | j                  S r\   )r?   rq   codegen_iteration_ranges_entryrl   rv   s    rb   r  zIterationRangesEntry._codegen|  s    	//5yyrd   c                   g }t        | j                  t        j                        r|S t        | j                  t        t
        f      sJ t        | j                               | j                  j                  dd  D ]l  }t        |t        j                  t        j                  f      r.|j                  }t        |      dkD  sIt        d |D              s\|j                  |       n |S )Nr@   r   c              3  P   K   | ]  }t        |t        j                           y wr\   )r   r   SIZE.0r   s     rb   	<genexpr>z8IterationRangesEntry.precomputed_args.<locals>.<genexpr>  s       ,56N1dii0,   $&)
isinstancer   r   r   r   r   typeargsIntegerr   lenallr   )rs   precomputed_argsargsymbolss       rb   r  z%IterationRangesEntry.precomputed_args  s    -/dii.##$))h%@AR4		?RA99>>!"% 	1CcEMM5<<#@A**w<!# ,:A, ) %++C0	1  rd   c                ,    t        | j                        S r\   )hashrl   rv   s    rb   __hash__zIterationRangesEntry.__hash__  s    DIIrd   c                X    t        |t              sJ | j                  |j                  k(  S r\   )r  r   rl   )rs   others     rb   __eq__zIterationRangesEntry.__eq__  s&    %!5666yyEJJ&&rd   )rl   r   rg   r   rh   r   r   r   r  rf   r   r   r   )rl   r   r   r   r   )r   r   r   r   )r&  objectr   r   )r   r   r   rk   r   r  r   r  r  r$  r'  r   r   s   @rb   r   r   Y  sf      	
    
.s
# 'rd   r   c                  ~     e Zd ZdZddd	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZddZddZddZdd	Zdd
Z	ddZ
ddZ xZS )DerivedIterationRangesRootae  A root with reduced numel/block_size derived from a parent tree.

    Used for the grouped reduction output: if the parent R tree covers 1024
    columns with local_reduction_size=128, the derived root covers 8 groups. It shares
    the parent's loop structure (same is_loop, prefix, loop variable) but
    has its own block geometry:

        parent R:  numel=1024, block_size=RBLOCK, offset=roffset
        derived R: numel=8,    block_size=RBLOCK//128, offset=roffset//128

    This lets the grouped reduction store at reduced-resolution offsets
    while still running inside the parent's reduction loop.
    reducedr   )name_suffixr   c                   t         |   | d|j                   ||j                  |j                  |j
                  |j                  |j                  |j                  |j                  |j                  
       || _        || _        || _        y )N_)
rl   ro   rp   r   rq   r   r   r   r   r   )rj   rk   rl   rp   r   rq   r   r   r   r   r   _block_size_block_offset_named_constants)rs   r  ro   r   r   r-  r   rt   s          rb   rk   z#DerivedIterationRangesRoot.__init__  s     	=&++/==,,==&& NN((____ 	 	
" &) /rd   c                    | j                   S r\   )r0  rv   s    rb   r   z%DerivedIterationRangesRoot.block_size  s    rd   c                    | j                   S r\   )r1  rv   s    rb   r   z'DerivedIterationRangesRoot.block_offset  s    !!!rd   c                ,    t        | j                        S r\   ry   rv   s    rb   r   z$DerivedIterationRangesRoot.index_sym  r{   rd   c                     | j                    dS )N_maskr  rv   s    rb   r   z$DerivedIterationRangesRoot.mask_name  s    ))E""rd   c                     yr   r   rv   s    rb   r   z1DerivedIterationRangesRoot.supports_constant_mask  s     rd   c                    | j                   S r\   )r2  rv   s    rb   r   z*DerivedIterationRangesRoot.named_constants  s    $$$rd   c                     yr   r   rv   s    rb   r   z4DerivedIterationRangesRoot.has_custom_codegen_header  r   rd   )r  r   ro   r   r   r   r   r   r-  r   r   r   r   r   r   r   r   r   r   )r   r   r   r   rk   r   r   r   r   r   r   r   r   r   s   @rb   r+  r+    s}    * %MO0#0 	0
 0 !0 0 K0 
0> "-#
%rd   r+  c                    | t        d      k(  ry| t        d      k(  ryt        j                  |       ryt        |       S )Ninfzfloat("inf")z-infzfloat("-inf")zfloat("nan"))floatmathisnanrepr)values    rb   constant_reprrB    s9    e	%-		E	;rd   CSEVariableType)boundra   c                  ,    e Zd ZU ded<   ded<   ded<   y)PartialAccumulater   buffer_namereduction_typer   rA  N)r   r   r   __annotations__r   rd   rb   rF  rF    s    Jrd   rF  c                  X    e Zd ZU dZded<   ded<   ded<   ded	<   ded
<   ded<   ded<   y)NodeInfozF
    Pre-computed node information for combo kernel partitioning.
    listnode_scheduledicttilingzdict | Nonetiling_scoresr   ro   rnumelrL   featuresr   is_persistent_reductionNr   r   r   r   rI  r   rd   rb   rK  rK    s0     LJK  !!rd   rK  c                  p    e Zd ZU dZeZded<   ded<   dZded<   ded	<   	 	 	 	 	 d;	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d< fd
Z	 	 d=dZ	d>dZ
d?dZd Zeed@d              ZdAdZdBdZed=d       ZdCdZ	 	 	 	 	 	 	 	 	 	 	 	 dDdZdEdZdFdZdGdZdCdZdCdZdHdZdIdZ	 	 	 	 	 	 dJdZ	 	 	 	 	 	 dJdZdKdZdLdZe 	 	 	 	 	 	 dMd        Z!e"e#jH                  jJ                  f	 	 	 	 	 	 	 dNd!       Z&e"e#jH                  jJ                  f	 	 	 	 	 	 	 dOd"       Z'	 	 	 	 dPd#Z(e"	 	 	 	 	 	 dQd$       Z)dRd%Z*dRd&Z+dSd'Z,	 	 	 	 dId(Z-dTd)Z.e/j`                  	 	 	 	 dUd*       Z1dVd+Z2dWd,Z3	 	 	 	 	 	 dXd-Z4d. Z5	 dY	 	 	 	 	 	 	 dZd/Z6e/j`                  d[d0       Z7d\d1Z8e d2        Z9d]d3Z:d4 Z;d5 Z<d6 Z=d7 Z>d8 Z?d9 Z@d^d:ZA xZBS )_r   zo
    Common base class for Triton/Halide codegen which both use flattened indexing rather than loop nests.
    zCallable[[sympy.Expr], str]sexprkexprFr   allow_block_ptrr   kernel_namec                    |i }t                    | _        |j                          _        t                _        t                _        |j                         D 	ci c]/  \  }}	|t        j                  j                  j                  |	      1 c}	} _        g  _        i  _        t!        j"                          _        |j'                          _        ||n j+                          _        | _        | _        ||n j3                          _        | _         j9                          _        d  _        t!        j"                          _        d _         tB        jD                  jF                  r j                  jH                  D ]h  }
tK        |
tL        jN                        stK        |
jP                  tR        jT                        sC|
jP                  jW                         dk(  sad _          n tX        jZ                  d fd       }| _.         j_                  |       d _0        d  _1        d  _2        g  _3         j                  ji                          _5        y c c}	}w )NFdotTc                    t         j                  j                  j                  | j	                               } j
                  D ]  }j                  | |      }  j                  |       S r\   )r?   r   r   simplify_with_rangesrn   range_treescombine_contiguous_dimscombine_modular_indexing_pairs)r   treers   s     rb   simplify_indexingz.SIMDKernel.__init__.<locals>.simplify_indexing?  sb    GG$$99%ARSE(( B44UDAB 66u==rd   r   )r   r   )6rj   rk   rR  get_mutations	mutationsr6   bodyindexing_coder}   r?   r   r   simplifynumelsr^  r   	itertoolscountr   rw   inside_reduction should_use_cooperative_reductioncooperative_reductionrP  rO  should_use_persistent_reductionpersistent_reductionmix_order_reductionwant_no_x_dimno_x_dimr   store_output_ctris_native_matmulr   r_   native_matmulrM  r  r   SchedulerNoder   r   ComputedBufferget_reduction_typer  cacherb  initialize_range_treersplit_size
min_xblock
min_rblocksaved_partial_accumulateselect_index_dtype_index_dtype)rs   rO  rR  r   override_persistent_reductionoverride_cooperative_reductionrP  rp  rp   valr   rb  rt   s   `           rb   rk   zSIMDKernel.__init__
  s    I !//1"$	+-FLlln
7BvsFAGG$$--c22
 79JL(0 ( 5 5 7 .9 +668 	"
 <I-3 -8 *557 	!
 *= **,%) ) 1 %==&&33 tY%<%<="499b.?.?@		446%?,0D) 
	> 
	> "3""9-&*&*AC% MM<<>g
s   "4I&c                    t         )u-  Generate template source code with fused prologues and epilogues.

        Subclasses override this to implement custom code generation.
        The default implementation raises NotImplementedError — the actual
        standard path lives in ``TritonTemplateKernel.codegen_template_body``.
        NotImplementedError)rs   
schedulingtemplate_nodeepilogue_nodesprologue_nodesbuf_name_to_prologue_groupprologue_preserves_zero_mask_fnrenders           rb   codegen_template_bodyz SIMDKernel.codegen_template_bodyP  s
      "!rd   c                    g S )a  Return epilogue nodes that were not fused into the kernel.

        These nodes need separate codegen (via ``call_kernel``) and must
        be excluded from ``mark_run`` in ``_codegen_single_template``.

        The standard path fuses all epilogues, so this returns ``[]``.
        ``ExternalTritonTemplateKernel`` overrides this for epilogues that
        don't read exactly one template output and cannot be fused.
        r   rv   s    rb   get_unfused_epiloguesz SIMDKernel.get_unfused_epiloguesb  s	     	rd   c                    d| dS )Nz<STORE_OUTPUT_>r   )rs   is     rb   _get_store_output_subgraph_namez*SIMDKernel._get_store_output_subgraph_namen  s    s!$$rd   c                n    t        | j                        }t        j                  |dz
  d      | _        |S )Nr@   )startstep)r   rs  ri  rj  )rs   totals     rb   get_store_output_countz!SIMDKernel.get_store_output_countq  s.    T**+ )eaia Hrd   c                :    t        d | j                  D              S )Nc              3  2   K   | ]  }t        |        y wr\   )r8   )r  rp   s     rb   r  z0SIMDKernel.num_reduction_dims.<locals>.<genexpr>y  s     I6&v.I   )sumrh  rv   s    rb   num_reduction_dimszSIMDKernel.num_reduction_dimsv  s     IT[[IIIrd   c                    t         r\   r  )rs   dtypes     rb   dtype_to_strzSIMDKernel.dtype_to_str{      !!rd   c                    | j                   S r\   )r  rv   s    rb   get_index_dtype_as_torch_dtypez)SIMDKernel.get_index_dtype_as_torch_dtype~  s       rd   c                @    | j                  | j                               S r\   )r  r  rv   s    rb   index_dtypezSIMDKernel.index_dtype  s      !D!D!FGGrd   c                     yr   r   rv   s    rb   rq  zSIMDKernel.want_no_x_dim  r   rd   c                   t        fdt        D              }| xs | }d	d}g d}	t        t        |	            }
ddg}|r|}n
|r|
}n|
|z   } |||      } ||	t              }g }t	        |      D ]s  \  }}t        |      }|j                  |      }|j                  |      }||n|}|j                  t        | d|   ||| ||xr | j                   ||dv 
             u |S )
Nc              3  ,   K   | ]  }|v s|  y wr\   r   )r  rp   rh  s     rb   r  z3SIMDKernel.construct_range_trees.<locals>.<genexpr>  s      %
61AF%
   	c                `    t        fd| D              D ci c]  \  }}||
 c}}S c c}}w )Nc              3  ,   K   | ]  }|v s|  y wr\   r   )r  r  r   s     rb   r  zOSIMDKernel.construct_range_trees.<locals>.filtered_index_map.<locals>.<genexpr>  s     2U3PT32Ur  )	enumerate)seqr   idxr  s    `  rb   filtered_index_mapz<SIMDKernel.construct_range_trees.<locals>.filtered_index_map  s3    )22U#2U)U%S#S  s   *)rX   rW   rV   rY   rZ   r   rV   )r   r   r   r   r   )r   zdict[Any, int])
r   all_prefixesrL  r   r  r8   r   r   r   ro  )rs   r   rk  rw   rh  rr  active_prefixesno_r_dimr  	grid_dimspointwise_tensor_dimsreduction_dimstensor_dimstensor_dim_mapgrid_dim_mapr^  r  rp   r   r   r   s       `                rb   construct_range_treesz SIMDKernel.construct_range_trees  s3    % %
!-%
 
 (';|+;	
 $	 $Xi%8 9(K/K/.@K ,KI))\B"?3 	IAv.v6L'++F3J#''/H!)AxE#he$6N'(J1J1J-J)% F]	& rd   c                    | j                  || j                  | j                  j                         | j                  | j
                        }| j                  j                  |       y r\   )r  rk  rR  rw   rh  rr  r^  extend)rs   r   r^  s      rb   rz  z SIMDKernel.initialize_range_tree  sR    00!!MM&&(KKMM
 	,rd   c                     y)zr
        Hook called right before codegen with every index that will be
        used in the fused kernel.
        Nr   )rs   indicess     rb   finalize_indexingzSIMDKernel.finalize_indexing  r  rd   c                v    | j                   }d| _         	 | j                  |||      || _         S # || _         w xY wr   )rk  store)rs   rl   r   rA  priors        rb   store_reductionzSIMDKernel.store_reduction  s;    %% %	*::dE51$)D!ED!s   / 	8c                     yr   r   rv   s    rb   rl  z+SIMDKernel.should_use_cooperative_reduction  r   rd   c                     yr   r   rv   s    rb   rn  z*SIMDKernel.should_use_persistent_reduction  r   rd   c                t    t        t        j                  j                  d | j                  D                    S )Nc              3  P   K   | ]  }|j                   j                            y wr\   )rn   r}   r  ra  s     rb   r  z(SIMDKernel.var_ranges.<locals>.<genexpr>  s"      *,0%%'*r  )rN  ri  chainfrom_iterabler^  rv   s    rb   rn   zSIMDKernel.var_ranges  s4    OO)) *484D4D* 
 	
rd   c                   t        |t              s|S |j                  d   }| j                  j	                  |      x}|S t        |||j                  i      }t        j                  j                  j                  |      }t        ||j                  j                         |j                  j                         j                         i      S Nr   )r  r   r  r   r   r;   r   r?   r   r   r`  rr   r   r   rz   )rs   r   rX   	tree_node	new_indexs        rb   r`  z)SIMDKernel.combine_modular_indexing_pairs  s    %1LJJqM..22155I>Luq)..&9:	GG$$CCIN	^^%%')B)B)D)K)K)MN
 	
rd   c                L   t        |t        j                        r*|j                  t              r|j                  |      \  }}nd }t        j                  j                  j                  ||      x}r!|\  }}t	        | j                  ||      |      S | j                  ||      S r\   )r  r   Addhasr   r   r?   r   r   expand_floor_div_combine_contiguous_dims)rs   r   ra  candidate_varsr/  
expand_resr  denominators           rb   r_  z"SIMDKernel.combine_contiguous_dims  s     eUYY'EIIh,? $ 3 3E :NA!N))::5.QQ:Q%/"I{D99)TJKXX00==rd   c                   t        |t        j                  t        j                  f      r|S |j	                  |      \  }}t        |      dk  r|S t        j                  j                  j                  ||t        |g||            \  }}}||k(  r|S |j                  |      }t        |t        t        | ||                        }	|	S )zI
        More aggressive simplification to merge contiguous dims
        r@   )r  r   r  r   r   r  r?   r   r   _simplify_loopsrC   r   r;   rN  zip)
rs   r   ra  r   r   	new_sizesreindex_prunenew_index_varsr  s
             rb   r  z#SIMDKernel._combine_contiguous_dims  s     eemmU\\:;L //6
Eu:?L%&WW%5%5%E%E7US&
"	7F L	2ud3z7>;R+S&TU	rd   c                      j                   d   j                  xs  j                  t        j                   fd       } |       S )Nc               3     K    j                   j                         s j                  rJ d  y r j                          d _        	 d  r j                          d _        y # d _        w xY ww)NFT)rR  rw   rk  codegen_body)rs   should_flushs   rb   ctxz)SIMDKernel.disable_reduction.<locals>.ctx  sl     ==--/0000 !!#$)D!-%%'(,%%s   AA5A) !A5)	A22A5)r^  r   rm  
contextlibcontextmanager)rs   r  r  s   ` @rb   disable_reductionzSIMDKernel.disable_reduction  sE    ''+33Qt7Q7Q		"	"	- 
#	-$ urd   c                    t        |      t        | j                        k(  sJ t        || j                        D cg c]  \  }}|j                  |       c}}S c c}}w r\   )r  r^  r  r   )rs   r   rh   rangess       rb   
set_rangeszSIMDKernel.set_ranges+  s]    7|s4#3#34444 #&gt/?/?"@
 V$
 	
 
s   Ac                    t        d |D              r| D cg c]  }g  c}g fS t        j                  j                  | D cg c]  }g  c}| D cg c]  }j	                  |       c}t        j                         dfd}	 	 	 	 	 	 dd}g }d}|D ];  }	g }
|	D ]  }j                  |d      r|
j                  d        )|t              k  r>j                  |   d      r)|dz  }|t              k  rj                  |   d      r)|dz   t              k  rj                  ||   |dz      z        rj                  ||   |dz      z        st        ||   |dz      z        |   }|dz      }t        |||z        }|
j                   |||g |||       ||dz   |       ||dz   |      g             0|dz   t              k  rj                  ||         sj                  t        ||         d      rfj                  ||         st        ||         |   }t        ||         }|
j                   ||g |||       ||dz   |      g             |t              k\  rt        |d      |
j                  t        j                   |||                   " |j                  |
       > t        d D              sJ d	 d
|        |fS c c}w c c}w c c}w )Nc              3  8   K   | ]  }t        |      d k(    ywr   Nr  )r  rh   s     rb   r  z5SIMDKernel._split_iteration_ranges.<locals>.<genexpr>9  s     6Fs6{a6s   c                    j                  |      }j                  |    |      st        |    |      t        |    |      | <   |    j	                  |       t              S r\   )rg  statically_known_multiple_of	CantSplitr   r   r   )r  r   
new_ranges	remainingsv	var_counts     rb   	add_rangez5SIMDKernel._split_iteration_ranges.<locals>.add_rangeA  sf    ;;t$D229Q<F	!d33#IaL$7IaLqM  &	?"rd   c                P     t              t               dz   k(  sJ d fd}|S )z
            Builds the nested expression:
              ((...((s1*v[i1] + v[i2]) * s2 + v[i3]) ... ) * sk + v[i(k+1)])
            r@   c                \    | d      }t        dd        D ]  \  }}||z  | |   z   } |S )Nr   r@   )r  )	flat_varsr   r   r  idxsr   s       rb   getterzISIMDKernel._split_iteration_ranges.<locals>.make_combined.<locals>.getterS  sH     a)!%ab2 5FAst8in4D5rd   )r  r   r   r   r  )r   r  r  s   `` rb   make_combinedz9SIMDKernel._split_iteration_ranges.<locals>.make_combinedJ  s+     t9E
Q... Mrd   r   r@   c                6    t         j                  j                  S r\   )r   r   Zero)r/  s    rb   r   z4SIMDKernel._split_iteration_ranges.<locals>.<lambda>a  s    EGGLL rd   r   c              3  t   K   | ]0  }t         j                  j                  j                  |      d k(   2 yw)r@   N)r?   r   r   guarding_hint_or_throwr  s     rb   r  z5SIMDKernel._split_iteration_ranges.<locals>.<genexpr>  s/      
@AAGG33A6!;
s   68zfailed to set ranges  )r  r   r   r   r   r   )r   r   r  z	list[int]r   z(Callable[[list[sympy.Expr]], sympy.Expr])r  r?   r   r   rg  ri  rj  r   r   r  statically_known_gtr  r  r   operator
itemgetter)groupsr   groupr/  gr  r  return_getters_groupscurrent_grouplength_groupreturn_getterssizesize1size2size3r  r  r  r  s                  @@@@rb   _split_iteration_rangesz"SIMDKernel._split_iteration_ranges2  s    6g66$*+5B+R//WW:@-AQb-A
-34R[[^4	OO%		# 	#	#	+4	5	" !## X	9LN$ U--dA6"))*@A#c)n49S9Sm,:
 "Q&M $c)n49S9Sm,: !1$s9~5":P:P)M2Y}q?P5QQ; ::i6=STCT9UU ( %m4yQRAR7SS 
 &m4E%ma&78E$T55=9E"))%"EN )- ? )-!*;U C )-!*;U C
 #Q&Y7**4=1IJ **8D)M:R+SUVW ::i6 (i.FGG%m4E$T9]+CDE"))%"G )- ? )-!*;U C	 %I6'a00")) ++ImT,JKeUl "((8qX	9t  
EN
 
 	9"9+Qwi8	9 
 000C , .B4s   	L	LLc                   t         j                  j                  }t        |d         dk(  r\|j	                  |t
        j                  j                        s2|j	                  t        |      t        |d         |z        r|d   |gfS |S )z1Fill in the reduction numel of lengths if missingr@   r   )	r?   r   r   r  r   r   r   r   r:   )clsr  r   reduction_numelr   s        rb   prepare_split_iteration_lengthsz*SIMDKernel.prepare_split_iteration_lengths  s{     77##wqz?a00%''++N00f%gaj)O;
 AJ 122rd   c                n    | j                  |||      }	 | j                  ||       y# t        $ r Y yw xY wNTF)r  r  r  )r  r  r   r  s       rb   is_compatiblezSIMDKernel.is_compatible  sB     55fgW	''8 		s   ( 	44c                >   | j                   D ci c]  }|j                  |j                   }}| j                  s0|D ]+  }t	        |      st
        j                  j                  ||<   - g |j                         }| j                  ||| j                        S c c}w )a5  
        Split and set iteration ranges for the kernel based on the provided lengths.

        This method maps the kernel's tiling structure to the node's iteration space,
        handling both pointwise and reduction dimensions appropriately.

        Args:
            lengths: A sequence of sequences of symbolic expressions representing
                    the sizes of different dimensions for each node.

        Returns:
            A list of lists of symbolic expressions representing the mapped
            iteration variables for each dimension.
        )r^  rp   ro   rk  r8   r   r   r   r   map_kernel_groups_to_node_sizesr  )rs   r   rtrO  rp   r  s         rb   split_and_set_rangeszSIMDKernel.split_and_set_ranges  s    $ 150@0@A""))RXX%AA $$  1&v.%*WW[[F6N1
 $6==?# 33FGT__UU Bs   Bc           
     F   t        |      t        |      k(  r!t        d t        ||      D              r || S | j                  ||      \  }}g t        j
                  j                   ||       }|D cg c]  }|D cg c]
  } ||       c} c}}S c c}w c c}}w )a  
        We may want to fuse `for i0 in s0*s1` into a tiled kernel with groups (s0, s1).

        To do this we need to split up the iteration space of i0 into something like:
            for i1 in s0:
              for i2 in s1:
                i0 = i1*s1 + i2
                ....

        This function matches and resplits lengths to the groups of
        this kernel to enable tiled + non-tiled fusions.
        c              3     K   | ]?  \  }}t         j                  j                  j                  t	        |      |z
        d k(   A ywr  r?   r   r   rg  r:   )r  rX   r  s      rb   r  z=SIMDKernel.map_kernel_groups_to_node_sizes.<locals>.<genexpr>  s@      /
1 GG%%mA&6&:;q@/
s   AA)r  r  r  r  ri  r  r  )	r  r  r   r  r  r  r   fnsfns	            rb   r  z*SIMDKernel.map_kernel_groups_to_node_sizes  s    & w<3v;&3 /
GV,/
 ,
 w'',/,G,GPW,X)
)LY__22:z3JKL8MN,"H,NN,Ns   7	B BBBc                6    t        |t        j                        S r\   )r   r   TMPrs   r   s     rb   is_indirect_indexingzSIMDKernel.is_indirect_indexing  s    "5$((33rd   c                    | j                  |      ry| j                         }dgt        |      z  }|j                  D ]t  }| j                  j                  |      !t        j                  t              sJ t        fdt        |      D        d       }|^||xx   j                  z  cc<   v t        j                  j                  j                  t!        fdt#        |d |D              D              S )NFr@   c              3  F   K   | ]  \  }}|j                   u s|  y wr\   )r  )r  r  ra  entrys      rb   r  z,SIMDKernel.is_broadcasted.<locals>.<genexpr>3  s      Rwq$TU\\=QRs   !!c              3  F   K   | ]  \  }} |       |      k7    y wr\   r   )r  	idx_range
iter_rangerg  s      rb   r  z,SIMDKernel.is_broadcasted.<locals>.<genexpr>@  s,      
%	: Y8J#77
   !c              3  4   K   | ]  }|j                     y wr\   )ro   r  s     rb   r  z,SIMDKernel.is_broadcasted.<locals>.<genexpr>C  s     CdtzzCs   )r#  active_range_treesr  r   r   r   r  r  r   r   r  rh   r?   r   r   rg  anyr  )rs   r   active_treesindex_numelsrz   tree_posr&  rg  s         @@rb   is_broadcastedzSIMDKernel.is_broadcasted#  s    $$U+..0sS..(( 	3F))--f5E}ell,?@@@ R)L"9RH  "ell2"#	3* 77##,, 
),ClC*
 
 	
rd   c                    t        |t              r)ddj                  t        | j                  |             dS | j                  | j                  |            S )a  
        Convert an index expr to a string that can be used in output code.
        e.g. a sympy expression "s2" may actually appear as "ks1" in the generated kernel.

        Index expressions often need to be passed in as arguments to the triton kernel.
        Rename_indexing and codegen_indexing keep track of the needed indices and add
        new parameters to the function signature.
        [r   ])r  rL  joinmapindex_to_strrW  rename_indexingr"  s     rb   r7  zSIMDKernel.index_to_strG  sN     eT"tyyT%6%6!>?@BBzz$..u566rd   c                ~   | j                  |      }t        |t        j                  j                  j
                        }t        |j                  t        j                              s(t        |j                  t        j                              r3|j                  t        j                  j                  j
                        }t        |j                  t        j                              r|j                  t        j                        D ]g  }|j                  }t        |      dkD  st        d |D              s1|t        j                  j                  j                  |      i}t        ||      }i | j                  |      }i }| j                         D ]e  }t        j                  j                  j!                  |j"                  d      s8|j$                  D ]  }t        j&                  j(                  ||<   ! g |rt        ||      }t+        |t,              s|n|j.                  d   }| j1                  |      S )Nr   c              3  p   K   | ].  }t        |t        j                  t        j                  f       0 y wr\   )r   r   r  PRECOMPUTED_SIZEr  s     rb   r  z.SIMDKernel.prepare_indexing.<locals>.<genexpr>h  s.      , #1tyy$2G2G&HI,s   46r@   )rb  r;   r?   r   r   precomputed_replacementsr  atomsr   floorceilingsubsr   r  lookup_precomputed_sizer,  r   ro   rm   r   r  r  r   r  codegen_indexing)	rs   r   ar!  replacements
simp_indexsingleton_replacementsra  rz   s	            rb   prepare_indexingzSIMDKernel.prepare_indexingT  s    &&u-5!''"2"2"K"KLu{{5;;'(CEMM0J,KJJqww//HHIE u{{5==)*[[/ 	< ..w<!# ,$, ) %&qww'7'7'O'OPQ'R#SL&ul;E	< ++E2
!#++- 	>D77##;;DJJJ -- >16&v.>	> "#J0FGJ )X>JJOOTUDV 	 $$Z00rd   c                r    | j                   D cg c]  }|j                  r| j                  s| c}S c c}w r\   )r^  rw   rk  )rs   ts     rb   r,  zSIMDKernel.active_range_trees  s5     ''
q~~AVAVA
 	
 
s   44c              #    K   | j                   }t        |      | _         | j                  j                          	 d || _         | j                  j                          y# || _         | j                  j                          w xY ww)z;Temporarily codegen against an alternate range-tree family.N)r^  rL  rb  r   )rs   r^  saveds      rb   use_range_treeszSIMDKernel.use_range_trees  ss     
   , 	**,	1$D""..0  %D""..0s   7BA  "B #BBc                4   t         j                  j                  j                  || j	                               }t        |j                  t              D ]  }|| j                  v si }| j                  |   j                         D ].  }t         j                  j                  j                  |      ||<   0 t        |      dkD  r5t        | j                  |   j                  |      | j                  |   _        | j                  |   j                           |S )Nr   r   )r?   r   r   r]  rn   sortedr   r   r   r  rA  r  r;   r   r  )rs   r   symrD  pss        rb   rB  zSIMDKernel.codegen_indexing  s    ww44T4??;LM$++5 	5Cd+++  "//4EEG TB'(ww'7'7'O'OPR'SL$T|$q(6@--c277$7D))#.3 %%c*224	5 rd   c                    t        d      )NzNYI: codegen_nan_checkr  rv   s    rb   codegen_nan_checkzSIMDKernel.codegen_nan_check  s    !":;;rd   c                    t        d      )Nz$NYI: iteration_ranges_codegen_headerr  )rs   r&  codes      rb   iteration_ranges_codegen_headerz*SIMDKernel.iteration_ranges_codegen_header  s    
 ""HIIrd   c                    t         j                  j                  }t        | j                  j
                        D ]  }|j                  |        y r\   )r?   r   wrapper_coder   r  workspace_argsgenerate_workspace_deallocation)rs   wrapperwss      rb   deallocate_workspacesz SIMDKernel.deallocate_workspaces  s=    ''&&499334 	8B33B7	8rd   c                    t        d      )NzNYI: call_kernelr  )rs   rl   r   deallocate_wss       rb   call_kernelzSIMDKernel.call_kernel  s     ""455rd   c              #     K   | j                   }| j                  }|rt        j                  ||      }t	        j
                  |      }|| _         || _        	 | || _         || _        y# || _         || _        w xY ww)z:Context manager to add an additional mask to tl.load/storeN)
_load_mask_load_otherr=   logical_andr>   _unwrap)rs   r   rA  r  	prior_vals        rb   
mask_loadszSIMDKernel.mask_loads  sy      $$	??4/D!!$' 	)J#DO(D $DO(Ds   AA=A* A=*A::A=c                (   | j                   j                         D ci c]  \  }}||j                   }}}t        ||      }i }| j                  D ]7  }t        |j                        }t        ||di      t        ||di      z
  ||<   9 |S c c}}w )a\  
        This gets the stride of the index for each of the tiling variables
        (technically, it does it at index 0)

        For example, if
        xindex = x0 + 512*x1 + 1024*r0
        x0 = (xindex//512)
        x1 = (xindex % 512)
        r0 = rindex // 1024

        this function would return
        {xindex: 512, rindex: 1024}
        r@   r   )r   r}   r   r;   r^  r9   rl   )	rs   r   kvindex_to_tile_indexesindex_in_tile_varsstrides
range_treer   s	            rb   get_strides_of_loadzSIMDKernel.get_strides_of_load  s     8<7L7L7R7R7T Utq!AFF U U'/DE** 	J":??3A#$6A?*"QFC GAJ	
  !Vs   Bc                \    t        |t              rt        t        | |            S  | |      S r\   )r  r   r6  )r  rA  s     rb   _map_tuple_or_scalarzSIMDKernel._map_tuple_or_scalar  s'    eU#R((%yrd   c                    t        j                  | j                  j                        D cg c]  }|j	                          }}t        t        d |            S c c}w r\   )rK   
only_nodesrR  rM  estimate_flopsr  filter)rs   r   flopss      rb   rs  zSIMDKernel.estimate_flops  sX     +55dmm6Q6QR
 !
 
 6$&''	
s   Ac           	        g }t        t        | j                  j                  j	                                     }| j                  j                         \  }}}}| j                  j                         }t        j                  j                  j                  t        | j                  j	                                     }t        |      D ]2  \  }}||vr|j                  d       t        j                  j!                  |      }	t        j                  j                  j                  |	      }
|
|kD  rwt#        t$                  }d}||   D ]M  }t'        |t(        t*        f      r|j-                  d|        |dz  }3|j-                  |j.                         O t        |      |z  }n|
}t        j                  j1                  |      }t3        |      }|j                  ||z  dt5        ||k        z   z         5 t7        |      S )a+  
        Try the best to estimate the total size (in bytes) of the
        kernel's inputs and outputs, which is used for estimating the memory
        throughput of this kernel. This information is used for checking how
        far we are from the peak memory bandwidth. It's important that
        we want to avoid overestimating the sizes of the inputs and outputs,
        because it can wrongfully give us a very large memory traffic value,
        which may be even larger than the theoretical bandwidth and thus
        become very misleading. This is particularly problematic for cases
        where we slice some inputs. In those cases, we should only count
        the size of the "slices" instead of the original inputs, because
        only the slices contribute to the real memory traffic.
        r   no_index_dep_r@   )r  r<   r  inplace_buffersr   python_argdefsrR  buf_accessesr?   r   r   r   r:   rh  r  r   	get_numelr   r   r  r!   r"   r   r   	get_dtyper5   r   r  )rs   nbytesninplace_argsr/  	call_argsrz  	out_numelr  r   	arg_numelbuf_sizer  no_index_dep_countdepro   r  
dtype_sizes                    rb   estimate_kernel_num_bytesz$SIMDKernel.estimate_kernel_num_bytes  s    F499#<#<#C#C#EFG!YY5579a}}113 GG$$66$++,,./
	  	* 	MFAs ,&a ))#.Iww''99)DH)# %S/+%&"', /C!#'9:m4F3G$HI*a/*CII./ Gy0 GG%%c*E'.JMM%*,CM8I4J0JKL;	M< 6{rd   c           	     &   t        | j                  j                        dk(  rEt        | j                  j                        dk(  r#t        | j                  j                        dk(  ry| j                  j                         \  }}}}d}|D ]F  }t        j                  j                  |      }|s&|j                         }	t        |	j                        dk(  sOt        |	j                  D 
cg c]
  }
|
dk(  s	|
 c}
      dk(  r|t        j                  |	j                        }||}||k7  st        d| dd| d	| z         }t        j!                  |       |D cg c]m  }t        j                  j                  |      rJt        j                  t        j                  j#                  |      j                         j                        ndo }}|D cg c]Z  }t        j                  j                  |      r7t        j                  j#                  |      j                         j                  nd\ }}|D cg c]@  }|t        j                  j$                  v rd
n|t        j                  j&                  v rdndB }}|D 
cg c]  }
|
j(                   }}
t        d| d| d| d| d| dz         }t        j!                  |        y t+        d| d      }t        j!                  |       yc c}
w c c}w c c}w c c}w c c}
w )zr
        Print message if the kernel have mixed layout inputs.
        Only care about 4D tensor for now.
        r@   r   N   r   zExpected stride order z, but found stride orderr   z for kernel 
GraphInputIntermediateBufferz  param names z
  buf names z
  strides z	
  sizes z
  sources 
z%All the inputs for the triton kernel z have uniform layout)r  r  input_buffersoutput_buffersrx  ry  r?   r   try_get_buffer
get_layoutr  r   get_stride_orderstrider.   logwarning
get_buffergraph_inputsname_to_bufferrl   r+   )rs   rY  argdefsr  
_signaturer/  uniform_stride_orderarg_namebuflayoutrX   stride_ordermsgrl   stride_order_list	size_listsource_listargdef_namess                     rb   warn_mix_layoutzSIMDKernel.warn_mix_layout;  s    		''(A-DII,,-2DII--.!3
 ,0II,D,D,F)J#! 0	H''((2C^^%F6;;1$6;;9a!q&9:a?!226==A'/+7()\9%01E0FF^_l^<}EFC KK$ %.) ! 7711$7 ++GG..t4??AHH "	")% ) %.	! ! 7711$7 **40;;=BB!"!I ! %.# !	  177#7#77 %  177#9#99 2!	"#K # 5<#<qAFF#<L#<%(nYK|\m[no&ykk]"MNC KK$a0	b 3K=@TU
 	C[ :)!# $=s'   -
K:
8K:
"A2K?AL?AL	
Lc                   t        j                  ||d|      }d| _        t        j                  | j                  j
                  |      }t        j                  ||      }d| _        t        j                  ||      }t        j                  ||      }t        j                  ||d|      }t        j                  |||f      S )Nr  FT)r=   	reductionrk  
index_exprrR  r  truedivsubmulr>   rd  )	rs   r  rA  sum_rQ  meandxdx2m2s	            rb   welford_reduce_fallbackz"SIMDKernel.welford_reduce_fallback  s    }}UE5%8 % = =uE{{4( $WWUD!ggb"o]]5%4!!4V"455rd   c                    t        j                  ||d|      }t        j                  ||      }t        j                  |      }t        j                  ||d|      }t	        j
                  ||f      S )Nmaxr  )r=   r  r  expr>   rd  )rs   r  rA  vmaxr  r  vsums          rb    prepare_softmax_twopass_fallbackz+SIMDKernel.prepare_softmax_twopass_fallback  s\    }}UE5%8ggeT"ggcl}}UE5#6!!4,//rd   c                    t         r\   r  rv   s    rb   codegen_kernelzSIMDKernel.codegen_kernel  r  rd   c                     y r\   r   rv   s    rb   r  zSIMDKernel.codegen_body      rd   c                     y r\   r   )rs   r&  s     rb   r  z)SIMDKernel.codegen_iteration_ranges_entry  r  rd   )NNNNF)rO  dict[str, sympy.Expr]rR  rL   r   r   r  bool | Noner  r  rP  dict[str, sympy.Expr] | Nonerp  r   r   r   r   )r   	list[Any])r  r   r   r   r(  )r  torch.dtyper   r   )r   r  r   )r   r   rk  r   rw   r   rh  r  rr  r   r   list[IterationRangesRoot])r   zdict[str, str]r   r   )r  Sequence[sympy.Expr]r   r   rl   r   r   r   rA  rB   r   r   )r   r   r   r   r   r   )r   r   ra  r   r   r   )r   z'contextlib.AbstractContextManager[None])r   r   r   r   )r  Iterable[sympy.Expr]r   Sequence[Sequence[sympy.Expr]]r   zStuple[list[list[sympy.Expr]], list[list[Callable[[list[sympy.Expr]], sympy.Expr]]]])r  r  r   r  r  r   r   r  )r  r  r   r  r  r   r   r   )r   r  r   list[list[sympy.Expr]])r  r  r   r  r   r  )r   r   r   r   )r   r   r   r   )r   r  )r^  zSequence[IterationRangesRoot]r   Iterator[None])r   r   r   r   r   )r&  r   rT  r6   r   r   r   )rl   r   r   zIRNode | Noner^  r   r   r   )r   zstr | OpsWrapperrA  int | floatr   zIterator[str])r   r   r   r   )r   r   )r&  r   )Cr   r   r   r   pexprrV  rI  rX  rk   r  r  r  r  r   r2   r  r  r  r  rq  r  rz  r  r  rl  rn  rn   r`  r_  r  r  r  staticmethodr  classmethodr   r   r   r  r  r  r  r#  r1  r7  rG  r,  r  r  rL  rB  rR  rU  r\  r_  rf  rn  rp  rs  r  r  r  r  r  r  r  r   r   s   @rb   r   r     s    */E&.&&!OT! ,0596:6:$)D?%D? %D? )	D?
 (3D? )4D? 4D? "D? 
D?L" 
"$
%
 J  J"! H H5(5 5 	5
 &5 5 
#5n-*

>>':>	>':	(0
 H1$H1/MH1
H1 H1T 
 ',ggkk	$ 0 $	
 
( & 
 ',ggkk	$ 0 $	
 
  V5 V	 VD O$O 0O
 
 O O84"
H70101 
01d
 181	1 1"<J"J J 
	J8 LP66,6DH6	6
 ) )"0  
(@DFP
60"rd   r   T)frozenc                  &    e Zd ZU dZded<   ded<   y)_IterationSpacea  Source domain used to remap a pointwise body.

    ``groups`` are source-domain extents. ``values`` are the matching
    codegen-side symbolic index expressions in the outer kernel.

    For a grouped reduction body with logical ranges [B, D // G, G],
    ``groups`` may be [B, D // G, G] while ``values`` are the concrete
    symbols the parent kernel is using for those coordinates.
    r  r  r   NrT  r   rd   rb   r  r    s     !   rd   r  c                      e Zd ZU dZded<    ej                  e      Zded<   dZ	ded	<   dd
Z
ddZej                  dd       ZddZej                  dd       Zy)_DerivedIterationFamilyaP  Iteration family for a nested-reduction consumer stage.

    Two configurations:
      - reduced-output: swaps in a derived grouped-axis tree; ``index_subs``
        rewrites body vars to the active tree symbols
      - parent-full: reuses outer trees; store_cache values are broadcast-lifted
        lazily through CSE when needed
    ztuple[IterationRangesRoot, ...]r^  )default_factoryr   
index_subsFr   _headers_emittedc                8    |j                  | j                        S r\   )r@  r  r"  s     rb   remap_indexz#_DerivedIterationFamily.remap_index  s    zz$//**rd   c                    t        |j                        t        | j                        k7  ryt        d t        |j                  | j                        D              S )NFc              3  *   K   | ]  \  }}||u   y wr\   r   )r  activeexpecteds      rb   r  z7_DerivedIterationFamily.is_active_on.<locals>.<genexpr>  s!      
  h
s   )r  r^  r  r  rs   rq   s     rb   is_active_onz$_DerivedIterationFamily.is_active_on  sP    v!!"c$*:*:&;; 
$'(:(:D<L<L$M
 
 	
rd   c              #     K   | j                  |      rd y| j                  |      5  d ddd       y# 1 sw Y   yxY ww)z>Activate this family if it isn't already active on ``kernel``.N)r  activater  s     rb   ensure_activez%_DerivedIterationFamily.ensure_active  s<      V$v&   s   (A8	AAAc                    | j                   ry | j                  D ]Y  }t        |t              s|j                  xr |j
                  }|r|j                  n|j                  }|j                  ||       [ d| _         y r   )	r  r^  r  r+  r   rk  rf  re  rU  )rs   rq   ra  emit_in_loop_bodytargets        rb   ensure_headersz&_DerivedIterationFamily.ensure_headers  sv    
   $$ 	ED$ :; %)LL$LV5L5L!1B--66tVD	E !%rd   c              #     K   | j                  |       |j                  | j                        5  d  d d d        y # 1 sw Y   y xY wwr\   )r  rL  r^  r  s     rb   r  z _DerivedIterationFamily.activate  sA      	F###D$4$45 		 	 	s   -A	=	A	AA	Nr  )rq   SIMDKernel[Any]r   r   )rq   r  )rq   r  r   r   )rq   r  r   r  )r   r   r   r   rI  dataclassesfieldrN  r  r  r  r  r  r  r  r  r  r   rd   rb   r  r    sv     101B1B1BSW1XJ.X"d"+
  %"  rd   r  c                  N    e Zd ZU dZded<   ded<   ded<   ded<   d	ed
<   d	ed<   y)_GroupedReductionVarsa  Variables used to call the grouped-reduction body.

    ``iter_remapped`` and ``reduce_remapped`` are the iteration/reduction
    arguments passed to the grouped reduction's loop body. ``passthrough`` is
    the non-grouped axis, while ``group_index`` and ``local_reduction`` are
    the two pieces of the grouped parent axis after it is split into
    [num_groups, local_reduction_size]. ``group_index_var`` is None only for
    the single-group case, where the group index is the constant 0.
    r   iter_remappedreduce_remappedr   passthrough_iter_varzsympy.Symbol | Nonegroup_index_varr   group_index_exprlocal_reduction_varNrT  r   rd   rb   r  r    s-     $#%%&&((  ##rd   r  c                     e Zd ZU dZded<   ded<   ded<   ded<   e	 	 	 	 	 	 	 	 d d	       Zed!d
       Zed!d       Z	ed"d       Z
ed#d       Zed$d       Zed$d       Zed#d       Zed#d       Zed%d       Zed#d       Zed#d       Zed&d       Zed"d       Zed'd       Z	 	 	 	 d(dZ	 	 	 	 d)dZ	 	 d*dZ	 	 	 	 d+dZ	 	 	 	 	 	 d,dZ	 	 	 	 	 	 d,dZ	 	 	 	 	 	 	 	 d-dZy)._GroupedReductionLayouta  Geometry of the grouped reduction (the consumer reduction in a
    nested-reduction pair). Describes how the outer kernel's tile is
    reshaped into [num_groups, local_reduction_size], which axis is reduced,
    and how reduced values are lifted back for epilogues.
    r   x_treer_treesympy.Integerlocal_reduction_sizer   local_reduction_in_rc                    t        |j                        dk(  sJ |j                  \  }}|j                  dk(  sJ  | ||||      S )Nr   rX   )r  r  r  r  )r  r^  rp   )r  rq   r  r  r  r  s         rb   from_kernelz#_GroupedReductionLayout.from_kernel  sW     6%%&!+++++}}###!5!5	
 	
rd   c                J    | j                   r| j                  S | j                  S )z<The tree decomposed into [num_groups, local_reduction_size].)r  r  r  rv   s    rb   
group_treez"_GroupedReductionLayout.group_tree!       #77t{{HT[[Hrd   c                J    | j                   r| j                  S | j                  S )z5The tree that passes through unchanged (not grouped).)r  r  r  rv   s    rb   passthrough_treez(_GroupedReductionLayout.passthrough_tree&  r  rd   c                "    | j                   rdS dS )Nr@   r   r  rv   s    rb   parent_axisz#_GroupedReductionLayout.parent_axis+       --q414rd   c                n    d| j                   j                  j                         j                  d      z   S )Nnested_r/  )r  rp   r   removesuffixrv   s    rb   group_prefixz$_GroupedReductionLayout.group_prefix0  s+    4??11779FFsKKKrd   c                L    t        j                  | j                   ddd      S )N_LOCAL_REDUCTION_SIZETr   r   r   r  rv   s    rb   local_reduction_size_symz0_GroupedReductionLayout.local_reduction_size_sym4  s+    ||  !!67
 	
rd   c                L    t        j                  | j                   ddd      S )N_REDUCED_BLOCKTr   r  rv   s    rb   reduced_block_symz)_GroupedReductionLayout.reduced_block_sym<  s*    ||  !0
 	
rd   c                6    | j                   j                         S r\   )r  r   rv   s    rb   parent_blockz$_GroupedReductionLayout.parent_blockD  s    --//rd   c                6    | j                   j                         S r\   )r  r   rv   s    rb   passthrough_blockz)_GroupedReductionLayout.passthrough_blockH  s    $$3355rd   c                V    t        | j                  j                  | j                        S r\   )r   r  ro   r  rv   s    rb   
num_groupsz"_GroupedReductionLayout.num_groupsL  s    --t/H/HIIrd   c                ,    t        | j                        S r\   )r   r  rv   s    rb   num_groups_strz&_GroupedReductionLayout.num_groups_strP  s    4))**rd   c                ,    t        | j                        S r\   )r   r  rv   s    rb   local_reduction_size_dimz0_GroupedReductionLayout.local_reduction_size_dimT  s    40011rd   c                    | j                   r#| j                  | j                  | j                  fS | j                  | j                  | j                  fS )z,Shape used before the local group reduction.)r  r  r  r  rv   s    rb   reshape_shapez%_GroupedReductionLayout.reshape_shapeX  sY     $$&&##--  ))""
 	
rd   c                "    | j                   rdS dS )z@Axis reduced in [passthrough, num_groups, local_reduction_size].r   r@   r  rv   s    rb   reduce_axisz#_GroupedReductionLayout.reduce_axisg  r   rd   c                z    | j                   r| j                  | j                  fS | j                  | j                  fS r\   )r  r  r  rv   s    rb   output_shapez$_GroupedReductionLayout.output_shapel  s9    $$**D,?,?@@##T%;%;<<rd   c                   t         j                  j                  j                  | j                  d      rEt
        j                  j                  }d }| j                  j                         j                         }n6| j                  j                  | j                  | j                  g      \  }}|}| j                  j                  | j                  j                  g      d   }|g}t        |j                         dk(  r| j"                  r||gn||g}n_t        |j                         dk(  r<| j"                  r|| j                  z  |z   n|| j                  j                  z  |z   g}nt%        d      t'        ||||||      S )Nr@   r   r   z1nested grouped reduction expects 1 or 2 iter vars)r?   r   r   r   r  r   r   r  r  r   rz   r   r  r  ro   r  	iter_varsr  AssertionErrorr  )rs   re  r  r  r  passthrough_varr  r  s           rb   construct_group_reduction_varsz6_GroupedReductionLayout.construct_group_reduction_varsr  sv    7733DOOQG  %ww||"O"&//"<"<">"E"E"G37??3L3L$";";<40O0  / //99""(()

 //t~~!# ,, !"23&8 
  A% ,,  $//14DD%(=(=(C(CCoUM !!TUU$
 	
rd   c                ~    d fd} | j                         } | j                        }i } j                  r|n|} j                  r|n|}|j                         j	                         ||j
                  <   |j                  +|j                         j	                         ||j                  <   t        |||f      S )Nc           	     p   | j                   ur| S j                  j                  df}j                  t	        | j                         j                        df}t        | t	        | j                  j                        j                  t	        | j                         j                        ||f      S )NT)ro   r   r   r   )	r  r  r  r  r   r   r+  ro   r   )ra  local_reduction_size_constreduced_block_constrs   s      rb   buildzA_GroupedReductionLayout.make_reduced_output_family.<locals>.build  s    4??* --))*& &&*D,I,IJ#
 .tzz4+D+DE11%%%')F)F "<=P Q rd   )r  r^  )ra  r   r   r   )r  r  r  r   rz   r  r  r  )rs   group_reduction_varsr%  reduced_x_treereduced_r_treer  reduced_passthrough_treereduced_group_trees   `       rb   make_reduced_output_familyz2_GroupedReductionLayout.make_reduced_output_family  s    	4 t{{+t{{+57
"77N^ 	! #77N^ 	 %//188: 	'<<=  //; #--/668 +;;< '!'8
 	
rd   c                F    t        | j                  | j                  f      S )N)r^  )r  r  r  rv   s    rb   make_parent_full_familyz/_GroupedReductionLayout.make_parent_full_family  s!     'dkk2
 	
rd   c                   | j                   s\| j                  | j                  | j                  j                  g}|j
                  |j                  |j                  g}t        ||      S t        | j                  j                  | j                  j                  g| j                  j                         j                         | j                  j                         j                         g      S r\   )r  r  r  r  ro   r  r  r  r  r  r  r   rz   )rs   r&  source_groupssource_valuess       rb   parent_full_iteration_valuesz4_GroupedReductionLayout.parent_full_iteration_values  s     (())%%++M %55$88$99M
 #  [[ 1 12[[##%,,.0F0F0H0O0O0QR
 	
rd   c                    |j                   J |j                  t        |j                        dk  r|S | j                  ||d      S )Nr   Fmaterialize_singletonr  r   r  %_broadcast_value_to_parent_resolutionrs   rq   rA  s      rb   *maybe_broadcast_value_to_parent_resolutionzB_GroupedReductionLayout.maybe_broadcast_value_to_parent_resolution  sV    
 {{&&&;;#ekk"2Q"6 L99"' : 
 	
rd   c                    |j                   J |j                  t        |j                        dk\  sJ d       | j                  ||d      S )Nr   z;grouped reduction input must have a known parent-tile shapeTr3  r5  r7  s      rb   ensure_parent_tile_resolutionz5_GroupedReductionLayout.ensure_parent_tile_resolution  sa    
 {{&&&{{&3u{{+;q+@ 	
I	
@ 99"& : 
 	
rd   c               \   |j                   J |j                  t        |j                        dk\  sJ t        |j                  | j                           }|| j
                  k(  s|dk(  r|s|S | j                  }| j
                  }|dk(  r&| j                  r|df}||f}||f}	nfd|f}||f}||f}	nY| j                  r'| j                  }
| j                  }||
df}||
|f}||f}	n&| j                  }
| j                  }|
d|f}|
||f}||f}	|j                  ||||	|j                   |	      }|S )Nr   r   r@   )rA  pre_broadcast_shapebroadcast_shapefinal_shaper  	out_shape)r  r   r  r   r  r  r  r  r  r  emit_broadcast_via_reshape)rs   rq   rA  r4  
parent_dimpassthrough_extentparent_extentr<  r=  r>  r  r  s               rb   r6  z=_GroupedReductionLayout._broadcast_value_to_parent_resolution  s    {{&&&{{&3u{{+;q+@@@T%5%567
***#&;L!33)) (('91&=##5}"E1=A'(*<&=##02D"E,.@A&&,,J#'#@#@ #5z1"E1:?STO-}=K,,J#'#@#@ #-q2D"E)+?ASTO(*<=K11 3+#++! 2 
 rd   N)rq   r  r  r  r  r   r   r  )r   r   r(  r   r   r   )r   ztuple[str, str, str])r   ztuple[str, str])re  r   r   r  )r&  r  r   r  )r   r  )r&  r  r   r  )rq   rQ   rA  rB   r   rB   )rq   rQ   rA  rB   r4  r   r   rB   )r   r   r   r   rI  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r+  r-  r1  r8  r:  r6  r   rd   rb   r  r    s8     ''

 ,
 #	

 
!
 
  I I I I 5 5 L L 
 
 
 
 0 0 6 6 J J + + 2 2 
 
 5 5 = =
/
/
 
/
b3
33
 
!3
j
	 

$9
	
.

 
 
	
$

 
 
	
 22 2
  $2 
2rd   r  c                  .    e Zd ZU dZded<   ded<   ddZy)	_ParentFullLoadTransformz9Lift loaded values to parent-full resolution when needed.rQ   rq   r  r  c                N    | j                   j                  | j                  |      S r\   )r  r8  rq   )rs   rA  s     rb   applyz_ParentFullLoadTransform.applyT  s#    {{EEKK
 	
rd   N)rA  rB   r   rB   )r   r   r   r   rI  rG  r   rd   rb   rE  rE  M  s    C##
rd   rE  c                  d     e Zd ZdZdd	 	 	 	 	 	 	 d fdZd	dZ	 	 	 	 	 	 	 	 	 	 d
dZddZ xZS )_GroupedReductionOpsHandlera	  Group-reduction stage handler: reshape parent-full tile into
    [num_groups, local_reduction_size], reduce over the local axis, and store
    the result.
    Loads go through the normal CSE path, then may be lifted to parent
    resolution before the reshape.
    Nload_transformc               \    t         |   |       || _        || _        || _        || _        y r\   )rj   rk   _kernel_layout_family_load_transform)rs   innerrq   r  familyrK  rt   s         rb   rk   z$_GroupedReductionOpsHandler.__init__c  s/     	-rd   c                    | j                   j                  ||      }| j                  | j                  j                  |      }|S r\   )_innerloadrP  rG  )rs   rl   r   rA  s       rb   rU  z _GroupedReductionOpsHandler.loadr  sA      u-+ ((..u5Erd   c                z   | j                   }| j                  j                  ||      }| j                  j	                  |       |j                  || j                  j                  |      }|xj                  dz  c_        |j                  ||| j                  j                  || j                  j                        S )zAReshape the parent-full tile and reduce over the local reduction.r@   )rM  rN  r:  rO  r  emit_reshaper  num_reductionemit_reducer  r  )rs   r  	src_dtyperH  rA  rh  reshapeds          rb   r  z%_GroupedReductionOpsHandler.reductionz  s     LL::1eD 	##A&>>%)C)CYO	1}}LL$$LL%%
 	
rd   c                    | j                   j                  |      }| j                   j                  | j                        5  | j                  j                  |||       d d d        y # 1 sw Y   y xY wr\   )rO  r  r  rM  rT  r  )rs   rl   r   rA  remapped_indexs        rb   r  z+_GroupedReductionOpsHandler.store_reduction  sW    11%8\\''5 	;KKdNE:	; 	; 	;s   A((A1)rq   rQ   r  r  rR  r  rK  _ParentFullLoadTransform | Nonerl   r   r   r   r   rB   )
r  r  rZ  r  rH  r   rA  rB   r   rB   r  )	r   r   r   r   rk   rU  r  r  r   r   s   @rb   rI  rI  [  sy     ;?. .
 (. (. 8.

 
 	

 
 

0;rd   rI  c                  Z     e Zd ZdZdd	 	 	 	 	 d fdZddZ	 d		 	 	 	 	 	 	 	 	 d
dZ xZS )_PointwiseRemapHandlera@  Pointwise bodies at a remapped iteration range.

    Unlike _GroupedReductionOpsHandler (which performs reshape+reduce), this
    handler runs pure pointwise bodies for reduced/parent-full prologues
    and epilogues. Loads use the normal CSE path after index remapping, then
    may be lifted to parent resolution.
    NrJ  c               N    t         |   |       || _        || _        || _        y r\   )rj   rk   rM  rO  rP  )rs   rQ  rq   rR  rK  rt   s        rb   rk   z_PointwiseRemapHandler.__init__  s(     	-rd   c                6   | j                   j                  |      }| j                   j                  | j                        5  | j                  j                  ||      }d d d        | j                  | j                  j                        }S # 1 sw Y   2xY wr\   )rO  r  r  rM  rT  rU  rP  rG  )rs   rl   r   r]  rA  s        rb   rU  z_PointwiseRemapHandler.load  s~    11%8\\''5 	;KK$$T>:E	;+((..u5E		; 	;s   BBc                    | j                   }| j                  j                  |      }| j                  j                  |      5  | j                  j                  ||||       d d d        y # 1 sw Y   y xY w)N)mode)rM  rO  r  r  rT  r  )rs   rl   r   rA  re  rh  r]  s          rb   r  z_PointwiseRemapHandler.store  sg     LL11%8\\''* 	FKKdNEE	F 	F 	Fs    A,,A5)rq   r   rR  r  rK  r^  r_  r\   )
rl   r   r   r   rA  rB   re  r   r   r   )r   r   r   r   rk   rU  r  r   r   s   @rb   ra  ra    su     ;?. .
 (. 8. 
F
F 
F 	
F
 
F 

Frd   ra  c                  @   e Zd ZU dZeZded<   d Zd ZeZ	eZ
d Zd Zd Zd	 Z	 d?	 	 	 d@dZd Zd Z	 dA	 dBdZ	 	 	 	 	 	 	 	 	 	 	 	 dCdZ	 	 	 	 	 	 	 	 dDdZe	 	 	 	 	 	 dEd       Z	 	 	 	 	 	 dFdZe	 	 	 	 	 	 dGd       Z	 	 dHdZd
d	 	 	 	 	 	 	 dIdZ	 dA	 	 	 dJdZ	 	 dKdZe	 	 	 	 	 	 dLd       Z	 dM	 	 	 	 	 	 	 dNdZdOdZ dPdZ!dd	 	 	 	 	 dQd Z"	 	 	 	 dRd!Z#d" Z$dd#d$Z%	 	 	 	 dSd%Z&dTd&Z'	 	 	 	 	 	 dUd'Z(dd
d(	 	 	 dVd)Z)d* Z*	 	 	 	 	 	 dWd+Z+dXd,Z,e	 	 	 	 	 	 dYd-       Z-	 	 dZ	 	 	 	 	 	 	 	 	 	 	 	 	 d[d.Z.d/ Z/e e0jb                  d0      d\d1              Z2e	 	 	 	 	 	 d]d2       Z3e	 	 	 	 	 	 d^d3       Z4e	 	 	 	 	 	 	 	 d_d4       Z5e	 	 d`d5       Z6e	 	 	 	 	 	 	 	 	 	 dad6       Z7e	 	 	 	 	 	 	 	 dbd7       Z8e	 	 	 	 	 	 	 	 dcd8       Z9ee:jv                  jx                  d
f	 	 	 ddd9       Z=ee:jv                  jx                  d
f	 	 	 ded:       Z>d; Z?dfd<Z@	 dg	 dhd=ZAd> ZBy
)iSIMDSchedulingzo
    Single Instruction Multiple Data parent class used for fusion across
    multiple different backends.
    z	type[Any]kernel_typec                &    t        d |D              S )Nc              3     K   | ]6  }t         j                  j                  j                  t	        |             8 y wr\   r  r  s     rb   r  z*SIMDScheduling.group_fn.<locals>.<genexpr>  s*     PQQWW%%..}Q/?@Ps   <>r   )rs   r   s     rb   group_fnzSIMDScheduling.group_fn  s    P%PPPrd   c                	   t        |t        j                        st        |t        j                        r t        j                  j                  ||      S |j                  \  }\  }|j                  \  }\  t        ||      }|j                         r)|j                         s|j                         rB |d       y|j                         r)|j                         s|j                         r	 |d       y|j                         r|j                         r|k(  xr k(  }|sddlm	} |j                  ||      }|s,ddlm
} |j                  ||      xr |j                  ||      }|s |d|       |r|j                         s|j                         ra|j                         s||}}| j                  |j                         |      t        fd|j                         D              s	 |d       y|S |j                         s|j                         s|k(  rk(  s|j!                         s |d	|       y|j                         D ]`  }	|	j!                         r nN|	j#                         |j%                         z  s7|	j                  \  }\  }
}||
k(  r|k(  rT |d
||
|        y ||fD ]  }|j!                         s y | j                  |j                         |      }| j                  |j                         |      }| j                  |j                         |j                         z   |      }t&        j(                  j*                  rVd}t-        |      dkD  r%t-        |      dkD  r||cxk(  xr |k(  nc }n||k(  }nt-        |      dkD  r||k(  }|s |d|||       yy|j                         s|j                         rɉdk(  rdk7  sJ |z  k(  rt        fd|j                         D              s	 |d       yt&        j(                  j.                  r\|j!                         sLt1        | j                  |j                         |      j3                               |dfdffv }|s |d       |S y|k7  r |d       |k(  S |j                         r|j                         rJ | j5                  ||      S )z
        Hook called by Scheduler to determine if the Triton backend
        can fuse node1 and node2.  These nodes might already be
        FusedSchedulerNodes.
        z&Split scan cannot fuse with reductionsFr   )MixOrderReduction)NestedReductionz1numel/rnumel mismatch (reduce) (%s, %s), (%s, %s)c              3     K   | ]8  }t         j                  j                         |j                                 : yw)r  N)r   r  r   
get_ranges)r  n2rnumel1rO  s     rb   r  z*SIMDScheduling.can_fuse.<locals>.<genexpr>  s>        ,,' - s   >Az/invalid loop order and tiling for native matmulz5numel/rnumel mismatch (non-reduce) (%s, %s), (%s, %s)z:numel/rnumel mismatch prologue mismatch (%s, %s), (%s, %s)Tr   ztiling mismatch (%s, %s, %s)r@   c              3  j   K   | ]*  }t         j                  f|j                                , y wr\   )r   r  rr  )r  r   numel2rnumel2s     rb   r  z*SIMDScheduling.can_fuse.<locals>.<genexpr>`  s1       ,,fg->Os   03z"nodes numel/rnumel incompatibilityzinvalid tiling for reductionznodes numel incompatibility)r  r   ForeachKernelSchedulerNodecan_fuser  r1   is_split_scanrw   torch._inductor.schedulerrn  ro  _is_dependent_reduction_pairrt  select_tiling	get_nodesr  is_templateused_buffer_namesget_buffer_namesr   r_    tiling_prevents_pointwise_fusionr   tiling_prevents_reduction_fusionr   r   can_fuse_horizontal)rs   node1node2r/  numel1whyreduction_can_fusern  ro  r   	pro_numel
pro_rnumelr   tiling1tiling2tiling3condis_reduction_tiling_validrv  rt  rw  rO  s                     @@@@rb   ry  zSIMDScheduling.can_fuse  s    eYAABj977G
 77@@NN${{FG${{FGu% )<)<)>!!#<=  "5+>+>+@!!#<=E$6$6$8!'6!1!Hg6H%G%6%?%?u%M"%
 F%4%Q%Q5& &=%..ue< # &G "&&(E,B,B,D --/#(%5E ++EOO,=vwO  $oo/	  IJ %%!!#E,>,>,@f$G);((*O ! !& 1 )++-!  $557%:P:P:RR$59ZZ22Iz &) 3:8M \ & ) ' * $)#)& U^  ==? 
 (():FGLG(():FGLG((!EOO$55vwG }}==w<!#7|a'&'<W<&'1\A%"g-D6	 !!!#(:(:(<a<GqL00')) "__.  <= MMBB!--/05**5??+<fELLN1  !,1- 5:;4412V##!!#E,>,>,@@@''u55rd   c           
     z   g t        t        j                            t               t               d fd}fd}fd}fd}t        j                  fd       }fd}	|D ]  }
|
v rj                  |
        ||
      r? |	|
      r |       5  	 d d d        r ||
      sxs t              nd  ||
       ` ||
      r" |       5  j                  |
       d d d        t        d d d	|
j                  d
           S # 1 sw Y   |xY w# 1 sw Y   xY w)Nc                b    | j                   \  }\  }}|k(  xr |k(  xs |z  k(  xr |dk(  S Nr@   r  r   r/  
node_numelnode_rnumelro   rQ  s       rb   fits_in_main_bodyz@SIMDScheduling.generate_node_schedule.<locals>.fits_in_main_body  sH    +,77(A(
K%'AK6,A efn,A1Ard   c                N    | j                   \  }\  }}|k(  xr |dk(  xr dk7  S r  r  r  s       rb   fits_outside_reductionzESIMDScheduling.generate_node_schedule.<locals>.fits_outside_reduction  s4    +,77(A(
K&K;!+;K!Krd   c                \    | j                   j                  D ]  }|j                  v s y yr  )read_writesreadsrl   )r   readcurrent_loop_buffer_usages     rb   expect_improved_memory_usagezKSIMDScheduling.generate_node_schedule.<locals>.expect_improved_memory_usage  s1    ++  99 99  rd   c                   j                  |        j                  |        j                  | j                  j                  D cg c]  }|j
                   c}       | j                         rt        | t        j                        rrt        | j                  t        j                        rNt        | j                  j                  t        j                        s j                  | j                                y j                  | j                  j                   D cg c]  }|j
                   c}       y c c}w c c}w r\   )r   r   updater  r  rl   rw   r  r   rv  r   r   rw  dataScanget_namewrites)r   rX   r  donerM  not_ready_yet_nodess     rb   schedule_node_in_loopzDSIMDScheduling.generate_node_schedule.<locals>.schedule_node_in_loop  s    HHQK  #%,,amm>Q>Q-Raff-RS
  q)"9"9:qvvr'8'89"166;;8#''

5)00!--BVBV1WQ!&&1WX .S 2Xs   D; E c               3  L  K   rd   t         u rj                          nj                  t               r1j	                  t               j	                  dz   t                d d  j                  t                j                           j                          y w)Nr  r@   )rI   popr   rH   insertclear)r  maybe_split_indexrM  r  s   rb   end_current_reduction_loopzISIMDScheduling.generate_node_schedule.<locals>.end_current_reduction_loop  s      r!2o!E!!#$$%56 $$%68HI$$%6%:OL$(!  1%%'%++-s   B!B$c                    dk(  ry| j                   z  sy|rt        |d   t        t        f      rJ t	              S )Nr@   Fr  )	ancestorsr  rI   rH   r   )r   rM  r  rQ  s     rb   #requires_closing_previous_reductionzRSIMDScheduling.generate_node_schedule.<locals>.requires_closing_previous_reduction  sN    {&7 b!O5E#F*   +,,rd   zunexpected group: (r   z) != r@   )
r   r   r/   r  r  r   r  r   r  r  )rs   r   ro   rQ  r  r  r  r  r  r  r   r  r  r  rM  r  s     ``       @@@@@rb   generate_node_schedulez%SIMDScheduling.generate_node_schedule  sW   #%)5568 0:|5?\!(,		L		Y" 
	"	"	. 
#	.	-  	Dt|HHTN &6t]K35  -5QRV5W(9(OS=O% )-%%d+'-/1 /!((./ / *)%6(%

1O -	4 ' / /s   (D%&D1%D.	1D:	c                    |j                   |j                  }}|j                  |j                         z  s|j                  |j                         z  rJ | j	                  ||       y r\   )r  r  r  get_operation_names_codegen_mix_order_reduction)rs   r   r  r  s       rb   codegen_mix_order_reductionz*SIMDScheduling.codegen_mix_order_reduction  s[    zz4::u OOe&?&?&AAOOe7799	
 
 	))%7rd   c                    |j                         }g }g }|D ]5  }|j                         r|j                  |       %|j                  |       7 ||fS r\   )r~  rw   r   )rs   r   r   
reductions	epiloguess        rb   #_split_mix_order_reduction_epiloguez2SIMDScheduling._split_mix_order_reduction_epilogue  s\     
	 	'D  "!!$'  &		'
 9$$rd   c           	        |j                   |j                  }}|j                  }| j                  |||dg|dddd      d   }|j                  sJ |j
                  sJ ||_        | j                  ||       |j                  j                  t        |j                        |j                  d   z  |j                  d   |j                  z   dz
  |j                  z  z  d	t        j                  
      \  }}	}
|
dk(  s
J d|
       |5  |j                          ddd       t!        j"                         }t%        j&                  |      5  |5  |r%|j)                  t+        j,                  d             |j/                         }ddd       ddd       |r)j1                  t3        t4        j6                        d      }||	fS # 1 sw Y   xY w# 1 sw Y   MxY w# 1 sw Y   QxY w)z
        for_benchmark:
            True if the generated code is for benchmarking. We need make
            sure benchmark harness code is generated.
        )rX   rY   NT)rR  rP  rp  r  r   rY   rX   r@   F)r  zws_off=benchmark_kerneltriton_)ro   r  rM  create_kernel_choicesro  rp  r{  !codegen_node_schedule_with_kernelr  	workspacer  r~  rh  r]   r=  r  r  	ExitStackr?   set_kernel_handlerenter_contextr   patchr  replacer   r7   KERNEL_NAME)rs   kernel_features
split_sizefor_benchmarkro   rQ  rM  rq   r/  ws_namews_offstacksrc_codes                rb   -_generate_kernel_code_for_mix_order_reductionz<SIMDScheduling._generate_kernel_code_for_mix_order_reduction  s    (--/N/Nv'55++()+!%'+15		
 	 ****))))'..}fE $[[22//0mmE"#c"V%7%77!;@R@RRT ++ 3 
7F {(wviL({ 	"!	" $$&!!&) 	/5 	/##FLL$$GH,,.H	/ 	/
 
  ''K,C,C(DiPHw((	" 	"	/ 	/ 	/ 	/s0   
GG8GGGG	GG(Nc                    t         r\   r  )rs   modn_spills_threshold
node_namess       rb   benchmark_codegened_modulez)SIMDScheduling.benchmark_codegened_module+	  s
     "!rd   c                8   %& t         j                  j                        \  &}&fd} |       }t        xj                  dz  c_         j                  |      \  }}g }|D ]C  }	|	j                          |	j                         }
|
j                          |j                  |
       E  j                  j                         |z   &|      }t        |&|      %t        j                  j                  j                   sqt        j"                  j$                  Wt        j"                  j&                  s t        j(                  st        j*                  r% fd}t-        j.                  ||d      } j1                  %|d      \  }}}t3        |d   j4                  j6                        }i }|rZ|D ]  }	|	j9                         d   j4                  j;                         }|	j9                         d   j<                  d   j4                  j9                         d   j4                  j;                         }|||<    j                   sJ  j                   j>                  jA                  |	j9                         d   j<                  d   j4                  j;                                tB        jD                  jF                  jA                  |        |jH                  D ]-  }|jK                  |jL                  |jL                        |_&        /  jO                  |||      }||_(        tS        |      |_)        tC        jT                  |      5  %jW                         D ]@  }|j9                         d   j4                  j;                         |vs1|jY                          B 	 d d d        tB        jD                  jZ                  j]                  d        j_                  |d        |ja                  |jP                  d	       tB        jD                  xjF                  |jF                  z  c_#        tB        jD                  xjb                  |jb                  z  c_1        te        |      te        |jH                        k(  sJ tB        jD                  jZ                  jg                  &|z   dz
  |z        }ti        |jH                        D ]  \  }}|jL                  }d
| d| d}| d| }d
| d| }ddd}|jK                  |jj                  |jj                        }| d| d| d| d| d| d| d}tB        jD                  jm                  |      } | l| jo                         jp                  D !cg c]+  }!tB        jD                  jZ                  jg                  |!      - }"}!ddjs                  |"       d}#|d|# dz  }tB        jD                  ju                  |      x}$t        jv                  k7  r	|d|$ dz  }tB        jD                  jZ                  jy                  |       tB        jD                  jZ                  jz                  jA                  |        |j}                          |r j                  |        j                          y # 1 sw Y   xY wc c}!w )Nc                 r   t         j                  j                  t         j                  j                  S t        j                  j                               } | j                  }|dz  }t        j                  j                  j                        }t        t        ||z        d      }t        |d      }|S )N         )r   r_   mix_order_reduction_split_sizer*   create
get_devicemulti_processor_countr?   r   r   r   r  r,   min)device_propnum_smestimated_num_splits
numel_hintr  r  ro   s        rb   _pick_split_sizezESIMDScheduling._codegen_mix_order_reduction.<locals>._pick_split_size3	  s    }};;G}}CCC +11%2B2B2DEK 66F#)A: 
 ));;EBJ_Z;O-OPRTUJZ-Jrd   r@   c                    j                  | d      \  }}}t        j                  |      }j                  |      \  }}|S )NTr  r  )r  r   rU  r  )candidate_split_sizer/  r  r  msr  rs   s        rb   _benchz;SIMDScheduling._codegen_mix_order_reduction.<locals>._benche	  sS    !%!S!S#3"& "T "1h
 "&&x077<A	rd   r  Fr  r   z!# Call mix order reduction kernel)r^  (z) * (r  z * z + 1) * aminamax)r  r  z = r3  z : z].view(r   z).z(dim=0)r4  z.view(z.to()Ar   rn  get_numel_rnumelr   r  r  cancel_reduction_splitextract_pw_from_reductionswap_pw_red_dimensionr   r  r~  rL   r]   r^   r   deterministicr_   r  'mix_order_reduction_autotune_split_sizemax_autotunecoordinate_descent_tuningr)   autotune_single_fieldr  r   r   _split_sizeget_outputsr  usersremoved_opsr   r?   r   removed_buffersr~  r   rG  define_kernelrY  r   r  scheduler_nodesmark_runrW  make_commentcodegen_commentr_  inplaced_to_remover  codegen_python_sizevarr  rH  r  r  r  r5  r|  r=  	writeline	allocatedr\  _codegen_nodesfree_buffers_in_scheduler)'rs   r  r  rQ  r  r  node2_reductionsnode2_epilogueconverted_nodessubnode	convertedrM  r  rq   r  r  is_split_reductionrenamebufnameusernamepartial_accumrY  r   nsplitr  rG  
stride_strr  endreduction_type2opopnamefinal_reducebufferr   r>  final_shape_strbuffer_dtyper  ro   s'   ``                                   @@rb   r  z+SIMDScheduling._codegen_mix_order_reduction0	  s    !33DDUKv	$ &'
 	++q0+ ,0+S+S,
(. ' 	.G**,99;I++-""9-		.
 33OO/
 -]E6J &&44<<DEE&&33 '<<J %)$V$V! %W %
! ""21"5":":"F"FG+ 5!--/277@@B'')!,U1T++-+ T((*	  #+w~~%~**..'')!,2215::CCE ''++G45 "(!@!@ ,2JJ!--}/H/H-)
 ((=&I($X.!!&) 	$'779 $ ##%a(--668FMMO$	$ 	
))*MN]D16--UC	6#9#99	""f&?&??" ?#s6+J+J'KKKK%%<<Z!#
2
 #,F,K,K"L !	<C'33KVHE&3Je3zl+EcU(:,/C! '**,,m.J.JF *]#gYawc#gfXUWX^W__abhaiipqL WW''4F! $..055 GG((??B  %&dii&<%=Q"?&(9 ;; !" 1 1+ >>5;;N$|nA 66GG  **<8 GG  **..{;C!	<F 	$$&/&&(y	$ 	$Ns   	AZ
Z
,0Z
Zc                >  # |j                   |j                  }}|j                  \  }\  }}|j                  \  }\  }}|j                  }	|j                  t
        j                  j                  j                  u }
t        |	      }t
        j                  j                  |||j                        }|J t        |      }t
        j                  j                  j                  }|D cg c]  \  }}||j                         v r||u r| }}}t!        |      }|j                         D cg c]  }||vr|
 }}|j"                  #| j%                  |j                         ||      }t'        #fd|D              sJ #j(                  }| j%                  |||      }t*        j,                  j.                  j0                  j2                  r|j5                         nd}t7        ||||      }| j9                  ||||      \  }}d|vrd|vsJ d       t:        xj<                  dz  c_        |d|d}t?        d	| jA                  ||g|      d
         }|
r||_!        n||_"        | jG                  ||       |5  |jI                          tJ        jM                  ||	|
      }|jO                  |      }|jQ                  |      }|jS                         } | jU                  ||jV                  |jX                        }!|j[                  |      }"| j]                  ||| |!t_        ||             | ja                  ||#||jV                  |jX                  |!|"|||        |jI                          ddd       | jc                  |||g |||       yc c}}w c c}w # 1 sw Y   /xY w)a  
        Generate a single kernel with an outer reduction, a group
        reduction, and optional reduced/parent-full epilogues for
        dependent cross-axis reductions that share a large input.

        The local group may split either parent tree. For example,
        group-in-R reshapes [XBLOCK, RBLOCK] into [XBLOCK, RBLOCK/G, G],
        while group-in-X reshapes it into [XBLOCK/G, G, RBLOCK].
        Nc              3  &   K   | ]  }|u  
 y wr\   r   )r  sngrouped_reductions     rb   r  z:SIMDScheduling.codegen_nested_reduction.<locals>.<genexpr>	
  s     Fr2**Fs   rW   rV   z2nested reduction does not support tiled reductionsr@   F)rR  r  rP  rQ   r   rJ  )2r  r  r  
group_sizegrouped_axisr   ro  GroupedAxisRr    _classify_nested_pointwise_nodesdomain_contextrN  PointwiseDomainLOCAL_REDUCTION_INPUTr~  r   r  r  r-  _bodyr]   r^   r   r_   coalesce_tiling_analysisget_coalesce_analysisrL   get_tiling_and_scoresr   codegen_nested_reductionr   r  r}  r|  r  r  r  r  r   r+  r-  !_local_reduction_iteration_valuesr  r  r1  _codegen_remapped_pointwiserE   _codegen_nested_grouped_schedule!_finalize_nested_reduction_kernel)$rs   r   
outer_nodegrouped_noder/  outer_numelouter_rnumelgrouped_numelgrouped_rnumelr  r  local_reduction_size_hintnested_pointwise_domainspointwise_domain_by_nodelocal_reduction_domainr  domainouter_local_reduction_pointwise#outer_local_reduction_pointwise_setouter_codegen_nodesgrouped_schedulegrouped_reduction_bodycombined_schedulecoalesce_analysisr  rO  tiling_scorekernel_kwargsrq   r  r&  reduced_output_familyparent_full_familylocal_reduction_sourceparent_full_sourcer  s$                                      @rb   r)  z'SIMDScheduling.codegen_nested_reduction	  sP    $(::tzzL
)3)9)9&&K-9-?-?**M>#!:!:!F!F!H!HH 	 %((<$=!%%FF## 	! (333 )* 	! %%55KK 	
 7+
FZ))++:P0P +
' +

 /9+/
+
 !**,
<< 
 

 6:5K5K484O4O""$5

 F5EFFFF!2!8!8 77
 %%,,EE ,,. 	
 -	
  $99	 
 & S%6 	
@	
6 	((A-('.3))

 && 	
   9F 9F
 	../@&I 5	" !.E.Q.Q$$/F 556LM ! 112FG " ..0  66*(66(88 # 392U2U$3 ,,/"&7G -  11 !$22$44&"(%" !k5	"n 	.."0 "			
G+

H5	" 5	"s   ("N*N4C1NNc                T   | j                  |xs |      }t        j                  |      5  t        j                  di |5  |j                         }d d d        d d d        | j                  ||      |_        t        |      |_        t        j                  |      5  |j                         D ]  }|j                           	 d d d        |D cg c]  }t        |t              s| }}| j                  ||       y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   TxY wc c}w r   )_collect_config_patchesr?   r  r   r  r  r  rY  r   r~  r  r  r/   _launch_kernel_and_cleanup)	rs   rq   r>  r   config_patch_scheduleconfig_patchesr  r  base_scheduler_nodess	            rb   r-  z0SIMDScheduling._finalize_nested_reduction_kernel
  s    55!6%6
 !!&) 	/6<<+I.+I 	/,,.H	/ 	/!//

 %X.!!&) 	nn& 	
 / 
*TCT2UD 
  
 	''0DE!	/ 	/ 	/ 	/	 	 
s;   DDD&DD%(D%D
	DDD"c           	        |j                   }t        |j                  D cg c]  }|j                  |    c}|      }t	        ||      }|D ]"  }|t
        t        fv rt        |t        j                        sJ ||u r| j                  ||||||
       J|	j                  |      }|J |t        j                  j                  j                  u r| j                  ||g|
|       |t        j                  j                  j                   u r|}nH|t        j                  j                  j"                  u r| j%                  |||      }nt'        d|       | j                  ||g|||       % yc c}w )a/  Interpret the local reduction schedule with nested emitters.

        ``local_reduction_source`` is the pre-reduction domain, including the
        local reduction lane. ``parent_full_source`` is the outer tile domain
        used by pointwise consumers after reduced values are broadcast back.
        Nz$unexpected nested pointwise domain: rJ  )r%  r  r  rn   rE  rH   rI   r  r   rv  _codegen_grouped_reductionr   ro  r#  REDUCEDr+  r$  PARENT_FULL(_select_full_resolution_pointwise_sourcer  )rs   rq   r<  r  r  r  r  rD  rE  r6  rB  rC  r=  ri  reduced_sourceparent_full_load_transformr  r8  sources                      rb   r,  z/SIMDScheduling._codegen_nested_grouped_schedule
  s   , "3!8!8( 099 '11!4 
 &>ff%M"" /	B&88
 b)"9"9:::&&//!#) -11"5F%%%22BBJJJ00D)"	 ,,<<RRS 0944DDPPPFF&* %'KF8%TUU,,"9 - S/	s   E3c                >   |j                         }| j                  |j                  |      r|S | j                  |j                  |      r|S t        j	                  |j                  |      r|S t        j	                  |j                  |      r|S t        d|       )Nz5unsupported full-resolution nested pointwise ranges: )rr  _ranges_exactly_match_groupsr  r   r  r  )rs   r  rE  rD  	sn_rangess        rb   rP  z7SIMDScheduling._select_full_resolution_pointwise_source
  s     MMO	,,-?-F-F	R%%,,-C-J-JIV))##$6$=$=yI%%##$:$A$A9M))CI;O
 	
rd   c                    |\  }}| xr5 t        |      t        |       k(  xr t        d t        ||       D              S )Nc              3  v   K   | ]1  \  }}t         j                  j                  j                  ||       3 y wr\   )r?   r   r   r   )r  r)  r  s      rb   r  z>SIMDScheduling._ranges_exactly_match_groups.<locals>.<genexpr>  s3      %J   88UKs   79)r  r  r  )r  r  iter_rangesreduction_rangess       rb   rU  z+SIMDScheduling._ranges_exactly_match_groups  sQ    
 )/%%   K CK/ ),[&)A 	
rd   c                .   |j                   }t        ||      }t        t        j                         ||||      }	t        j
                  |	      5  |j                  |      5   |||d       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)N)rq   r  rR  rK  T)allow_same_symbol_in_index)r%  rE  rI  r?   get_ops_handlerset_ops_handlerset_current_node)
rs   rq   r  r  r  r  rB  r=  rK  handlers
             rb   rM  z)SIMDScheduling._codegen_grouped_reduction  s     "3!8!81&&A-()
 w' 	)@)@AR)S 	"+/	 	 	 	 	 	s$   B"A?.B?B	BBc                    |j                   }|j                  t        |      t              k(  sJ 	 	 	 	 dfd}t        j	                  |||      S )Nc                 b    t        |       D cg c]  \  }}t        ||       c}}S c c}}w r\   )r  r3   )r  rA  r  r   s      rb   split_values_by_rangeszRSIMDScheduling._map_iteration_values_to_node_sizes.<locals>.split_values_by_ranges5  s7    
 &)%<!E6  v.  s   +)r  r  r   r  )r  r   r  r   r  )r  rS  r   r  rc  r   s        @rb   #_map_iteration_values_to_node_sizesz2SIMDScheduling._map_iteration_values_to_node_sizes+  s`     6{c&k)))	-	#	 99"
 	
rd   c                    g |j                   |j                  }|D cg c]  }|j                  |    }}t        |g ||      S c c}w r\   )r  reduce_varsrn   r  )rs   re  r  r  	body_varsri  r  s          rb   r*  z0SIMDScheduling._local_reduction_iteration_valuesC  sV     9dnn8t'7'78	.78$//!$88v'I'I'IJJ 9s   ArJ  c          	        |j                  |      5  |D ]  }| j                  ||j                               \  }}|rJ t        t	        j
                         |||      }	t	        j                  |	      5  |j                  |      5  |j                  |       ddd       ddd        	 ddd       y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   yxY w)zEmit pointwise nodes under an explicit nested iteration family.

        `source` describes the coordinate space whose values are remapped into
        each pointwise node's own loop ranges.
        )rq   rR  rK  N)	r  rd  rr  ra  r?   r]  r^  r_  r%  )
rs   rq   pointwise_nodesrR  rS  rK  r  r  reduction_varsr`  s
             rb   r+  z*SIMDScheduling._codegen_remapped_pointwiseM  s     __V$ 	(% (,0,T,TMMO-)	> *))0%%'!!#1	 &&w/ (1H1H1L (HHY'( ( ((	( 	(( ( ( (	( 	(s<   A#C5CB5	C!
C5B>:CC
CCc                b   | j                   sJ |D cg c]+  }|j                         | j                   j                  vs*|- }}|sy t        |d       j                  \  }\  }}| j                  |||      }t        j                  d|       | j                  t        ||||            S c c}w )Nc                4    t        | j                               S r\   r   rw   rX   s    rb   r   z/SIMDScheduling._codegen_nodes.<locals>.<lambda>v  s    c!..:J6K rd   r   zSchedule:
 %s)
r   r  r  r  r  r  schedule_logdebugcodegen_node_schedulerL   )rs   r   r?  r   r/  ro   rQ  rM  s           rb   r  zSIMDScheduling._codegen_nodesk  s    
 ~~~"
dmmoT^^=W=W&WD
 
  ,KLRR?E633E5&I+];))}eV=NO
 	

s
   +B,B,c                   | j                   sJ |j                         D cg c]*  }|j                         | j                   j                  vr|, }}t	        |      dk(  ryt
        j                  j                  j                  j                  rdt	        |      t	        j                               k7  r.| j                   sJ t        j                  | j                   |      }|j                         }nd}| j                  ||      S c c}w )zI
        Given a set of pre-fused nodes, generate a SIMD kernel.
        r   N)r   r~  r  r  r  r]   r^   r   r_   r&  FusedSchedulerNoder'  r  )rs   r   r   r?  s       rb   codegen_nodezSIMDScheduling.codegen_node  s     ~~~ (
}}dnn&@&@@ 
 

 u:???!!((AA5zS!122~~%~ 33DNNEJ $ : : < $""5*;<<!
s   /Dc                   t        j                  t         j                        j                  }t	        |       sy|D cg c]0  }|j                         r|j                         j                         2 }}|D ]}  }|j                         rt        |t        j                        s/|j                         }||D cg c]0  }|j                         r|j                         j                         2 c}z  } t        d |D              syt        j                  j                  j!                  | |       |D ],  }t        j                  j                  j!                  ||       . yc c}w c c}w )NFc              3  2   K   | ]  }t        |        y wr\   )r4   )r  r  s     rb   r  z8SIMDScheduling.can_use_32bit_indexing.<locals>.<genexpr>  s     FD)$/Fr  T)r]   iinfoint32r  r4   has_tensor_outputr  storage_sizer  r   MutationOutputget_mutation_buffersr  r?   r   r   	check_leq)ro   buffersint_maxr  	buf_sizesmutated_bufsr  s          rb   can_use_32bit_indexingz%SIMDScheduling.can_use_32bit_indexing  s=   
 ++ekk*..%e, 
$$& NN))+
	 
  	C((*z#r?P?P/Q"779+,,. NN$113 		 FIFF 	
""5'2 	6DGG&&tW5	6/
s   5E&;5E+Fc                   | j                  ||       |sIt        j                  |      5  t        j                  |      D ]  }|j                           	 ddd       t        j                  xj                  |j                  z  c_        t        j                  xj                  |j                  z  c_        y# 1 sw Y   dxY w)ze
        Process a kernel by generating code for its node schedule and updating graph state.
        N)	r  r?   r  rK   rr  r  r   r  r   )rs   rq   rM  only_gen_src_coder   s        rb   process_kernelzSIMDScheduling.process_kernel  s     	..}fE %%f- $.99-H $DMMO$$ 	
6#9#99	""f&?&??"	$ $s   +B99Cc                    i }|D ]N  }t        |t              s|j                  !|j                  j                         }|s>|j	                  |       P |S )zCollect and merge config_patches from all operations in the node schedule.

        This enables scoped config (e.g., coordinate_descent_tuning) for kernels
        that contain decomposition operations.
        )r  r/   r   get_config_patchesr  )rs   rM  merged_patchesr   patchess        rb   rG  z&SIMDScheduling._collect_config_patches  sW     *,! 	3D$ 12tyy7L))668"))'2		3
 rd   c                   |j                   }| j                  ||j                  |j                  |j                        \  }}| j                  ||g||d      }|D ]  }| j                  ||        t        j                  |       | j                  |      }|D ]  }t        j                  |      5  t        j                  di |5  |j                         }ddd       ddd       | j                  ||      }	t         j#                  d|	       |	|_        t'        |      |_         ~t)        |      dkD  rt        |      }
n|\  }
t        j                  |
      5  |j+                         D ]  }|j-                           	 ddd       |D cg c]  }t/        |t0              s| }}| j3                  |
|d       t        j4                  j6                  j8                  rt        j:                  r|d   j<                  j?                         }|j+                         D ]  }|jA                         }||vr|jB                  J |jB                  jE                         }|CtF        d   d	xx   dz  cc<   t        j4                  j6                  jI                  d
|jJ                  d| d        | jM                          y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   ]xY wc c}w )z<
        Generate code for nodes in kernel_features
        rR  rP  Nz+Generating kernel code with kernel_name: %sr@   Ffree_buffersr   inductorintermediate_hookszrun_intermediate_hooks(r   r  r   )'rM  r(  ro   r  r?  r  r  rF   merge_workspaces_inplacerG  r?   r  r   r  r  r  r  rp  rY  r   r  r  r  r  r/   rH  r   rW  supports_intermediate_hooksgenerate_intermediate_hooksr  live_output_buffersr  r   get_origin_noder   r  rl   r  )rs   r  rM  rO  r@  kernelsrq   rJ  r  rY  final_kernelr   rK  	live_outsrl   origin_nodes                   rb   rq  z$SIMDScheduling.codegen_node_schedule  s    (55#99!!++--	 
 ,,H(<H

  	JF22=&I	J,,W5 55mD 	3F%%f- 3v||/Mn/M 3!0023 3,,X}fMKIIC[Q!,F(2F	3  w<!&w/L%O\!!,/ 	 '779   	  +9
j?P.QD9
 9
 	''  	( 	
 GG  <<22  
;;=I'779 
}}y(yy,,,"ii779*Z()=>!C>GG((221+2B2B1ERvQO
 	&&(_3 3 3 3	  	 
9
s<   2KKK&K!K.(K.KKK	!K+Tr  c               R   | j                  ||j                         t        j                  j                  r\t
        j                  j                  j                          t
        j                  j                  j                  |j                  |       |j                  |j                         t        j                  j                  r(t
        j                  j                  j                          t        j                  r|j                          t        j                  r|j                  |j                         t
        j                  xj                  |j                  z  c_        t
        j                  xj                   |j                   z  c_        |r| j#                          y y r\   )r  rY  r   cppenable_kernel_profiler?   r   rW   write_kernel_context_guard_beginwrite_kernel_context_guardr_  write_kernel_context_guard_endnan_assertsrR  r  r  r   r  )rs   rq   rK  r  s       rb   rH  z)SIMDScheduling._launch_kernel_and_cleanup'  s    	163E3EF::++GG  AACGG  ;;""$ 	6--.::++GG  ??A$$&!!""6#5#56	6#9#99	""f&?&??"**, rd   c                (     | j                   |i |gS r\   )rh  )rs   r  kernel_argsrA  s       rb   r  z$SIMDScheduling.create_kernel_choicesD  s)     D
 	
rd   c           	     f   |5  t        j                         }i }|D ]  }|t        u r |j                  |j	                                +|t
        u r|j                          D|j                          |j                  |j                               }|j                  t        j                  |j                  j                  |      j                                       |j!                  |j#                                |D ]  }|t        u r |j                  |j	                                +|t
        u r|j                          Dt%        |j                         t'        |j                         |j                  |j                               }|j)                  |        	 d d d        y # 1 sw Y   y xY wr\   )r  r  rH   r  r  rI   closedecide_inplace_updater  rr  r  rN  fromkeysr%  indexing_from_argsr   r  keysr(   r'   r  )rs   rM  rq   r  all_indexingr   r   s          rb   r  z0SIMDScheduling.codegen_node_schedule_with_kernelN  s_     	-((*EL & ++''(@(@(BC_,KKM..0!'!<!<T__=N!OJ '' JJ99*ELLN $$\%6%6%89 & 
-++''(@(@(BC_,KKM 6djjA4TZZ@!'!<!<T__=N!OJLL,
-- 	-  	-  	-s   FF''F0r  c          	     J   i }|j                         }g }	|D ]  }
|
j                         }|	j                  |
       ||z  s*t        |      dk(  sJ |	|t	        t        |            <   |j                  j                  t	        t        |                   g }	 t        |	      dk(  sJ |j                  D ](  }|j                  j                  j                  |d       * |j                  | ||||t        |      }t        j                  rH|j                         dz  }|j!                          d| d|j#                  |      j%                          }g |||}|r|S t'        |j)                         D cg c]  }t+        |       c}      }t-        j.                  |      5  |j1                          |D ]   }t+        |      |vs|j1                          " |D ]  }|j1                           	 ddd       | j3                  |||      |_        |S c c}w # 1 sw Y   (xY w)zK
        Helper method to codegen a single template kernel variant
        r@   r   Ng    eAr  )r  r  r   r  r   iterprologue_fused_inputsr   r  r  r  r  r   r   r  r  imports_for_benchmark_kernelcodegen_kernel_benchmarkgetvaluer   r  idr?   r  r  r  rY  )rs   rq   r  r  r  r  r  r  template_readsprologue_groupprologuenamesbuf_namer  num_gbrM  r   unfused_setr   s                      rb   _codegen_single_templatez'SIMDScheduling._codegen_single_templateq  s2    &("&88:& 	$H--/E!!(+~%5zQ&@N*4U+<=,,00d5k1BC!#	$ >"a''' 44 	:HKK%%))(D9	: //&(
 ""557#=F6689*B226:CCEFH  J.I-I.IO !1M1M1O!PA"Q%!PQ!!&) 	 ""$& $d8;.MMO$ '   	  "//-P "Q	  	 s   6H$#H)HH"c                   ddl m fdg }t        |j                        |gz   D ]S  }t	        |t        t
        f      r$|j                  t        fd|D                     =|j                   |             U t        |      S )Nr   r$   c                    t        |       sy t        | t        j                        r| j                         } | j	                         x}y t        d |D              S )Nc              3      K   | ]  }|  y wr\   r   r  s     rb   r  zKSIMDScheduling._get_multikernel_shapes.<locals>.get_size.<locals>.<genexpr>  s     )q)s   )r  r   BaseViewunwrap_viewmaybe_get_sizer   )r   r  r%   s     rb   get_sizez8SIMDScheduling._get_multikernel_shapes.<locals>.get_size  sR    c6*#r{{+oo'**,,5)D)))rd   c              3  .   K   | ]  } |        y wr\   r   )r  _argr  s     rb   r  z9SIMDScheduling._get_multikernel_shapes.<locals>.<genexpr>  s      @D$ @s   )r   r%   rL  inputsr  r   r   )rs   r   outr   r%   r  s       @@rb   _get_multikernel_shapesz&SIMDScheduling._get_multikernel_shapes  st     	 	* $v- 	*C#e}-

5 @C @@A

8C=)		*
 Szrd   c                H    | j                  |      }t        d |D              S )Nc              3  @   K   | ]  }t        d  |D                yw)c              3     K   | ];  }t        |t        j                        xr t        |t        j                          = y wr\   r  r   Exprr  r  s     rb   r  zFSIMDScheduling._kernel_has_dynamic_shapes.<locals>.<genexpr>.<genexpr>  s9       1ejj)N*Q2N.NNs   AAN)r-  )r  r   s     rb   r  z<SIMDScheduling._kernel_has_dynamic_shapes.<locals>.<genexpr>  s,      

 	   
s   )r  r-  )rs   r   shapess      rb   _kernel_has_dynamic_shapesz)SIMDScheduling._kernel_has_dynamic_shapes  s.    --d3 

  
 
 	
rd   c                N    | j                  |      }t        fd|D              S )zk
        Returns cache key for hint-based multi-graph; key is tuple of shapes with hint filled in.
        c              3  F   K   | ]  }t        fd |D                yw)c              3     K   | ]<  }t        |t        j                        rt        |t        j                        sn| > y wr\   r  )r  r   hints     rb   r  zASIMDScheduling._make_shape_cache_key.<locals>.<genexpr>.<genexpr>  s@        a,Z5==5Q s   AANrk  )r  r   r  s     rb   r  z7SIMDScheduling._make_shape_cache_key.<locals>.<genexpr>  s/      
    	 
r*  )r  r   )rs   r   r  r  s     ` rb   _make_shape_cache_keyz$SIMDScheduling._make_shape_cache_key  s1     --d3 
  
 
 	
rd   r  hint_overridec          	        |j                   \  }\  }}|dk(  sJ t        |j                  t              r|j                  j                  rt        |j                  j                        dkD  r| j                  |j                        ri }	g }
|j                  j                  j                         D ]  \  }} ||j                  |      \  }}|r;| j                  |||||d      }t        |t              sJ |
j                  |       Z|]| j                  |||||d      }|dn| j                  |j                  |      }||	|<    |rdj                  |
      S t        j                  t        |	j!                                      t#        |	      }g |||}| j%                  ||j&                         |j)                  |j&                         t*        j,                  xj.                  |j.                  z  c_        t*        j,                  xj0                  |j0                  z  c_        | j3                          y|j                  j5                  |j                  |      \  }}|r| j                  |||||d      S | j                  |||||d      }g |||}| j%                  ||j&                         |j)                  |j&                  |j                         t*        j,                  xj.                  |j.                  z  c_        t*        j,                  xj0                  |j0                  z  c_        | j3                          y)z
        Codegen a triton template with multi-kernel dispatch support

        If `only_gen_src_code=True` the src code will be returned instead of being
        codegenned into the wrapper
        r@   )r  Tr  NFz

)r  r  r   r   _make_kernel_rendersr  r  r}   r  r   r   r  r5  rF   r  rL  r   rG   r  rY  r_  r?   r   r  r   r  make_kernel_render)rs   r  r  r  r  r  r/  _numelrQ  r  	src_codes	size_hintr  rq   r  r  shape_cache_keymulti_kernelrM  s                      rb   codegen_templatezSIMDScheduling.codegen_template  sD     ,11FF{{ }))+>?""77M&&;;<q@//0B0BCGI
 ##88>>@%6 "!3!&&m" %#<<%&&*.  =  H &h444$$X. ( !::%&&*/ ; F %, !778J8JIV $ 06GO,K%6N !{{9--00gnn6F1GH.w7LMnMmMnMM  0H0HI$$\%=%=>GG##|'C'CC#GG&&,*I*II&**,*//BB""- C NFF !44!""&* 5   66!""&+ 7  !R. Q- Q. Q$$]F4F4FG""6#5#5}7I7IJ''6+A+AA'**f.G.GG*..0rd   c                    t         j                  j                  j                  t         j                  j                  j                                y r\   )r?   r   rW  r  
device_opssynchronizerv   s    rb   codegen_synczSIMDScheduling.codegen_sync]  s-    	&&qww'9'9'E'E'GHrd   c                v   i }| j                   j                  |j                  |        | j                   |j                  f|j                  |j                  d|}| j                  ||j                  |       t        j                  |      5  |j                         }d d d        ||fS # 1 sw Y   |fS xY w)Nr  )
rh   apply_feature_required_overridesrR  rO  rP  r  rM  r?   r  r  )rs   	node_infor  rA  rq   r  s         rb   _codegen_standalone_kernelz)SIMDScheduling._codegen_standalone_kernel`  s     )+99	
 "!!
''#11
 	
 	FI$;$;=NO!!&) 	/,,.H	/	/s   B,,B8c                   ddl m}m}m} ddlm} t        dd      }| j                  j                  |j                  |        | j                  |j                  f|j                  |j                  d|}t        j                  d	      5  t        j                  |      5  | j!                  |j"                  |       |j%                         }d d d        d d d        |j&                  j)                         D 	
ci c]X  \  }	}
t+        |	      r|j,                  r<|	t/        t1        t        j2                  j4                  j7                  |
                  Z }}	}
i |j9                         |j;                         }|j                  r> |||j                  j=                  |j                        |j>                  |d
      }||fS |j,                  r> |||j                  j=                  |j                        |j>                  |d
      }||fS |j@                  jC                         \  }}}} |||j>                   |||      |d      }||fS # 1 sw Y   xY w# 1 sw Y   xY wc c}
}	w )Nr   )ro  	pointwiser  r@   )select_tile_hintTF)is_combo_kernelr  r  r  )reduction_hinttriton_metainductor_metareturn_configs)r  	tile_hintr  r  )"runtime.triton_heuristicsro  r  r  triton_utilsr  rN  rh  r  rR  rO  rP  r   r  r?   r  r  rM  r  rh  r}   r8   rk  r-   r   r   r   r   inductor_meta_commoninductor_meta_per_kernelget_reduction_hintr  r  ry  )rs   r  ro  r  r  r  rA  rq   r/  rp   ro   
size_hintsr  configs	signatures                  rb   _probe_subkernel_heuristicz)SIMDScheduling._probe_subkernel_heuristicr  s`   	
 	

 	3(, +0)
 	99	
 "!!
''#11
 	
 \\51 	(13G3G3O 	(2293J3JFS%%'A	( 	( "(!4!4!6
&v.&2I2I OC(8(8(J(J5(Q$RSS

 


))+
--/
 &&*%AA&BVBVW"..+#G0 # $$%AA&BVBVW"..+#G   "(!;!;!=Aq)Q"..*:yA+#G M	( 	( 	( 	(
s+   I"$-II">AI/I	I""I,c                (   dd l }d	d}d
dt        t        t        |             fd      }| |   }t	        |       D ci c]0  \  }} ||      D ]   }| d| t        |j                  |         " 2 }	}}}|	j                  |j                  j                         D 
ci c]  \  }}
|j                  d      s||
 c}
}       |j                  |	t        |j                        t        |j                              S c c}}}w c c}
}w )Nr   c                d    | j                   D cg c]  }|j                  d      s| c}S c c}w )Nr   )kwargsendswith)cfgrh  s     rb   
block_keysz@SIMDScheduling._stitch_no_bench_combo_config.<locals>.block_keys  s%    "zzA!QZZ-@AAAAs   --c                    t        d t        j                  | j                  j                  | j                  j
                        D              S )Nc              3  ^   K   | ]%  }t         j                  j                  |d        ' yw)T)count_bytesN)r?   r   get_dep_size_hint)r  ds     rb   r  zYSIMDScheduling._stitch_no_bench_combo_config.<locals>.node_total_bytes.<locals>.<genexpr>  s,       ))!)>s   +-)r  ri  r  r  r  r  )snodes    rb   node_total_byteszFSIMDScheduling._stitch_no_bench_combo_config.<locals>.node_total_bytes  sB     "%%++U->->-E-E  rd   c                     |          S r\   r   )r  chosen_nodesr  s    rb   r   z>SIMDScheduling._stitch_no_bench_combo_config.<locals>.<lambda>  s    *<?; rd   r   r/  r   )	num_warps
num_stages)r  r   r   z	list[str])r  r   r   r   )r_   r  ranger  r  r   r  r  r}   r  Configr  r  )chosen_configsr  r_   r  
winner_idx
winner_cfgr  r  r   stitched_kwargsrA  r  s    `         @rb   _stitch_no_bench_combo_configz,SIMDScheduling._stitch_no_bench_combo_config  s)    		B	 #n%&;

 $J/
 $N3+
 +
3!#+
  e1QCL#cjjo..+
+
 +

 	 #-"3"3"9"9";C||G, U
	
 }}*../:001  
 	
+
s   5D'D
c           
     ~   ddl m} ddlm} t	        | j
                  |      sJ |D 	cg c]  }	|	j                          }
}	i }t        ||
      D ]A  \  }}t        |d       j                  \  }\  }}| j                  |||      }d}|rt        j                  j                  j                   j                  r=t        |t         j"                  t         j$                  f      sJ |j'                         }nd}t)        ||||      }| j+                  ||||j,                        \  }}n t)        |||      }| j/                  |||      }|j1                         xr! t2        j4                  j7                  |d	      }t9        |||||||
      ||<   D |j;                  || ||      }t<        j?                  dtA        |      |D cg c]  }tA        |       c}       g }|D ]J  }tA        |      dk(  rtA        |      dk(  rI||d      }|r|jC                  dd|f       @| jE                  ||      \  }}|jC                  |||f       j|xr | xr | }g }g } g }!|r|D ]  }| jG                  ||         \  }"}#t        jH                  jJ                  r|"xr |#jL                   }$n|"xr tA        |"      dk  }$|$r&|jC                  |       | jC                  |"d          |!jC                  |        tA        |      dk  rg }g } tO        |      }!ntO        |      }tA        |      dk\  r || j
                  |||      }|D ]s  }||   }|jQ                  |jR                  |jT                  | | j
                  |jV                  |      }%| jY                  |j[                  |%      |j\                  |       u |r| r| j_                  | |      |_0        |jc                         }|jC                  |||f       |!D ]/  }| jE                  ||   |      \  }&}'|jC                  |&|'|gf       1 M |S c c}	w c c}w )al  
        Generate kernel code for combo kernel partitions.

        Partitions subkernel_nodes using horizontal_partition(), then generates
        kernel code for each partition. Single-node partitions are generated as
        regular kernels, while multi-node partitions use ComboKernel.

        Returns a list of (src_code, kernel, node_group) tuples.
        r@   rP   )ComboKernelc                4    t        | j                               S r\   rm  rn  s    rb   r   z;SIMDScheduling.generate_combo_kernel_code.<locals>.<lambda>      #ann>N:O rd   r   N)r?  F)rm  )rM  rO  rP  ro   rQ  rR  rS  )r   triton_schedulingcustom_algorithmnode_info_mapz1ComboKernels: %d nodes partitioned into %s groupsr   r   )triton_kernel_clsenable_autotunemixed_sizesper_subkernel_blocks)rR  optimize_maskr  rP  r  )2r_   rQ   triton_combo_kernelr  
issubclassrh  r~  r  r  r  r  r]   r^   r   r&  r  r   rs  rv  r'  rL   r(  r?  r}  rw   r?   choicesrn  rK  horizontal_partitionr  rp  r  r   r  r  versionhipautotune_hintsrL  create_triton_kernelrO  rR  rP  r  create_sub_kernelrM  r  no_bench_stitched_configr  )(rs   subkernel_nodescustom_part_algorithmr  r  r  r  rQ   r  r   fused_node_listsnode_schedule_mappnr   r/  ro   rQ  rM  rP  r?  rR  rO  rS  
partitionspkernel_code_list
node_groupr  r  rq   no_bench_mode
fusion_pnsfusion_configscarve_out_pnsr  probe_kernelfuse_ok	subkernelco_src_code	co_kernels(                                           rb   generate_combo_kernel_codez)SIMDScheduling.generate_combo_kernel_code  s   $ 	)4 $**L9999HIDNN,II13_.>? &	IB!$U0O!P!V!VAv 77ufMM M#??))00II%Y999;R;RS   )+(@(@(B%(,%-!5&DU )-(B(B!..	)% .mUFK++M5&I%%' II==E >  $ %-++!(?%b!=&	P !55!"2+	 6 

 			? '(SV(	

 $ U	LJ:!#:!#-jm<	$$++T4,DE'+'F'F!#4($Hf %++Xvz,JK ) .++.-- 
 )+
,.+- ( 5040O0O-b11- !==,,&-&Ql6Q6Q2QG&-&C#g,!2CG"&--b1*11'!*=)0045 :*%'
)+(,Z(8!%j!1Jz?a'(*.*:*:(7$/-A	F ) $5b$9	$/$D$D%,,%.%7%7.9/.2.>.>*3*A*A1E %E %	 ++"44Y?%33-  % >> .
 7  &446H$++Xvz,JK' LB-1-L-L)"-/@.*K %++[)bT,JKLaU	Ln  [ Jf )s   P5P:c                   |j                         }|j                  }|j                  }t        j                  dkD  xs t        j                  dk(  xr |}|j
                  }| j                  |||||      }|D ]]  \  }}	}
| j                  ||g|	      }| j                  |j                  |       t        j                  d|       |	j                  |       _ | j                          y )Nr@   )r  z"ComboKernels: generated kernel %s.)get_subkernel_nodesuse_custom_partition_algor  r   combo_kernel_allow_mixed_sizesr  r6  r  r  snodesr  rp  r_  r  )rs   combo_kernel_noder$  r%  r  r  r  r+  r  rq   r/  rY  s               rb   codegen_combo_kernelz#SIMDScheduling.codegen_combo_kernel  s    +??A 1 K K+;;;;a? 
11Q6P;P 	  1EE::!!5 ; 
 $4 	,Hfa,,X8I7JFSK  !2!9!9;GII:KH{+		, 	&&(rd       c           
        
 dk(  }d 
fd}|j                         \  }
t        |      dk  rt        
      dk  st        |
z         rg S |j                         \  }
 |||r|n
|j                  |            }|D cg c]?  }t	         j                  |j                  |      |j                  |j                        A }	}|	S c c}w )Nr@   c                d   t        |j                        t        |      k(  sJ d|j                  d|       |j                  |j                  g}t	        d t
        j                  j                  |      D              sJ t
        j                  j                  |      D cg c]:  }|j                  t        j                  j                  vrt        |t              r|< }}t        |j                  D cg c]  }|j                   c}      }dd}t        j!                   ||      g|       dd      g}|D ]  }t        j                  j"                  j%                  |j&                  |j                        }	t        |	      t        |      k(  sJ 	 |	j'                  d      dz   }
|
t        |      k(  rt	        d	 |	|
d
 D              r	  ||d
|
        |||
d
       f}t        j                  j"                  j+                  t-        d t/        ||	      D                    }|j                  |v r|dz  }t        j1                  |d         r|dz  }t        j1                  |d         r|dz  }t        j                  j"                  j+                  |t-        t        j                  |            z
        dk\  s|j3                  t        j!                   ||d
|
        |||
d
       g      ||j                                |S c c}w c c}w # t(        $ r Y w xY w)zX
            Compute tiling candidates by dividing up the iteration ranges.
            zrw.range_vars=z ranges=c              3  H   K   | ]  }t        |t        t        f        y wr\   )r  r    r!   )r  r  s     rb   r  zHSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.<genexpr>  s$       3G 45s    "c                f    t         j                  j                  j                  t	        |             S r\   r  )r  s    rb   collapse_rangeszNSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.collapse_ranges  s"    ww''00v1FGGrd   noner   )rO  rl   scorer@   c              3  &   K   | ]	  }|d k(    ywr  r   r  s     rb   r  zHSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.<genexpr>  s     ;a16;s   Nc              3  2   K   | ]  \  }}|d k7  s|  ywr  r   )r  r  r  s      rb   r  zHSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.<genexpr>  s       "!-vST"s   r   rO  rE  rl   )r  r  r   r   )r  
range_varsr  r  r  ri  r  r  rl   r?   r   r  r  r    r   CandidateTilingcreate_partial_tilingr   stride_hintsr   
ValueErrorr   r:   r  is_good_sizer   )is_pointwiser  rwdep_sourcesr  depswrite_namesrC  tilingsrl  splittiled_groupsrE  r  r  rZ  s                rb   tile_rangesz5SIMDScheduling.candidate_tilings.<locals>.tile_ranges  s    r}}%V4S8H	&6SS4 88RYY/K $??88E    %??88E88177#:#::sI. D  %"))%D3chh%DEKH
  44(01<  G  4''**77		2==Q7|s6{222
#MM!,q0EF+ ;756?;; ! < $F6EN3#F56N3  ((::! "14VW1E" 
 88{*QJE"//Q@QJE"//Q@QJE GG$$66ioofFV.W XX 
 NN'#&#<#<$3F6EN$C$3F56N$C!" !0$ #(!$
Q4l N[ &E: " s$   $?L8L"L"=L""	L/.L/rH  )rO  r   r   list[CandidateTiling])	rr  r  r   "pointwise_or_reduction_read_writesrJ  complete_partial_tilingrO  rE  rl   )r  r   ro   r  rO  rW  pointwise_rangespartial_tilingsrO  full_tilingsrZ  s   `  `      @rb   candidate_tilingsz SIMDScheduling.candidate_tilings  s     '!+\	| .2__->** !Q&$%*$%58H%HII .2__->**% ,2B33LA
 *	
  22MM5/ ll[[	
 	
 	
s   ACc           	         d}d}t        |      t        |      k  sJ t        |      t        |      k  sJ t        g t        |t        |       d |d      t        ||d            S )zK
        Create a tiling dict from pointwise and reduction splits.
        )rV   rW   rX   )rY   rZ   NF)strict)r  r   r  )r  	pw_tilingreduction_tilingpw_prefixesreduction_prefixess        rb   create_tilingzSIMDScheduling.create_tiling  s     &+9~[!1111#$,>(????[#i.!23YuM')9%H
 	
rd   c                >    | j                  |r|ng |s|      S g       S r\   )re  )r  rO  rO  s      rb   rK  z$SIMDScheduling.create_partial_tiling2  s0       "F&F
 	
,.
 	
rd   c                    t        |j                               }d|v }||z  }|t        |      z  g}|r||fn||f} | j                  | S )zb
        Given a tiling for only pointwise or reduction dimensions, adds the missing one.
        rX   )rL  r   r:   re  )	r  rO  ro   r  splitsrO  total_numelmissing_tilingtiling_argss	            rb   rZ  z&SIMDScheduling.complete_partial_tiling=  sf     fmmo&f}o-%f(==> )5V^$>6:R 	 !s  +..rd   c                    	 	 	 	 	 	 dfd	 	 d	fd}dk(  }t        t        t        t        j                  f             }t        j                  |      D ]  }t        |t        j                        s|j                         }|st        |d         dk(  rC |d         }	 |d         }
|	|
fg}|j                  j                         D cg c],  }t        |t              rt        |j                        dkD  r|. }}|D ]  g j                  j!                         }t        j"                  j$                  }t&        j(                  j*                  }d}t-        |      D ]#  \  }\  }}||z  }|}|j/                  |      s# n |j1                  |      s|dz   }|d| }|rdn||d } |||d         }|rt        j"                  j$                  fn |||d         }t        |      st        |      dkD  s|j3                  ||f        t5        j6                  t9        |  D ]&  \  }}|j;                  | j=                  ||             ( 	 t?        |t        d      }|S c c}w )
z
        Creates N-dimensional tiling candidates, attempting to simplify loads/stores
        by tiling the kernel into higher dimensions.

        Returns a list of tilings ranked by dimensionality.
        c                ^   | s|fS t        d      }t        j                  j                  j	                  d      r6t        j                  j                  j                  d      rt        |d      }t        dt        |       |z
        }|dz   }t        | d|       }|ft        | |d       z   S )zU
            Collapse dimensions to the maximum allowed number of tiles.
            r   r@   r   N)rc   r?   r   r   r   r  r  r  r  r:   r   )dimsfallback_numelr`   num_leading_dimsfirst_trailing_dimcollapsed_leading_dimpointwise_numelr  s         rb   collapse_dimsz4SIMDScheduling.get_nd_tilings.<locals>.collapse_dims`  s     &((%a(Iww77''""66J  	1-	"1c$i)&;<!1A!5$1$7J8J2K$L!)+eD9K9L4M.NNNrd   c                   g }| D ]  \  }}t        j                  j                  |      }t        d|j	                  t
              |j	                  t              z   t        |            }t        j                  ||||      }||d   n|g}	|j                  |	        |D 
cg c]F  }
t        j                  j                  j                  |
t        j                  j                         s|
H }}
 ||      S c c}
w )Nr   r   )rA   get_subexpr_involving_symbolr   r  rj  r   r   r  match_mod_div_block_exprr  r?   r   r   r   r   r   r   )rn   ranges_to_tileri  rO  varro   r   num_dimsmatch_resultrn  dimrt  r  s              rb   tile_var_rangesz6SIMDScheduling.get_nd_tilings.<locals>.tile_var_rangest  s     F( $
U+HHTWX KK)EKK,HH'  3KK3x  +7*B|Ad#!$* "ww''??UWW[[Q F  !55s   $AC:r@   r   NT)r   reverse)rn  r  ro  r   r   tuple[sympy.Expr, ...])r   r  ) r   r   r   r   r  rI   rt  r  r   rv  rr  r  r  reads_and_writesr    r  r}   r   r   r?   r   r   r  statically_known_geqr   r   ri  productr  r   re  rN  )r  rM  rs  r  r}  rO  rT  r   node_rangesdefault_pointwise_tilingdefault_reduction_tilingnode_tilingsr  memory_depsall_var_rangespointwise_vars_numelr   pointwise_end_idxr  _varro   reduction_start_idxpointwise_var_rangesreduction_var_rangespointwise_tilingrb  ranked_tilingsrt  s     ``        `              @rb   get_nd_tilingszSIMDScheduling.get_nd_tilingsR  s   	O&	O8B	O#	O(	6#	6@ '!+^CO<=?#**=9 F	SDdI$;$;< //+KCA$71$< (5[^_'U$'4[^_'U$57OPQL  ++<<>c9-#cjj/A2E K 
 # (N "73::#3#3#5!6',ww{{$77++$%!*3N*C &C$(E1((+%44,o   77(/  '8!&;#'56J7J'K$(Dn=P=Q.R % $3(+a./$ 
 $ WW[[N(,k!no ! '(S1A-BQ-F '')9;K(LMQ(NX 7@6G6Gl#7 S2 "2 C--.>@PQRSGF	ST  
 }s   1I;c                x   j                   sdnj                   j                  j                  j                  j                  j                  j                  j
                  }D cg c]  }||   	 c}D cg c]  }||   	 c}t        j                  j                  j                  }t        j                   |t                     |      k(  fd       t        j                   |t                     |      k(  fd       i g }	 	 	 d	 	 	 	 	 	 	 df	d}	|j                   |	d       |	d      f       r$|j                   |	fdd       |	d      f       j                  j                         z  }
|
D ]%  }|j                   |	|fd       |	d      f       ' t!        d	
      d	k(  rBdk(  r=t#        j$                  |
d      D ]$  }|j                   |	|d       |	d      f       & g }|D ]b  \  \  }}\  }}t'        | j)                  ||      t+        |      t+        |      z         }| j)                  ||      }|j                  ||f       d | j)                  gg      }ddt+        j,                  j/                               fd}t1        ||      D ]  \  }}| j3                  |j4                        s|j4                  |k(  rt7        |j4                        dk(  rdndz
  }|t!        d	
      kD  rDt8        j;                  d|t        j<                  j>                  j@                  jB                         |j4                  |fc S |j4                  |k(  s|j4                  |fc S  |dfS c c}w c c}w )zr
        Generates a tiling, and a score of each tile according to each tile's coalesced memory accesses.
        Nc                      d d  S Nr   r   )rM  rs  	pw_rangess   rb   r   z8SIMDScheduling.compute_tiling_strategy.<locals>.<lambda>  s    ykO#4B}oF rd   c                      d d  S r  r   )rM  
red_rangesr  s   rb   r   z8SIMDScheduling.compute_tiling_strategy.<locals>.<lambda>	  s    zl"_$5RG rd   Fc                @  	 |rn}|rn}|s|r|gg fS g g fS t        |       ||f}j                  |      x}r|S |rn}g }g }	d}
d}t        ||      D ]  \  }}|| vr"|
|z  }
j                  j                  |d      }-|r|k(  rj                  }|J |j
                  }t        ||j
                        }|j                  |
|z         |	j                  |j                         |j                  |       |	j                  j                  j                  |d             d}
d}|
|z  }
|j                  |
       |	j                  j                  j                  |d             d}
 |
dk7  s|r0t        |      dk(  r"|j                  |
       |	j                  |       t        t        |            D ]S  }t        j                  j                  j                  ||   d      }t        |d      }t!        |	|   |z  dz        |	|<   U ||	f|<   ||	fS )z]
            Generate a tiling, and a tiling score, given vars to use as splits.
            r@   r   r>  fallbackr  )r@  r   r  coalesced_by_varsuggested_splittiling_factorr   r   rE  r  r  r?   r   r   r   r  r   )vars_to_useuse_split_varrO  r  target_numelr   r  splitting_varsrh  split_scoresprodprev_var_coalesced_scoreri  v_range
var_tilingtile	remainderr  r   all_iter_varsall_red_varsr?  rs  r  r  r  scored_sub_split
tiling_vars                      rb   process_node_varszASIMDScheduling.compute_tiling_strategy.<locals>.process_node_vars  sX    #/YJF.:?L)NB//8O$m\BC&**3//s/
.:]NFLD'($ ".&9 
7K'GOD/@/Q/Q/U/U10,  Q*_!2!B!BJ%111%33D (*2J2J KIMM$"23 ''
(8(89MM$' ''(9(J(J(N(NqRS(TUD/0,d###$5$F$F$J$J1a$PQ;> qy\c&kQ.>d###$<= 3v;' ?GG$$66vay26N1I"%l1o&9A&=">Q?
 &,\$:S!L))rd   T)rO  )r  rO  r   ra   r@   r   )rE  gffffff?gGz?c                    d}| d   j                   j                         D ]"  }t        j                  |      s|z  }|z  }$ dz  }| d   j                  |z    |z  S )Ng      ?r   g?)rO  r   rJ  rN  rE  )rI  score_factor	tile_sizeuncoalesced_penalty"bad_size_additional_tiling_penaltygood_size_tiling_penaltytotal_uncoalesceds       rb   	score_modz9SIMDScheduling.compute_tiling_strategy.<locals>.score_mod  sw    LqT[[//1 K	&33I>#/2T#TL#/2J#JL	K #4d":qTZZ"556EErd   r   r   zmFound optimal tiling with %s tiles but torch._inductor.config.triton.max_tiles set to %s. Consider increasing)r   FF)r  r  r  r   rO  r   r   ztuple[list[int], list[int]])"r  ry  norm_read_writesr   rf  rn   r?   r   r   r   r]   _checkr:   r   r  r  rc   ri  combinationsrJ  re  r  uncoalesced_addrsr   rN  tiling_is_compatiblerO  r  perf_hint_loginfor^   r   r_   r`   ) r  rM  rs  r  r?  r  ri  get_hintscore_splitr  overlapping_iter_varsr  rT  pw_splitpw_score	red_split	red_score	candidater@  default_tilingr  cand
tiling_lenr  r  r  r  r  r  r  r  r  s     ````                  @@@@@@@@@rb   compute_tiling_strategyz&SIMDScheduling.compute_tiling_strategy  s    %44 "2266 	 *::EE(99EE"33>>(561VAY6	)56AfQi6
 77##55]9-.(?2KKF	

 	]:./8O3LLG	
 DF  	
 35"'!&K	*/K	*K	* K	* )	K	* K	*\ 	!t4!u5	
 %#T &59	 ->>CCEE 	 ' 	A%qd>%59	 #q(_-A(556KQO "")+DI)u= RT<G 	68 Xx"89i'!!(I6(mc)n4I ,,XyALNNI|45	6 **O+<>OP .3*#(  1 C C J J LM	F #)i"@ 	1D,((!?OT[[ ;;.0 !-o6JPQR
a 88!&&9"..55??	 {{L00 {{n,{{L00/	12 t##K 76s   8N2N7c                T    t        t              sJ t        fd|D              S )Nc              3     K   | ]R  }t        |t        j                        r6t        j	                  j                         |j                                 T ywrq  )r  r   rv  r   r  r   rr  )r  r   r  rO  s     rb   r  z6SIMDScheduling.tiling_is_compatible.<locals>.<genexpr>  sO      
 $	 7 78	 $$!2O % 
s   AA)r  rN  r  )r  rM  ro   r  rO  s      ``rb   r  z#SIMDScheduling.tiling_is_compatible  s1     &$''' 
 &	
 
 	
rd   c                B    |D ]  }| j                  ||||      s|c S  y r\   )r  )r  rM  ro   r  r  rO  s         rb   get_first_compatible_tilingz*SIMDScheduling.get_first_compatible_tiling  s1     % 	F''uovV	 rd   c                0    | j                  ||||      d   S r  )r(  )r  rM  ro   r  r?  s        rb   r}  zSIMDScheduling.select_tiling  s)     ((5/3D

 	rd   c                   |dk(  }| j                  |g|g      }t        j                  |      D ]  }t        |j                  t
        j                        s(|j                  j                         dk(  sFt        j                  j                  sa|j                         }|d   }	|d   }
| j                  |	|
      }|dfc S  t        j                  j                  j                  j                  r0|r.t        j                  j                  s| j!                  ||||      S |st        j                  j"                  rt%        d      dk  rt&        j(                  t*        j,                  k  rt        j                  |      D ]i  }t        j                  j"                  rt/        | j1                  |||            dkD  s>t&        j3                  t5        j6                  d              |dfS  |dfS t9               }t;        j<                         }t        j                  |      D ]g  }| j1                  |||      D ]O  }|j>                  |v r|j>                  |jA                  |j>                         ||xx   |jB                  z  cc<   Q i |jE                         D cg c]  \  }}|jF                   }}}t%        d      dk\  r?|r=	 	 	 	 	 	 dd	}tI        dt/        |            D ]  } ||d   ||         }||g|z   } n t/        |      dkD  rt&        j3                  d
|       t        j                  j                  r| jK                  |||      |z   }| jM                  ||||      x}r|dfS |dfS c c}}w )z
        Heuristics to decide how to tile kernels.
        Currently, we tile based on stride-1 dimensions.

        Returns:
            `(tile1, tile2, reduction_numel)` s.t. `tile1 * tile2 == numel`

        r@   r[  r   Nr   r  z
                                Reduction over non-contiguous dims.
                                Consider setting config.triton.tile_reductions to True.
                                r   c                   | d   | j                  dd      }}|d   |j                  dd      }}t        j                  j                  j                  } |||z
        dk(  ry  |||z
        dk  r||f||fc\  }}\  }} |||z
        dkD  sJ t        j                  j                  j                  ||      sy |t        ||      || d   d}|S )NrX   rW   r@   r   rY   )rV   rW   rX   rY   )r   r?   r   r   r  r  r   )tiling0r  a0a1b0b1r  
new_tilings           rb   convert_tiling_to_3dzBSIMDScheduling.get_tiling_and_scores.<locals>.convert_tiling_to_3dK  s     !w{{3':B w{{3':B ww''>>R=A%R=1$*,bB8&HRhr2BG}q(((ww''DDRL !"b)"5>	
 "!rd   zpossibly bad tiling: %s)r  r  r  r  r   r  )'re  rI   rt  r  r   r   rw  rx  r   r_   ru  rr  r]   r^   r&  prefer_nd_tilingr  tile_reductionsrc   r  levelloggingWARNINGr  r^  r  textwrapdedentr   collectionsr   rl   r   rE  most_commonrO  r  r  r  )r  rM  ro   r  r?  rO  r  r   r  	range_y_xrange_rrO  
seen_namescandidate_tilescandidate_tilingrE  r  r  r  new_3d_tilings                       rb   r(  z$SIMDScheduling.get_tiling_and_scores  s   " '!+ **E7_4EF $**=9 	(D$))R%6%67II002e;33 #'//"3K +AI)!nG ..y'BF!4<'	(  OO""))BB!MM22..uo7H  V]]%B%B}H
H ""goo5+22=A D"MM99 5 5dE? STWXX%**$OO!$ !4'' "4''&0l
4?4G4G4I#**=9 	LD$'$9$9$$W L #((J6%**6NN#3#8#89 015E5K5KK1L	L ,;+F+F+H7
' % ##7
 7

 #q(\"."9N"-"@ 1c.12  4"1%~a'8! !,&3_~%EN ~"8.I ==))""=%I ! 
 445/>
 
6 
 4<t##I7
s   "M;c                     y r\   r   rv   s    rb   flushzSIMDScheduling.flush  r  rd   c                     yr   r   rv   s    rb   ready_to_flushzSIMDScheduling.ready_to_flush  r   rd   c                   t        d |D              st        |d       j                  \  }\  }}| j                  |||      }| j	                  |||      }| j                  |t        |||            }	| j                  ||	       | j                  |      }
||
d<   t        j                  d
i |
5  t        j                  |	      5  |	j                         }d d d        d d d        nM|d   j                  |      \  }}}t        j                  d|      5  | j                  |||d|      }d d d        j!                  t#        t$        j&                        d	      }|S # 1 sw Y   xY w# 1 sw Y   @xY w# 1 sw Y   LxY w)Nc              3  <   K   | ]  }|j                           y wr\   )r  )r  r   s     rb   r  zASIMDScheduling.generate_kernel_code_from_nodes.<locals>.<genexpr>  s     2q1==?2s   c                4    t        | j                               S r\   rm  rn  s    rb   r   z@SIMDScheduling.generate_kernel_code_from_nodes.<locals>.<lambda>  r  rd   r   )rR  r  r   Tr  r  r   )r-  r  r  r  r}  rh  rL   r  rG  r   r  r?   r  r  get_prologue_template_epiloguer  r  r   r7   r  )rs   r   r  r  r/  ro   rQ  rM  rO  rq   rJ  r  r  templateepilogues                  rb   generate_kernel_code_from_nodesz.SIMDScheduling.generate_kernel_code_from_nodes  s    2E22!$U0O!P!V!VAv 77ufMM''ufEF%%+M5&I & F 22=&I!99-HN1AN-..~.3$$V,3 "002	3 3 3 ,18+R+R,(Hh 02BC 00&*"/ 1  ##C(?(?$@)L)3 3 3 3 s0   2E.E"E.E:"E+	'E..E7:Fc                    t         r\   r  )rs   r  rM  rq   s       rb   r  zSIMDScheduling.define_kernel  r  rd   )r  N)r  zOrderedSet[str] | Noner   ztuple[float, str]r\   r   )r  scheduler.SchedulerNoder  r  rD  r  rE  r  r6  zBdict[BaseSchedulerNode, scheduler.NestedReduction.PointwiseDomain]r   r   )r  r  rE  r  rD  r  r   r  )r  r  r  r  r   r   )r  r  r  r  r   r   )rS  r  r   r  r   r  )r   r  )rR  r  rS  r  rK  r^  r   r   )r   z!Sequence[scheduler.SchedulerNode]r?  CoalesceVarAnalysis | None)r   z6scheduler.FusedSchedulerNode | scheduler.SchedulerNode)ro   r   r~  zCIterable[ir.Buffer | ir.TensorBox | ir.TorchBindObject | ir.IRNode]r   r   )F)rq   r   rM  list[NodeScheduleEntry]r  r   r   r   )rM  r  r   zdict[str, Any])r  rL   )rK  list[BaseSchedulerNode]r  r   r   r   )r  rL   r   zlist[SIMDKernel])r   r   r   tuple[tuple[int, ...], ...])r   r   r   r   )r   r   r  r   r   r  )r  r   r   
str | None)r  rK  r  r   r   ztuple[str, TritonKernel])r  rK  r   ztuple[list[Any], Any])r	  r  r  r  r   r   )FF)r$  r  r%  r   r  r   r  r   r  r   r  r   r   z!list[tuple[str | None, Any, Any]])r   rX  )ra  r  rb  r  r   immutable_dict[str, sympy.Expr])rO  r  rO  r   r   r  )rO  r  ro   r   r  r   r   r  )r   z%list[immutable_dict[str, sympy.Expr]])
rM  r  rs  r   r  r   r?  rR   r   :tuple[dict[str, sympy.Expr], dict[str, sympy.Expr] | None])rM  r  ro   r   r  r   rO  r  )rM  r  ro   r   r  r   r  zlist[dict[str, sympy.Expr]])r?  r  r   r  )r?  r  r   r  r   )FN)r  r   )Cr   r   r   r   r   rh  rI  rl  ry  can_fuse_verticalr  r  r  r  r  r  r  r)  r-  r,  rP  r  rU  rM  r  rd  r*  r+  r  rt  r  r  rG  rq  rH  r  r  r  r  r  r  r  r  r  r  r  r6  r=  r  r  r^  re  rK  rZ  r  r  r  r  r   r   r   r}  r(  r  r  r  r  r   rd   rb   rg  rg    s   
 (K'Qm6^ !"^@8
%2)l OS"5K"	"
d)Ll
f #F 
F8N 3	N
 (N !0N ,N#
N 
N`
#
 ,
 !0	

 

& 
$
.
 

 
 3 (	 
2 

 0
 
 	
 
.K
 
K" ;?( (	(
  ( 8( 
(B 9=
0
 6
(=J=2 ""T" 
" "P #(	@@ /@  	@
 
@"I)` "- 6-
 - 
-:
1
	
!-V  IV'	$,

'
/2
	$
.  $(n "n 
n`I ! 6: 	! $;z +
!+
1:+
	+
 +
f #(%*e 0e   $e  	e 
 e   e  #e  
+e N)4 Y}  }~ 
,
@T
	(
 
$ 
$
 
 
)	
 
 /%/ / $	/
 
)/ /( S
 
/S Sj Y$.Y$ $Y$ $	Y$
 /Y$ 
DY$ Y$v 
.
 
 $	

 &
 
  .  $	
 4  
 8<	
 6	 
	 	 
 8<S$
 6S$ 
DS$ S$j JN#<F#J"rd   rg  c                  @    e Zd ZU ded<   ded<   dZded<   ed        Zy)	rJ  r  rO  r   rE  Nr  rl   c                v    t         j                  j                  j                  | d      } | dk\  xr | dz  dk(  S )z@Somewhat arbitrary heuristic used to boost scores for some sizesi    r  r>  r   )r?   r   r   r   )r   s    rb   rN  zCandidateTiling.is_good_size  s:     GG..q4.@Bw(AFaK(rd   )r   r   r   rI  rl   r  rN  r   rd   rb   rJ  rJ    s)    !!JD*) )rd   rJ  c                  $     e Zd Z fdZd Z xZS )r  c                >    t         |           || _        || _        y r\   )rj   rk   r   r  )rs   r   r  rt   s      rb   rk   zCantSplit.__init__  s    	"rd   c                8    | j                    d| j                   S )Nz not divisible by )r   r  rv   s    rb   __str__zCantSplit.__str__  s    )).t~~.>??rd   )r   r   r   rk   r  r   r   s   @rb   r  r    s    #
@rd   r  )r   )ra   r   r   r   )rA  r  r   r   )
__future__r   r  r  r  r  ri  r  r>  r  r  r   typingr   r   r   r   r	   typing_extensionsr
   r   r]   torch._loggingtorch._inductorr   torch._inductor.irr   %torch.fx.experimental.symbolic_shapesr   torch.fx.immutable_collectionsr   torch.utils._ordered_setr   torch.utils._sympy.functionsr   r   r   torch.utils._sympy.symbolr   r   r   r   _dynamo.utilsr    r   r   r   analyze_preserves_zero_maskr   	codecacher   r   dependenciesr    r!   r"   collections.abcr#   r%   ops_handlerr&   optimize_indexingr'   r(    runtime.coordinate_descent_tunerr)   runtime.hintsr*   runtime.runtime_utilsr+   r,   r-   r.   r/   r0   r1   utilsr2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   virtualizedr=   r>   r?   block_analysisrA   commonrB   rC   rD   rE   r  rF   rG   simd_kernel_featuresrH   rI   rJ   rK   rL   rM   rN   rO   torch._inductor.codegen.tritonrQ   torch._inductor.tiling_utilsrR   	getLoggerr   r  _logginggetArtifactLoggerr  ro  
fusion_logdoprintr  r  rc   	dataclassrf   r   r   r+  rB  rC  rF  rK  r   r  r  r  r  rE  rI  ra  rg  rJ  	Exceptionr  r   rd   rb   <module>r&     s   "           @ @ %    # 2 G 9 / L L  & $ $ F . 6 6 ( ( = ,  E D    - , / P P :  <<;@ g!00<H~~//*E^^--hA
 	78;
 3+ 3+ 3+lr;/ r;j;'? ;'|C!4 CL +;T   "z "^('/*B ^B d#! ! $! ; ; ;| d#$ $ $$& d#F F $FR
 d#

 

 $

:;. :;z(F^ (FVn'"^ n'"bO d#	) 	) $	)@	 @rd   