Ë
    ÄÓ^j± ã                  ó„  — d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZmZmZmZmZ d dlmZ d dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d d	lmZ d d
lm Z  d dl!m"Z"m#Z#m$Z$ d dl%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6m7Z7m8Z8 erd dl9m:Z: ddl.m;Z; ddl<m=Z= ddl>m?Z?m@Z@ ddlAmBZB ddlCmDZD ddlEmFZFmGZGmHZHmIZI ddl/mJZJmKZKmLZL ddlMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZX ddlYmZZZm[Z[m\Z\ ddl]m^Z^ dd l_m`Z`maZambZbmcZc dd!ldmeZemfZf dd"lgmhZhmiZimjZjmkZkmlZl erd d#l9mmZmmnZnmoZo d d$lpmqZq d d%lrmsZs  ejè                  eu«      Zvejî                  jñ                  eud&«      Zyejî                  jñ                  eud'«      Zzejî                  jñ                  eud(«      Z{ ec«       jø                  Z} e g d)¢«      Z~dRdSd*„Ze�j                    G d+„ d,«      «       Z� G d-„ d.e�«      Z‚ G d/„ d0e�«      Zƒ G d1„ d2e‚«      Z„dTd3„Z… ed4e`e`¬5«      Z†e�j                    G d6„ d7«      «       Z‡ G d8„ d9e«      Zˆ G d:„ d;ebe†   ee†   «      Z‰ e�j                   d<¬=«       G d>„ d?«      «       ZŠe�j                    G d@„ dA«      «       Z‹ e�j                   d<¬=«       G dB„ dC«      «       ZŒ e�j                   d<¬=«       G dD„ dE«      «       Z� e�j                   d<¬=«       G dF„ dG«      «       ZŽ G dH„ dIe=«      Z� G dJ„ dKe=«      Z� G dL„ dMeK«      Z‘ e�j                   d<¬=«       G dN„ dO«      «       Z’ G dP„ dQe“«      Z”y)Ué    )ÚannotationsN)ÚCounter)ÚAnyÚcastÚGenericÚ
NamedTupleÚTYPE_CHECKING)ÚTypeVar)Úmetrics)ÚMultiTemplateBuffer)Úfree_unbacked_symbols)Úimmutable_dict)Ú
OrderedSet)ÚFloorDivÚIdentityÚModularIndexing)Úfree_symbol_is_typeÚ
prefix_strÚsymbol_is_typeÚSymTé   )Úcountersé   )ÚconfigÚirÚ	scheduler)Úprologue_preserves_zero_mask)Ú	code_hashÚPyCodeCache)Ú	MemoryDepÚStarDepÚWeakDep)ÚCallable©ÚIRNode)ÚWrapperHandler)Ú convert_index_expr_to_value_exprÚ!indexing_dtype_strength_reduction)ÚCoordescTuner)ÚDeviceProperties)Ú
green_textÚlast_power_of_2Únext_power_of_2Úyellow_text)ÚBaseSchedulerNodeÚBaseSchedulingÚ	WhyNoFuse)Úcache_property_on_selfÚdecompose_indexÚexpr_fits_within_32bitÚget_dtype_sizeÚIndentedBufferÚPlaceholderÚprefix_is_reductionÚsympy_index_symbolÚsympy_productÚ
sympy_subsÚunique)ÚopsÚ
OpsWrapperÚVé   )ÚBlockPatternMatcher)ÚCSEVariableÚindex_prevent_reorderingÚKernelÚPythonPrinter)ÚMultiKernelÚSizeHintMultiKernel)ÚDisableReductionÚEnableReductionÚNodeScheduleEntryÚNodeScheduleMarkerÚSIMDKernelFeatures)ÚIterableÚIteratorÚSequence©ÚTritonKernel)ÚCoalesceVarAnalysisÚ
perf_hintsÚscheduleÚfusion)ÚzÚyÚxÚr0_Úr1_c                ój   — t         j                  j                  j                  j                  }|�|S | S ©N)ÚtorchÚ	_inductorr   ÚtritonÚ	max_tiles)Údefaultr`   s     úg/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/codegen/simd.pyÚget_max_tilesrc   h   s-   € Ü—‘×&Ñ&×-Ñ-×7Ñ7€IØ!Ð-ˆ9Ð:°7Ð:ó    c                  óÚ   ‡ — e Zd ZdZej
                  j                  ej
                  j                  dœ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zee	dd„«       «       Z
d	d„Zee	d
d„«       «       Zˆ xZS )ÚIterationRangesaÀ  
    Each range tree represents multiple sets of iteration indexing
    in a single tiled dimension in the output kernel.

    If you have two loops ranges one (4, 3, 2) and another (4, 6),
    then the range tree will be:
            4 (i0)
        3 (i1)  6 (i3)
        2 (i2)
    Where i0 is shared between both loops, but then the split into
    different indexing vars.  All loop ranges must iterate over
    the same number of elements.
    )ÚdivisorÚlengthc               ó    •— t         ‰
| �  «        || _        || _        || _        || _        || _        || _        || _        || _	        |	| _
        y r\   )ÚsuperÚ__init__ÚnameÚvar_listÚ
var_rangesÚnumelÚprefixrg   rh   ÚkernelÚroot)Úselfrl   rm   rn   ro   rp   rq   rg   rh   rr   Ú	__class__s             €rb   rk   zIterationRanges.__init__}   sO   ø€ ô 	‰ÑÔØˆŒ	Ø ˆŒØ$ˆŒØˆŒ
ØˆŒØˆŒØˆŒØˆŒØˆ�	rd   c                ó,   — t        | j                  «      S r\   )r8   rp   ©rs   s    rb   Úis_reductionzIterationRanges.is_reduction•   s   € ô # 4§;¡;Ó/Ð/rd   c                ó,   — t        | j                  «      S r\   ©r9   rl   rv   s    rb   ÚsymbolzIterationRanges.symbolš   ó   € Ü! $§)¡)Ó,Ð,rd   c                óz   — t        j                  «       D ��ci c]  \  }}||“Œ
 }}}|| j                     S c c}}w r\   )r   Úitemsrp   )rs   Úsymtrp   Úprefix_to_symts       rb   r~   zIterationRanges.symt�   s>   € ô <F×;KÑ;KÓ;M×N©<¨4°˜& $™,ÐNˆÑNØ˜dŸk™kÑ*Ð*ùó Os   ˜7)rl   Ústrrm   úlist[sympy.Symbol]rn   údict[sympy.Symbol, sympy.Expr]ro   ú
sympy.Exprrp   r€   rq   Ú
SIMDKernelrr   ÚIterationRangesRootÚreturnÚNone©r†   Úbool©r†   úsympy.Symbol)r†   r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚsympyÚSÚOnerk   Úpropertyr2   rw   rz   r~   Ú__classcell__©rt   s   @rb   rf   rf   m   s°   ø„ ñð. —‘—‘Ø�w‰w�{‰{ñàðð %ðð 3ð	ð
 ðð ðð ðð "ðð 
õð0 Øò0ó ó ð0ó-ð Øò+ó ó ô+rd   rf   c                  óê   ‡ — e Zd ZdZ	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zdd„Zdd„Zdd„Zdd„Zdd„Z	dd„Z
dd	„Zdd
„Zdd„Zdd„Zdd„Zdd„Zdd„Zdd„Z	 	 	 	 d d„Zd!d„Z	 	 	 	 d"d„Zˆ xZS )#r…   z½
    Root of a iteration range tree that represents a single
    tiled dimension in the output kernel. It contains multiple
    sets of iteration represented with IterationRangesEntry.
    c          	     óÀ   •— |€i }t         ‰| �  |g i |||| ¬«       || _        i | _        || _        |r| j
                  r|	�J ‚|| _        || _        |	| _        |
| _	        y )N)rl   rm   rn   ro   rp   rq   rr   )
rj   rk   ÚindexÚnodesÚ	pid_cacherw   Úis_loopÚ
tensor_dimÚgrid_dimÚhas_zdim)rs   rl   ro   rp   r˜   rq   rš   r›   rœ   r�   rž   rt   s              €rb   rk   zIterationRangesRoot.__init__«   s„   ø€ ð ÐØˆIÜ‰ÑØØØØØØØð 	ô 	
ð ˆŒ
à=?ˆŒ
ð *3ˆŒñ
 ˜t×0Ò0°XÐ5EÐFÐFØˆŒà$ˆŒà ˆŒØ ˆ�rd   c                ó<   — d| j                   ›d| j                  › d�S )NzIterationRangesRoot(ú, z, ...))rl   ro   rv   s    rb   Ú__repr__zIterationRangesRoot.__repr__Ö   s   € Ø% d§i¡i ]°"°T·Z±Z°LÀÐGÐGrd   c                óh   — t        j                  | j                  j                  «       › d�dd¬«      S )NÚBLOCKT©ÚintegerÚpositive)r�   ÚSymbolrp   Úupperrv   s    rb   Ú
block_sizezIterationRangesRoot.block_sizeÙ   s+   € Ü�|‰|˜tŸ{™{×0Ñ0Ó2Ð3°5Ð9À4ÐRVÔWÐWrd   c                ó4   — t        | j                  «       «      S r\   )r€   r©   rv   s    rb   Úblock_size_strz"IterationRangesRoot.block_size_strÜ   s   € Ü�4—?‘?Ó$Ó%Ð%rd   c                óL   — t        j                  | j                  › d�dd¬«      S )NÚoffsetT)r¥   Únonnegative)r�   r§   rp   rv   s    rb   Úblock_offsetz IterationRangesRoot.block_offsetß   s"   € ô �|‰|˜tŸ{™{˜m¨6Ð2¸DÈdÔSÐSrd   c                ó    — | j                   › d�S )NÚmask)rp   rv   s    rb   Ú	mask_namezIterationRangesRoot.mask_nameä   s   € Ø—+‘+�˜dÐ#Ð#rd   c                ó(   — || j                  «       k(  S r\   )r²   )rs   Úmask_vars     rb   Ú	owns_maskzIterationRangesRoot.owns_maskç   s   € Ø˜4Ÿ>™>Ó+Ñ+Ð+rd   c                óx   — | j                   €ydg|z  }| j                  «       || j                   <   t        |«      S )N© Ú1)rœ   r«   Útuple)rs   Útensor_ndimÚshapes      rb   Ú
mask_shapezIterationRangesRoot.mask_shapeê   s<   € Ø�?‰?Ð"Øà�˜Ñ#ˆØ!%×!4Ñ!4Ó!6ˆˆd�o‰oÑÜ�U‹|Ðrd   c                 ó   — y©NTr·   rv   s    rb   Úsupports_constant_maskz*IterationRangesRoot.supports_constant_maskò   ó   € Ørd   c                 ó   — y©NFr·   rv   s    rb   Úhas_custom_codegen_headerz-IterationRangesRoot.has_custom_codegen_headerõ   ó   € Ørd   c                 ó   — y©Nr·   r·   rv   s    rb   Únamed_constantsz#IterationRangesRoot.named_constantsø   s   € Ørd   c                ób   — | j                   j                  «       D ]  }|j                  «        Œ y r\   )r™   ÚvaluesÚcache_clear)rs   Únodes     rb   rÊ   zIterationRangesRoot.cache_clearû   s*   € Ø—J‘J×%Ñ%Ó'ò 	ˆDØ×ÑÕñ	rd   c                ó2   — t        | j                  › d�«      S )Nr˜   )r9   rp   rv   s    rb   Ú	index_symzIterationRangesRoot.index_symÿ   s   € Ü! T§[¡[ M°Ð"7Ó8Ð8rd   c                ó’  — t         j                  j                  j                  ||z  | j                  «      rt        | j                  «       |«      }nt        | j                  «       ||«      }|| j                  vr¾t        | j                  › t        t         j                  j                  «      › �|||| «      }|t         j                  j                  |j                  «       <   | j                   j#                  |j                  «       «       || j$                  |j                  «       <   || j                  |<   | j                  |   S )zF
        Lookup a given RangeTreeEntry, creating it if needed
        )r?   ÚgraphÚsizevarsÚstatically_known_equalsro   r   rÍ   r   r™   ÚIterationRangesEntryrp   Únextrq   Úiter_vars_countÚrange_tree_nodesrz   rm   Úappendrn   )rs   rg   rh   ÚexprrË   s        rb   ÚlookupzIterationRangesRoot.lookup  sö   € ô �7‰7×Ñ×3Ñ3°G¸fÑ4DÀdÇjÁjÔQÜ˜DŸN™NÓ,¨gÓ6‰Dä" 4§>¡>Ó#3°W¸fÓEˆDà�t—z‘zÑ!Ü'Ø—;‘;�-¤¤Q§X¡X×%=Ñ%=Ó >Ð?Ð@ØØØØóˆDð 8<ŒA�H‰H×%Ñ% d§k¡k£mÑ4Ø�M‰M× Ñ  §¡£Ô/Ø-3ˆD�O‰O˜DŸK™K›MÑ*Ø#ˆD�J‰J�tÑØ�z‰z˜$ÑÐrd   c                ój   — | j                  t        j                  j                  | j                  «      S )zÃReturn the canonical entry for this root's unsplit logical index.

        This is ``lookup(1, numel)``, so the entry is interned in
        ``range_tree_nodes`` just like split entries.
        )rØ   r�   r‘   r’   ro   rv   s    rb   Ú
full_rangezIterationRangesRoot.full_range  s!   € ð �{‰{œ5Ÿ7™7Ÿ;™;¨¯
©
Ó3Ð3rd   c                óÀ   — t         j                  j                  }g }t        |«      D ](  }|j	                  | j                  ||«      «       ||z  }Œ* g t        |«      ¢S r\   )r�   r‘   r’   ÚreversedrÖ   rØ   )rs   Úlengthsrg   Úitervarsrh   s        rb   Úconstruct_entriesz%IterationRangesRoot.construct_entries!  s]   € ô —'‘'—+‘+ˆØˆÜ˜wÓ'ò 	'ˆFØ�O‰O˜DŸK™K¨°Ó8Ô9Ø Ñ&‰Gð	'ð %”˜(Ó#Ð$Ð$rd   c                óf   — | j                  |«      D �cg c]  }|j                  «       ‘Œ c}S c c}w r\   )rß   rz   )rs   rÝ   Úes      rb   Ú	constructzIterationRangesRoot.construct+  s'   € Ø$(×$:Ñ$:¸7Ó$CÖD˜q�—‘•
ÒDÐDùÒDs   ”.c           
     óF  ‡‡‡	‡
— dd„Š|j                   D �cg c]+  }t        j                  j                  j	                  |«      ‘Œ- }}|D �cg c]  }|sŒ|j
                  | u sŒ|‘Œ }}|j                  ˆfd„¬«       t        j                  j                  Šg Š	g Š
ˆˆ	ˆ
fd„}|D ]v  }t        j                  j                  j                  |j                  ‰«      s8 || j                  ‰t        |j                  ‰«      «      «       |j                  Š ||«       Œx t        j                  j                  j                  | j                   ‰«      s, || j                  ‰t        | j                   ‰«      «      «       g t#        ‰	«      ¢g t#        ‰
«      ¢fS c c}w c c}w )z,Figure out vars from this tree used in indexc                óÞ   — t         j                  j                  j                  | j                  «      }t         j                  j                  j                  | j
                  «      dk(  }|| fS )a:  
            Gets the key for sorting nodes. When two nodes have the
            same divisor, the node with length as 1 should be handled
            first so the current divisor is not changed after multiplied
            node.length. Returns `not length_is_one_hint` for ascending
            sort.
            r@   )r?   rÏ   rÐ   Úoptimization_hintrg   rh   )rX   Údivisor_hintÚlength_is_one_hints      rb   Úget_sort_keyz8IterationRangesRoot.vars_and_sizes.<locals>.get_sort_key3  sV   € ô Ÿ7™7×+Ñ+×=Ñ=¸a¿i¹iÓHˆLÜ!"§¡×!1Ñ!1×!CÑ!CÀAÇHÁHÓ!MÐQRÑ!RÐØ Ð&8Ð"8Ð9Ð9rd   c                ó   •—  ‰| «      S r\   r·   )rX   rè   s    €rb   ú<lambda>z4IterationRangesRoot.vars_and_sizes.<locals>.<lambda>A  s   ø€ ¡¨a£€ rd   ©Úkeyc                ó˜   •— ‰j                  | j                  «       «       ‰j                  | j                  «       ‰| j                  z  Šy r\   )rÖ   rz   rh   )rË   rg   Ú
index_varsÚsizess    €€€rb   Úaddz/IterationRangesRoot.vars_and_sizes.<locals>.addF  s5   ø€ à×Ñ˜dŸk™k›mÔ,Ø�L‰L˜Ÿ™Ô%Ø §¡Ñ+‰Grd   )rX   rÒ   r†   ztuple[int, bool])Úfree_symbolsr?   rq   rÕ   Úgetrr   Úsortr�   r‘   r’   rÏ   rÐ   rÑ   rg   rØ   r   ro   rÜ   )rs   r˜   Úsr™   Únrð   rË   rg   rè   rî   rï   s          @@@@rb   Úvars_and_sizesz"IterationRangesRoot.vars_and_sizes.  sF  û€ ó

	:ð <A×;MÑ;MÖN°a”—‘×*Ñ*×.Ñ.¨qÕ1ÐNˆÐNØ!Ö:�q¢Q¨1¯6©6°Tª>’Ð:ˆÐ:Ø�
‰
Ó0ˆ
Ô1Ü—'‘'—+‘+ˆØˆ
Øˆö	,ð ò 	ˆDÜ—7‘7×#Ñ#×;Ñ;¸D¿L¹LÈ'ÔRá�D—K‘K ¬°$·,±,ÀÓ)HÓIÔJØŸ,™,�Ù��Ið	ô �w‰w×Ñ×7Ñ7¸¿
¹
ÀGÔLá�—‘˜G¤X¨d¯j©j¸'Ó%BÓCÔDà&”˜*Ó%Ð&Ð(:¬(°5«/Ð(:Ð:Ð:ùò/ OùÚ:s   —0FÁFÁFÁ$Fr\   )rl   r€   ro   rƒ   rp   r€   r˜   Úintrq   r„   rš   údict[str, str] | Noner›   r‰   rœ   ú
int | Noner�   rù   rž   r‰   r†   r‡   ©r†   r€   ©r†   rƒ   )r´   r€   r†   r‰   )rº   r÷   r†   ztuple[str, ...]rˆ   ©r†   ú1tuple[tuple[sympy.Symbol, sympy.Expr, bool], ...]©r†   r‡   rŠ   )rg   rƒ   rh   rƒ   r†   rÒ   )r†   rÒ   )rÝ   úlist[sympy.Expr]r†   zlist[IterationRangesEntry])rÝ   rÿ   r†   r�   )r˜   rƒ   r†   z+tuple[list[sympy.Symbol], list[sympy.Expr]])rŒ   r�   rŽ   r�   rk   r¡   r©   r«   r¯   r²   rµ   r¼   r¿   rÃ   rÇ   rÊ   rÍ   rØ   rÚ   rß   râ   rö   r”   r•   s   @rb   r…   r…   ¤   sø   ø„ ñð ,0ð)!àð)!ð ð)!ð ð	)!ð
 ð)!ð ð)!ð )ð)!ð ð)!ð ð)!ð ð)!ð ð)!ð 
õ)!óVHóXó&óTó
$ó,óóóóóó9ó ó.4ð%Ø'ð%à	#ó%óEð(;Øð(;à	4÷(;rd   r…   c                  óp   ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 d	ˆ fd„Zd
d„Zdd„Zdd„Zd
d„Zdd„Zdd„Z	dd„Z
ˆ xZS )rÒ   c                ó$  •— t         ‰| �  ||j                  |z  |j                  |j                  |j
                  |||j                  |j                  ¬«	       || _         t        j                  d «      | j                  «      | _        || _        y )N)	rl   ro   rm   rn   rp   rg   rh   rq   rr   )rj   rk   ro   rm   rn   rp   rq   rr   ÚparentÚ	functoolsÚ	lru_cacheÚ_codegenÚcodegenr×   )rs   rl   rg   rh   r×   r  rt   s         €rb   rk   zIterationRangesEntry.__init__Z  s~   ø€ ô 	‰ÑØØ—,‘, Ñ'Ø—_‘_Ø×(Ñ(Ø—=‘=ØØØ—=‘=Ø—‘ð 	ô 
	
ð ˆŒØ0”y×*Ñ*¨4Ó0°·±Ó?ˆŒØˆ�	rd   c                óŠ   — d| j                   › d| j                  › d| j                  › d| j                  › d| j                  › d�S )NzIterationRangesEntry(r    ú))rl   rg   rh   r×   rn   rv   s    rb   r¡   zIterationRangesEntry.__repr__q  sH   € Ø& t§y¡y k°°D·L±L°>ÀÀDÇKÁKÀ=ÐPRÐSW×S\ÑS\ÐR]Ð]_Ð`d×`oÑ`oÐ_pÐpqÐrÐrrd   c                óL   ‡— ˆfd„| _         d„ | j                   _        ‰| _        y )Nc                 ó   •— ‰ S r\   r·   ©rl   s   €rb   rê   z/IterationRangesEntry.set_name.<locals>.<lambda>u  s   ø€ ˜t€ rd   c                  ó   — y r\   r·   r·   rd   rb   rê   z/IterationRangesEntry.set_name.<locals>.<lambda>v  ó   � rd   )r  rÊ   rl   )rs   rl   s    `rb   Úset_namezIterationRangesEntry.set_namet  s   ø€ Û#ˆŒÙ#/ˆ�‰Ô Øˆ�	rd   c                ó8   — | j                   j                  «        y r\   )r  rÊ   rv   s    rb   rÊ   z IterationRangesEntry.cache_cleary  s   € Ø�‰× Ñ Õ"rd   c                óX   — t         j                  j                  | «       | j                  S r\   )r?   rq   Úcodegen_iteration_ranges_entryrl   rv   s    rb   r  zIterationRangesEntry._codegen|  s   € Ü	�‰×/Ñ/°Ô5Ø�y‰yÐrd   c                óÚ  — g }t        | j                  t        j                  «      r|S t        | j                  t        t
        f«      sJ t        | j                  «      «       ‚| j                  j                  dd  D ]l  }t        |t        j                  t        j                  f«      rŒ.|j                  }t        |«      dkD  sŒIt        d„ |D «       «      sŒ\|j                  |«       Œn |S )Nr@   r   c              3  óP   K  — | ]  }t        |t        j                  «      –— Œ  y ­wr\   )r   r   ÚSIZE©Ú.0rô   s     rb   ú	<genexpr>z8IterationRangesEntry.precomputed_args.<locals>.<genexpr>‰  s    è ø€ ò ,Ø56”N 1¤d§i¡i×0ñ,ùó   ‚$&)Ú
isinstancer×   r�   r§   r   r   ÚtypeÚargsÚIntegerrñ   ÚlenÚallrÖ   )rs   Úprecomputed_argsÚargÚsymbolss       rb   r  z%IterationRangesEntry.precomputed_args€  s¾   € à-/ÐÜ�d—i‘i¤§¡Ô.Ø#Ð#Ü˜$Ÿ)™)¤h´Ð%@ÔAÐRÄ4ÈÏ	É	Ã?ÓRÐAØ—9‘9—>‘> ! "Ð%ò 	1ˆCÜ˜c¤E§M¡M´5·<±<Ð#@ÕAØ×*Ñ*�Ü�w“< !Ó#¬ñ ,Ø:Aô,õ )ð %×+Ñ+¨CÕ0ð	1ð  Ðrd   c                ó,   — t        | j                  «      S r\   )Úhashrl   rv   s    rb   Ú__hash__zIterationRangesEntry.__hash__�  s   € Ü�D—I‘I‹Ðrd   c                óX   — t        |t        «      sJ ‚| j                  |j                  k(  S r\   )r  rÒ   rl   )rs   Úothers     rb   Ú__eq__zIterationRangesEntry.__eq__’  s&   € Ü˜%Ô!5Ô6Ð6Ð6Ø�y‰y˜EŸJ™JÑ&Ð&rd   )rl   r€   rg   rƒ   rh   rƒ   r×   rƒ   r  rf   r†   r‡   rú   )rl   r€   r†   r‡   rþ   )r†   rÿ   ©r†   r÷   )r&  Úobjectr†   r‰   )rŒ   r�   rŽ   rk   r¡   r  rÊ   r  r  r$  r'  r”   r•   s   @rb   rÒ   rÒ   Y  sf   ø„ ðàðð ðð ð	ð
 ðð  ðð 
õó.sóó
#óó ó÷'rd   rÒ   c                  ó~   ‡ — e Zd ZdZdddœ	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zdd„Zdd„Zdd„Zdd	„Zdd
„Z	dd„Z
dd„Zˆ xZS )ÚDerivedIterationRangesRootae  A root with reduced numel/block_size derived from a parent tree.

    Used for the grouped reduction output: if the parent R tree covers 1024
    columns with local_reduction_size=128, the derived root covers 8 groups. It shares
    the parent's loop structure (same is_loop, prefix, loop variable) but
    has its own block geometry:

        parent R:  numel=1024, block_size=RBLOCK, offset=roffset
        derived R: numel=8,    block_size=RBLOCK//128, offset=roffset//128

    This lets the grouped reduction store at reduced-resolution offsets
    while still running inside the parent's reduction loop.
    Úreducedr·   )Úname_suffixrÇ   c               ó   •— t         ‰| �  |› d|j                  › �||j                  |j                  |j
                  |j                  |j                  |j                  |j                  |j                  ¬«
       || _        || _        || _        y )NÚ_)
rl   ro   rp   r˜   rq   rš   r›   rœ   r�   rž   )rj   rk   rl   rp   r˜   rq   rš   r›   rœ   r�   rž   Ú_block_sizeÚ_block_offsetÚ_named_constants)rs   r  ro   r©   r¯   r-  rÇ   rt   s          €rb   rk   z#DerivedIterationRangesRoot.__init__¦  s„   ø€ ô 	‰ÑØ�=  &§+¡+ Ð/ØØ—=‘=Ø—,‘,Ø—=‘=Ø×&Ñ&ð —N‘NØ×(Ñ(Ø—_‘_Ø—_‘_ð 	ô 	
ð" &ˆÔØ)ˆÔØ /ˆÕrd   c                ó   — | j                   S r\   )r0  rv   s    rb   r©   z%DerivedIterationRangesRoot.block_sizeÅ  s   € Ø×ÑÐrd   c                ó   — | j                   S r\   )r1  rv   s    rb   r¯   z'DerivedIterationRangesRoot.block_offsetÈ  s   € Ø×!Ñ!Ð!rd   c                ó,   — t        | j                  «      S r\   ry   rv   s    rb   rÍ   z$DerivedIterationRangesRoot.index_symË  r{   rd   c                ó    — | j                   › d�S )NÚ_maskr  rv   s    rb   r²   z$DerivedIterationRangesRoot.mask_nameÎ  s   € Ø—)‘)�˜EÐ"Ð"rd   c                 ó   — yrÂ   r·   rv   s    rb   r¿   z1DerivedIterationRangesRoot.supports_constant_maskÑ  s   € ð rd   c                ó   — | j                   S r\   )r2  rv   s    rb   rÇ   z*DerivedIterationRangesRoot.named_constantsÖ  s   € Ø×$Ñ$Ð$rd   c                 ó   — yr¾   r·   rv   s    rb   rÃ   z4DerivedIterationRangesRoot.has_custom_codegen_headerÙ  rÀ   rd   )r  r…   ro   rƒ   r©   rƒ   r¯   rƒ   r-  r€   rÇ   rý   r†   r‡   rû   rŠ   rú   rˆ   rü   )rŒ   r�   rŽ   r�   rk   r©   r¯   rÍ   r²   r¿   rÇ   rÃ   r”   r•   s   @rb   r+  r+  —  s}   ø„ ñð* %ØMOñ0à#ð0ð ð	0ð
 ð0ð !ð0ð ð0ð Kð0ð 
õ0ó> ó"ó-ó#óó
%÷rd   r+  c                ó€   — | t        d«      k(  ry| t        d«      k(  ryt        j                  | «      ryt        | «      S )NÚinfzfloat("inf")z-infzfloat("-inf")zfloat("nan"))ÚfloatÚmathÚisnanÚrepr)Úvalues    rb   Úconstant_reprrB  Ý  s9   € Ø”�e“ÒØØ	”%˜“-Ò	ØÜ	�‰�EÔ	ØÜ�‹;Ðrd   ÚCSEVariableType)Úboundra   c                  ó,   — e Zd ZU ded<   ded<   ded<   y)ÚPartialAccumulater€   Úbuffer_nameÚreduction_typer   rA  N)rŒ   r�   rŽ   Ú__annotations__r·   rd   rb   rF  rF  ê  s   … àÓØÓØ„Jrd   rF  c                  óX   — e Zd ZU dZded<   ded<   ded<   ded	<   ded
<   ded<   ded<   y)ÚNodeInfozF
    Pre-computed node information for combo kernel partitioning.
    ÚlistÚnode_scheduleÚdictÚtilingzdict | NoneÚtiling_scoresr   ro   ÚrnumelrL   Úfeaturesr‰   Úis_persistent_reductionN©rŒ   r�   rŽ   r�   rI  r·   rd   rb   rK  rK  ñ  s0   … ñð ÓØƒLØÓØƒJØƒKØ Ó Ø!Ô!rd   rK  c                  óp  ‡ — e Zd ZU dZeZded<   ded<   dZded<   ded	<   	 	 	 	 	 d;	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d<ˆ fd
„Z	 	 d=d„Z	d>d„Z
d?d„Zd„ Zeed@d„«       «       ZdAd„ZdBd„Zed=d„«       ZdCd„Z	 	 	 	 	 	 	 	 	 	 	 	 dDd„ZdEd„ZdFd„ZdGd„ZdCd„ZdCd„ZdHd„ZdId„Z	 	 	 	 	 	 dJd„Z	 	 	 	 	 	 dJd„ZdKd„ZdLd„Ze 	 	 	 	 	 	 dMd „«       Z!e"e#jH                  jJ                  f	 	 	 	 	 	 	 dNd!„«       Z&e"e#jH                  jJ                  f	 	 	 	 	 	 	 dOd"„«       Z'	 	 	 	 dPd#„Z(e"	 	 	 	 	 	 dQd$„«       Z)dRd%„Z*dRd&„Z+dSd'„Z,	 	 	 	 dId(„Z-dTd)„Z.e/j`                  	 	 	 	 dUd*„«       Z1dVd+„Z2dWd,„Z3	 	 	 	 	 	 dXd-„Z4d.„ Z5	 dY	 	 	 	 	 	 	 dZd/„Z6e/j`                  d[d0„«       Z7d\d1„Z8e d2„ «       Z9d]d3„Z:d4„ Z;d5„ Z<d6„ Z=d7„ Z>d8„ Z?d9„ Z@d^d:„ZAˆ xZBS )_r„   zo
    Common base class for Triton/Halide codegen which both use flattened indexing rather than loop nests.
    zCallable[[sympy.Expr], str]ÚsexprÚkexprFr‰   Úallow_block_ptrr€   Úkernel_namec                óØ  •‡ — |€i }t         ‰‰ �  «        |‰ _        |j                  «       ‰ _        t        «       ‰ _        t        «       ‰ _        |j                  «       D ��	ci c]/  \  }}	|t        j                  j                  j                  |	«      “Œ1 c}	}‰ _        g ‰ _        i ‰ _        t!        j"                  «       ‰ _        |j'                  «       ‰ _        |�|n‰ j+                  «       ‰ _        |‰ _        |‰ _        |�|n‰ j3                  «       ‰ _        |‰ _        ‰ j9                  «       ‰ _        d ‰ _        t!        j"                  «       ‰ _        d‰ _         tB        jD                  jF                  r�‰ j                  jH                  D ]h  }
tK        |
tL        jN                  «      sŒtK        |
jP                  tR        jT                  «      sŒC|
jP                  jW                  «       dk(  sŒad‰ _          n tX        jZ                  dˆ fd„«       }|‰ _.        ‰ j_                  |«       d‰ _0        d ‰ _1        d ‰ _2        g ‰ _3        ‰ j                  ji                  «       ‰ _5        y c c}	}w )NFÚdotTc                óÜ   •— t         j                  j                  j                  | ‰j	                  «       «      } ‰j
                  D ]  }‰j                  | |«      } Œ ‰j                  | «      S r\   )r?   rÏ   rÐ   Úsimplify_with_rangesrn   Úrange_treesÚcombine_contiguous_dimsÚcombine_modular_indexing_pairs)r˜   Útreers   s     €rb   Úsimplify_indexingz.SIMDKernel.__init__.<locals>.simplify_indexing?  sb   ø€ ä—G‘G×$Ñ$×9Ñ9¸%ÀÇÁÓARÓSˆEØ×(Ñ(ò B�Ø×4Ñ4°U¸DÓA‘ðBð ×6Ñ6°uÓ=Ð=rd   r   )r˜   rƒ   )6rj   rk   rR  Úget_mutationsÚ	mutationsr6   ÚbodyÚindexing_coder}   r?   rÏ   rÐ   ÚsimplifyÚnumelsr^  rÕ   Ú	itertoolsÚcountrÔ   rw   Úinside_reductionÚ should_use_cooperative_reductionÚcooperative_reductionrP  rO  Úshould_use_persistent_reductionÚpersistent_reductionÚmix_order_reductionÚwant_no_x_dimÚno_x_dimr   Ústore_output_ctrÚis_native_matmulr   r_   Únative_matmulrM  r  r   ÚSchedulerNoderË   r   ÚComputedBufferÚget_reduction_typer  Úcacherb  Úinitialize_range_treeÚrsplit_sizeÚ
min_xblockÚ
min_rblockÚsaved_partial_accumulateÚselect_index_dtypeÚ_index_dtype)rs   rO  rR  rš   Úoverride_persistent_reductionÚoverride_cooperative_reductionrP  rp  rp   ÚvalrË   rb  rt   s   `           €rb   rk   zSIMDKernel.__init__
  s  ù€ ð ÐØˆIÜ‰ÑÔØ ˆŒØ!×/Ñ/Ó1ˆŒÜ"Ó$ˆŒ	Ü+Ó-ˆÔàFLÇlÁlÃn÷
Ù7B°v¸sˆF”A—G‘G×$Ñ$×-Ñ-¨cÓ2Ñ2ó
ˆŒð 79ˆÔØJLˆÔÜ(Ÿ™Ó0ˆÔØ (× 5Ñ 5Ó 7ˆÔð .Ð9ñ +à×6Ñ6Ó8ð 	Ô"ð
 <IˆÔØ-3ˆŒð -Ð8ñ *à×5Ñ5Ó7ð 	Ô!ð
 *=ˆÔ Ø×*Ñ*Ó,ˆŒØ%)ˆŒä )§¡Ó 1ˆÔØ %ˆÔÜ�=‰=×&Ò&ØŸ™×3Ñ3ò �ä˜t¤Y×%<Ñ%<Õ=Ü" 4§9¡9¬b×.?Ñ.?Õ@ØŸ	™	×4Ñ4Ó6¸%Ó?à,0�DÔ)Ùðô 
�‰ô	>ó 
ð	>ð "3ˆÔØ×"Ñ" 9Ô-àˆÔØ&*ˆŒØ&*ˆŒØACˆÔ%Ø ŸM™M×<Ñ<Ó>ˆÕùóg
s   Á"4I&c                ó   — t         ‚)u-  Generate template source code with fused prologues and epilogues.

        Subclasses override this to implement custom code generation.
        The default implementation raises NotImplementedError â€” the actual
        standard path lives in ``TritonTemplateKernel.codegen_template_body``.
        ©ÚNotImplementedError)rs   Ú
schedulingÚtemplate_nodeÚepilogue_nodesÚprologue_nodesÚbuf_name_to_prologue_groupÚprologue_preserves_zero_mask_fnÚrenders           rb   Úcodegen_template_bodyz SIMDKernel.codegen_template_bodyP  s
   € ô  "Ð!rd   c                ó   — g S )a«  Return epilogue nodes that were not fused into the kernel.

        These nodes need separate codegen (via ``call_kernel``) and must
        be excluded from ``mark_run`` in ``_codegen_single_template``.

        The standard path fuses all epilogues, so this returns ``[]``.
        ``ExternalTritonTemplateKernel`` overrides this for epilogues that
        don't read exactly one template output and cannot be fused.
        r·   rv   s    rb   Úget_unfused_epiloguesz SIMDKernel.get_unfused_epiloguesb  s	   € ð ˆ	rd   c                ó   — d|› d�S )Nz<STORE_OUTPUT_ú>r·   )rs   Úis     rb   Ú_get_store_output_subgraph_namez*SIMDKernel._get_store_output_subgraph_namen  s   € Ø ˜s !Ð$Ð$rd   c                ón   — t        | j                  «      }t        j                  |dz
  d¬«      | _        |S )Nr@   )ÚstartÚstep)rÓ   rs  ri  rj  )rs   Útotals     rb   Úget_store_output_countz!SIMDKernel.get_store_output_countq  s.   € Ü�T×*Ñ*Ó+ˆÜ )§¡°e¸a±iÀaÔ HˆÔØˆrd   c                ó:   — t        d„ | j                  D «       «      S )Nc              3  ó2   K  — | ]  }t        |«      –— Œ y ­wr\   )r8   )r  rp   s     rb   r  z0SIMDKernel.num_reduction_dims.<locals>.<genexpr>y  s   è ø€ ÒI°6Ô& v×.ÑIùó   ‚)Úsumrh  rv   s    rb   Únum_reduction_dimszSIMDKernel.num_reduction_dimsv  s   € ô ÑI¸T¿[¹[ÔIÓIÐIrd   c                ó   — t         ‚r\   r…  )rs   Údtypes     rb   Údtype_to_strzSIMDKernel.dtype_to_str{  ó   € Ü!Ð!rd   c                ó   — | j                   S r\   )r€  rv   s    rb   Úget_index_dtype_as_torch_dtypez)SIMDKernel.get_index_dtype_as_torch_dtype~  s   € Ø× Ñ Ð rd   c                ó@   — | j                  | j                  «       «      S r\   )r¡  r¤  rv   s    rb   Úindex_dtypezSIMDKernel.index_dtype�  s   € à× Ñ  ×!DÑ!DÓ!FÓGÐGrd   c                 ó   — yrÂ   r·   rv   s    rb   rq  zSIMDKernel.want_no_x_dim…  rÄ   rd   c                óØ  ‡— t        ˆfd„t        D «       «      }| xs | }d	d„}g d¢}	t        t        |	«      «      }
ddg}|r|}n
|r|
}n|
|z   } |||«      } ||	t        «      }g }t	        |«      D ]s  \  }}t        |«      }|j                  |«      }|j                  |«      }|€|n|}|j                  t        |› d�‰|   ||| ||xr | j                   ||d‰v ¬«
      «       Œu |S )
Nc              3  ó,   •K  — | ]  }|‰v sŒ|–— Œ y ­wr\   r·   )r  rp   rh  s     €rb   r  z3SIMDKernel.construct_range_trees.<locals>.<genexpr>�  s   øè ø€ ò %
Ø°¸6Ò1AŒFñ%
ùó   ƒ	�c                ó`   ‡— t        ˆfd„| D «       «      D ��ci c]  \  }}||“Œ
 c}}S c c}}w )Nc              3  ó,   •K  — | ]  }|‰v sŒ|–— Œ y ­wr\   r·   )r  rƒ  r±   s     €rb   r  zOSIMDKernel.construct_range_trees.<locals>.filtered_index_map.<locals>.<genexpr>—  s   øè ø€ Ò2U¸3ÈÐPTÊ´3Ñ2Uùrª  )Ú	enumerate)Úseqr±   Úidxrƒ  s    `  rb   Úfiltered_index_mapz<SIMDKernel.construct_range_trees.<locals>.filtered_index_map•  s3   ø€ ä)2Ó2UÀ#Ô2UÓ)U÷Ù%˜S #��S‘óð ùó s   ™*)rX   rW   rV   rY   rZ   r˜   rV   )rš   r›   rœ   r�   rž   )r†   zdict[Any, int])
r   Úall_prefixesrL  rÜ   r­  r8   rò   rÖ   r…   ro  )rs   rš   rk  rw   rh  rr  Úactive_prefixesÚno_r_dimr°  Ú	grid_dimsÚpointwise_tensor_dimsÚreduction_dimsÚtensor_dimsÚtensor_dim_mapÚgrid_dim_mapr^  r“  rp   rœ   r�   r˜   s       `                rb   Úconstruct_range_treesz SIMDKernel.construct_range_treesˆ  s3  ø€ ô %ó %
Ü!-ô%
ó 
ˆð (Ð'Ò;¨|Ð+;ˆó	ò
 $ˆ	Ü $¤X¨iÓ%8Ó 9ÐØ ˜ˆÙØ(‰KÙØ/‰Kà/°.Ñ@ˆKñ ,¨K¸ÓIˆÙ)¨)´\ÓBˆàˆÜ" ?Ó3ò 	‰IˆAˆvÜ.¨vÓ6ˆLØ'×+Ñ+¨FÓ3ˆJØ#×'Ñ'¨Ó/ˆHØ!Ð)‘A¨xˆEØ×ÑÜ#Ø�h˜eÐ$Ø˜6‘NØØØØ'Ø(ÒJ°×1JÑ1JÐ-JØ)Ø%Ø  F˜]ôõð	ð& Ðrd   c                óÐ   — | j                  || j                  | j                  j                  «       | j                  | j
                  «      }| j                  j                  |«       y r\   )rº  rk  rR  rw   rh  rr  r^  Úextend)rs   rš   r^  s      rb   rz  z SIMDKernel.initialize_range_tree¿  sR   € Ø×0Ñ0ØØ×!Ñ!Ø�M‰M×&Ñ&Ó(Ø�K‰KØ�M‰Mó
ˆð 	×Ñ×Ñ Õ,rd   c                 ó   — y)zr
        Hook called right before codegen with every index that will be
        used in the fused kernel.
        Nr·   )rs   Úindicess     rb   Úfinalize_indexingzSIMDKernel.finalize_indexingÉ  r  rd   c                óv   — | j                   }d| _         	 | j                  |||«      || _         S # || _         w xY wrÂ   )rk  Ústore)rs   rl   r˜   rA  Úpriors        rb   Ústore_reductionzSIMDKernel.store_reductionÏ  s;   € Ø×%Ñ%ˆØ %ˆÔð	*Ø—:‘:˜d E¨5Ó1à$)ˆDÕ!ø EˆDÕ!ús   •/ ¯	8c                 ó   — yrÂ   r·   rv   s    rb   rl  z+SIMDKernel.should_use_cooperative_reduction×  rÄ   rd   c                 ó   — yrÂ   r·   rv   s    rb   rn  z*SIMDKernel.should_use_persistent_reductionÚ  rÄ   rd   c                ót   — t        t        j                  j                  d„ | j                  D «       «      «      S )Nc              3  óP   K  — | ]  }|j                   j                  «       –— Œ  y ­wr\   )rn   r}   ©r  ra  s     rb   r  z(SIMDKernel.var_ranges.<locals>.<genexpr>ß  s"   è ø€ ò *Ø,0�—‘×%Ñ%×'ñ*ùr  )rN  ri  ÚchainÚfrom_iterabler^  rv   s    rb   rn   zSIMDKernel.var_rangesÝ  s4   € ÜÜ�O‰O×)Ñ)ñ *Ø48×4DÑ4Dô*ó ó
ð 	
rd   c                óœ  — t        |t        «      s|S |j                  d   }| j                  j	                  |«      x}€|S t        |||j                  i«      }t        j                  j                  j                  |«      }t        ||j                  j                  «       |j                  j                  «       j                  «       i«      S ©Nr   )r  r   r  rÕ   rò   r;   r×   r?   rÏ   rÐ   r`  rr   rÍ   rÚ   rz   )rs   r˜   rX   Ú	tree_nodeÚ	new_indexs        rb   r`  z)SIMDKernel.combine_modular_indexing_pairsä  s©   € Ü˜%¤Ô1ØˆLØ�J‰J�q‰MˆØ×.Ñ.×2Ñ2°1Ó5Ð5ˆIÐ>ØˆLÜ˜u q¨)¯.©.Ð&9Ó:ˆ	Ü—G‘G×$Ñ$×CÑCÀIÓNˆ	äØØ�^‰^×%Ñ%Ó'¨¯©×)BÑ)BÓ)D×)KÑ)KÓ)MÐNó
ð 	
rd   c                óL  — t        |t        j                  «      r*|j                  t        «      r|j                  |«      \  }}nd }t        j                  j                  j                  ||«      x}r!|\  }}t	        | j                  ||«      |«      S | j                  ||«      S r\   )r  r�   ÚAddÚhasr   rö   r?   rÏ   rÐ   Úexpand_floor_divÚ_combine_contiguous_dims)rs   r˜   ra  Úcandidate_varsr/  Ú
expand_resrÎ  Údenominators           rb   r_  z"SIMDKernel.combine_contiguous_dimsò  s�   € ô �eœUŸY™YÔ'¨E¯I©I´hÔ,?Ø $× 3Ñ 3°EÓ :ÑˆN™Aà!ˆNÜŸ™×)Ñ)×:Ñ:¸5À.ÓQÐQˆ:ÐQØ%/Ñ"ˆI�{Ü˜D×9Ñ9¸)ÀTÓJÈKÓXÐXà×0Ñ0°¸Ó=Ð=rd   c                ó–  — t        |t        j                  t        j                  f«      r|S |j	                  |«      \  }}t        |«      dk  r|S t        j                  j                  j                  ||t        |g||«      «      \  }}}||k(  r|S |j                  |«      }t        |t        t        | ||«      «      «      «      }	|	S )zI
        More aggressive simplification to merge contiguous dims
        r@   )r  r�   r  r§   rö   r  r?   rÏ   rÐ   Ú_simplify_loopsrC   râ   r;   rN  Úzip)
rs   r˜   ra  rî   rï   Ú	new_sizesÚreindexÚ_pruneÚnew_index_varsrÎ  s
             rb   rÓ  z#SIMDKernel._combine_contiguous_dimsÿ  s¹   € ô �eœeŸm™m¬U¯\©\Ð:Ô;ØˆLØ ×/Ñ/°Ó6Ñˆ
�EÜˆu‹:˜Š?ØˆLÜ%&§W¡W×%5Ñ%5×%EÑ%EØ˜Ô7¸¸ÀÈUÓSó&
Ñ"ˆ	�7˜Fð ˜ÒØˆLØŸ™¨	Ó2ˆÜ˜u¤d¬3¨z¹7À>Ó;RÓ+SÓ&TÓUˆ	ØÐrd   c                ó”   ‡ ‡— ‰ j                   d   j                  xs ‰ j                  Št        j                  ˆ ˆfd„«       } |«       S )Néÿÿÿÿc               3  óî   •K  — ‰ j                   j                  «       s‰ j                  rJ ‚d –— y ‰r‰ j                  «        d‰ _        	 d –— ‰r‰ j                  «        d‰ _        y # d‰ _        w xY w­w)NFT)rR  rw   rk  Úcodegen_body)rs   Úshould_flushs   €€rb   Úctxz)SIMDKernel.disable_reduction.<locals>.ctx  sl   øè ø€ à—=‘=×-Ñ-Ô/Ø×0Ò0Ð0Ð0ÛØÙð ×!Ñ!Ô#Ø$)ˆDÔ!ð-ÛÙà×%Ñ%Ô'à(,�Õ%ø¨�Õ%üs   ƒAA5ÁA) Á!A5Á)	A2Á2A5)r^  r›   rm  Ú
contextlibÚcontextmanager)rs   rã  râ  s   ` @rb   Údisable_reductionzSIMDKernel.disable_reduction  sE   ù€ Ø×'Ñ'¨Ñ+×3Ñ3ÒQ°t×7QÑ7Qˆä	×	"Ñ	"ô	-ó 
#ð	-ñ$ ‹uˆrd   c                óÄ   — t        |«      t        | j                  «      k(  sJ ‚t        || j                  «      D ��cg c]  \  }}|j                  |«      ‘Œ c}}S c c}}w r\   )r  r^  rÙ  râ   )rs   rÝ   rh   Úrangess       rb   Ú
set_rangeszSIMDKernel.set_ranges+  s]   € Ü�7‹|œs 4×#3Ñ#3Ó4Ò4Ð4Ð4ô #& g¨t×/?Ñ/?Ó"@÷
á�˜ð ×Ñ˜VÕ$ó
ð 	
ùó 
s   ½Ac                ó   ‡‡‡‡— t        d„ |D «       «      r| D �cg c]  }g ‘Œ c}g fS t        j                  j                  Š| D �cg c]  }g ‘Œ c}Š| D �cg c]  }‰j	                  |«      ‘Œ c}Št        j                  «       Šdˆˆˆˆfd„}	 	 	 	 	 	 dd„}g }d}|D �];  }	g }
|	D �]  }‰j                  |d«      r|
j                  d„ «       Œ)|t        ‰«      k  r>‰j                  ‰|   d«      r)|dz  }|t        ‰«      k  r‰j                  ‰|   d«      rŒ)|dz   t        ‰«      k  rª‰j                  |‰|   ‰|dz      z  «      rŒ‰j                  |‰|   ‰|dz      z  «      st        |‰|   ‰|dz      z  «      ‚‰|   }‰|dz      }t        |||z  «      }|
j                   |||g |||«       ||dz   |«       ||dz   |«      g«      «       �Œ0|dz   t        ‰«      k  rš‰j                  |‰|   «      s‰j                  t        |‰|   «      d«      rf‰j                  |‰|   «      st        |‰|   «      ‚‰|   }t        |‰|   «      }|
j                   ||g |||«       ||dz   |«      g«      «       �ŒÛ|t        ‰«      k\  rt        |d«      ‚|
j                  t        j                   |||«      «      «       �Œ" |j                  |
«       �Œ> t        d„ ‰D «       «      sJ d	‰› d
|› �«       ‚‰|fS c c}w c c}w c c}w )Nc              3  ó8   K  — | ]  }t        |«      d k(  –— Œ y­w©r   N©r  )r  rh   s     rb   r  z5SIMDKernel._split_iteration_ranges.<locals>.<genexpr>9  s   è ø€ Ò6 FŒs�6‹{˜aÕÑ6ùs   ‚c                óÐ   •— ‰j                  |«      }‰j                  ‰|    |«      st        ‰|    |«      ‚t        ‰|    |«      ‰| <   ‰|    j	                  |«       t        ‰«      S r\   )rg  Ústatically_known_multiple_ofÚ	CantSplitr   rÖ   rÓ   )r“  r×   Ú
new_rangesÚ	remainingÚsvÚ	var_counts     €€€€rb   Ú	add_rangez5SIMDKernel._split_iteration_ranges.<locals>.add_rangeA  sf   ø€ Ø—;‘;˜tÓ$ˆDØ×2Ñ2°9¸Q±<ÀÔFÜ 	¨!¡¨dÓ3Ð3ä# I¨a¡L°$Ó7ˆI�a‰LØ�q‰M× Ñ  Ô&Ü˜	“?Ð"rd   c                óP   ‡ ‡— t        ‰«      t        ‰ «      dz   k(  sJ ‚dˆˆ fd„}|S )z„
            Builds the nested expression:
              ((...((s1*v[i1] + v[i2]) * s2 + v[i3]) ... ) * sk + v[i(k+1)])
            r@   c                ó\   •— | ‰d      }t        ‰‰dd  «      D ]  \  }}||z  | |   z   }Œ |S )Nr   r@   )rÙ  )Ú	flat_varsr×   rô   r¯  Úidxsrï   s       €€rb   ÚgetterzISIMDKernel._split_iteration_ranges.<locals>.make_combined.<locals>.getterS  sH   ø€ Ø   a¡Ñ)�Ü! %¨¨a¨b¨Ó2ò 5‘F�A�sØ˜t™8 i°¡nÑ4‘Dð5à�rd   )rø  rÿ   r†   rƒ   rí  )rï   rù  rú  s   `` rb   Úmake_combinedz9SIMDKernel._split_iteration_ranges.<locals>.make_combinedJ  s+   ù€ ô �t“9¤ E£
¨Q¡Ò.Ð.Ð.öð ˆMrd   r   r@   c                ó6   — t         j                  j                  S r\   )r�   r‘   ÚZero)r/  s    rb   rê   z4SIMDKernel._split_iteration_ranges.<locals>.<lambda>a  s   € ´E·G±G·L±L€ rd   r   c              3  ót   K  — | ]0  }t         j                  j                  j                  |«      d k(  –— Œ2 y­w)r@   N)r?   rÏ   rÐ   Úguarding_hint_or_throwr  s     rb   r  z5SIMDKernel._split_iteration_ranges.<locals>.<genexpr>·  s/   è ø€ ò 
Ø@AŒA�G‰G×Ñ×3Ñ3°AÓ6¸!Õ;ñ
ùs   ‚68zfailed to set ranges ú )r“  r÷   r×   rƒ   r†   r÷   )rï   rÿ   rù  z	list[int]r†   z(Callable[[list[sympy.Expr]], sympy.Expr])r  r?   rÏ   rÐ   rg  ri  rj  rÑ   rÖ   r  Ústatically_known_gtrï  rð  r   ÚoperatorÚ
itemgetter)ÚgroupsrÝ   Úgroupr/  Úgrõ  rû  Úreturn_getters_groupsÚcurrent_groupÚlength_groupÚreturn_gettersÚsizeÚsize1Úsize2Úsize3rñ  rò  ró  rô  s                  @@@@rb   Ú_split_iteration_rangesz"SIMDKernel._split_iteration_ranges2  s¯  û€ ô Ñ6¨gÔ6Ô6Ø$*Ö+˜5’BÒ+¨RÐ/Ð/ä�W‰W×ÑˆØ:@Ö-A°QªbÒ-Aˆ
Ø-3Ö4¨�R—[‘[ •^Ò4ˆ	Ü—O‘OÓ%ˆ	÷	#ð 	#ð	Ø#ð	Ø+4ð	à5ó	ð" !#ÐØˆØ#ó X	9ˆLØˆNØ$ó U�Ø×-Ñ-¨d°AÔ6Ø"×)Ñ)Ñ*@ÔAØà#¤c¨)£nÒ4¸×9SÑ9SØ˜mÑ,Øô:ð
 " QÑ&�Mð $¤c¨)£nÒ4¸×9SÑ9SØ˜mÑ,Øõ:ð ! 1Ñ$¤s¨9£~Ò5¸"×:PÑ:PØ˜) MÑ2°Y¸}ÈqÑ?PÑ5QÑQô;ð ×:Ñ:Ø˜i¨Ñ6¸À=ÐSTÑCTÑ9UÑUôô (Ø Ø% mÑ4°yÀÐQRÑARÑ7SÑSóð ð
 & mÑ4�EØ% m°aÑ&7Ñ8�EÜ$ T¨5°5©=Ó9�EØ"×)Ñ)á%Ø" E˜Ná )¨-¸Ó ?Ù )¨-¸!Ñ*;¸UÓ CÙ )¨-¸!Ñ*;¸UÓ Cðóö
ð # QÑ&¬¨Y«Ò7Ø×*Ñ*¨4°¸=Ñ1IÔJð ×*Ñ*¬8°D¸)ÀMÑ:RÓ+SÐUVÔWð ×:Ñ:Ø˜i¨Ñ6ôô (¨¨i¸Ñ.FÓGÐGà% mÑ4�EÜ$ T¨9°]Ñ+CÓD�EØ"×)Ñ)á%Ø"˜Gá )¨-¸Ó ?Ù )¨-¸!Ñ*;¸UÓ Cðóö	ð %¬¨I«Ò6Ü'¨¨aÓ0Ð0Ø"×)Ñ)ä ×+Ñ+©I°mÀTÓ,JÓKöðeUðl "×(Ñ(¨Ö8ðqX	9ôt ñ 
ØENô
ô 
ð 	9à" 9 +¨Q¨w¨iÐ8ó	9ð 
ð Ð0Ð0Ð0ùòC ,ùò .BùÚ4s   ›	LÁ	LÁLc                ó  — t         j                  j                  }t        |d   «      dk(  r\|j	                  |t
        j                  j                  «      s2|j	                  t        |«      t        |d   «      |z  «      r|d   |gfS |S )z1Fill in the reduction numel of lengths if missingr@   r   )	r?   rÏ   rÐ   r  rÑ   r�   r‘   r’   r:   )Úclsr  rÝ   Úreduction_numelrÐ   s        rb   Úprepare_split_iteration_lengthsz*SIMDKernel.prepare_split_iteration_lengths½  s{   € ô —7‘7×#Ñ#ˆÜˆw�q‰z‹?˜aÒØ×0Ñ0°Ä%Ç'Á'Ç+Á+ÔNØ×0Ñ0Ü˜fÓ%Ü˜g a™jÓ)¨OÑ;ôð
 ˜A‘J Ð 1Ð2Ð2àˆrd   c                ón   — | j                  |||«      }	 | j                  ||«       y# t        $ r Y yw xY w©NTF)r  r  rð  )r  r  rÝ   r  s       rb   Úis_compatiblezSIMDKernel.is_compatibleÑ  sB   € ð ×5Ñ5°f¸gÀÓWˆð	Ø×'Ñ'¨°Ô8ØøÜò 	Ùð	ús   •( ¨	4³4c                ó>  — | j                   D �ci c]  }|j                  |j                  “Œ }}| j                  s0|D ]+  }t	        |«      sŒt
        j                  j                  ||<   Œ- g |j                  «       ¢}| j                  ||| j                  «      S c c}w )a5  
        Split and set iteration ranges for the kernel based on the provided lengths.

        This method maps the kernel's tiling structure to the node's iteration space,
        handling both pointwise and reduction dimensions appropriately.

        Args:
            lengths: A sequence of sequences of symbolic expressions representing
                    the sizes of different dimensions for each node.

        Returns:
            A list of lists of symbolic expressions representing the mapped
            iteration variables for each dimension.
        )r^  rp   ro   rk  r8   r�   r‘   r’   rÉ   Úmap_kernel_groups_to_node_sizesré  )rs   rÝ   ÚrtrO  rp   r  s         rb   Úsplit_and_set_rangeszSIMDKernel.split_and_set_rangesà  s�   € ð$ 15×0@Ñ0@ÖA¨"�"—)‘)˜RŸX™XÑ%ÐAˆÐAð ×$Ò$Ø ò 1�Ü& vÕ.Ü%*§W¡W§[¡[�F˜6’Nð1ð
 $�6—=‘=“?Ð#ˆð ×3Ñ3°F¸GÀTÇ_Á_ÓUÐUùò Bs   �Bc           
     óF  — t        |«      t        |«      k(  r!t        d„ t        ||«      D «       «      r ||Ž S | j                  ||«      \  }}g t        j
                  j                   ||Ž «      ¢}|D ��cg c]  }|D �cg c]
  } ||«      ‘Œ c}‘Œ c}}S c c}w c c}}w )a¡  
        We may want to fuse `for i0 in s0*s1` into a tiled kernel with groups (s0, s1).

        To do this we need to split up the iteration space of i0 into something like:
            for i1 in s0:
              for i2 in s1:
                i0 = i1*s1 + i2
                ....

        This function matches and resplits lengths to the groups of
        this kernel to enable tiled + non-tiled fusions.
        c              3  ó’   K  — | ]?  \  }}t         j                  j                  j                  t	        |«      |z
  «      d k(  –— ŒA y­wrì  ©r?   rÏ   rÐ   rg  r:   )r  rX   r  s      rb   r  z=SIMDKernel.map_kernel_groups_to_node_sizes.<locals>.<genexpr>  s@   è ø€ ò /
á��1ô �G‰G×Ñ×%Ñ%¤m°AÓ&6¸Ñ&:Ó;¸qÕ@ñ/
ùs   ‚AA)r  r  rÙ  r  ri  rÉ  rÊ  )	r  r  rÝ   ré  rñ  r  rÞ   ÚfnsÚfns	            rb   r  z*SIMDKernel.map_kernel_groups_to_node_sizes  s™   € ô& ˆw‹<œ3˜v›;Ò&¬3ñ /
ä˜G VÓ,ô/
ô ,
ñ ˜wÐ'Ð'à,/×,GÑ,GÈÐPWÓ,XÑ)ˆ
Ð)ØL”Y—_‘_×2Ñ2±:¸zÐ3JÓKÐLˆØ8M×N°¨Ö, "‘�H•Ô,ÓNÐNùÒ,ùÓNs   Á7	BÂ BÂBÂBc                ó6   — t        |t        j                  «      S r\   )r   r   ÚTMP©rs   r˜   s     rb   Úis_indirect_indexingzSIMDKernel.is_indirect_indexing  s   € ä" 5¬$¯(©(Ó3Ð3rd   c                ó   ‡‡— | j                  |«      ry| j                  «       }dgt        |«      z  }|j                  D ]t  }| j                  j                  |«      Š‰€Œ!t        ‰j                  t        «      sJ ‚t        ˆfd„t        |«      D «       d «      }|€Œ^||xx   ‰j                  z  cc<   Œv t        j                  j                  j                  Št!        ˆfd„t#        |d„ |D «       «      D «       «      S )NFr@   c              3  óF   •K  — | ]  \  }}|‰j                   u sŒ|–— Œ y ­wr\   )r  )r  r“  ra  Úentrys      €rb   r  z,SIMDKernel.is_broadcasted.<locals>.<genexpr>3  s    øè ø€ ÒR‘w�q˜$¸TÀUÇ\Á\Ò=Q”ÑRùs   ƒ!š!c              3  óF   •K  — | ]  \  }} ‰|«       ‰|«      k7  –— Œ y ­wr\   r·   )r  Ú	idx_rangeÚ
iter_rangerg  s      €rb   r  z,SIMDKernel.is_broadcasted.<locals>.<genexpr>@  s,   øè ø€ ò 
á%�	˜:ñ �YÓ¡8¨JÓ#7Õ7ñ
ùó   ƒ!c              3  ó4   K  — | ]  }|j                   –— Œ y ­wr\   )ro   rÈ  s     rb   r  z,SIMDKernel.is_broadcasted.<locals>.<genexpr>C  s   è ø€ ÒC¨d˜tŸz�zÑCùs   ‚)r#  Úactive_range_treesr  rñ   rÕ   rò   r  r  r…   rÓ   r­  rh   r?   rÏ   rÐ   rg  ÚanyrÙ  )rs   r˜   Úactive_treesÚindex_numelsrz   Útree_posr&  rg  s         @@rb   Úis_broadcastedzSIMDKernel.is_broadcasted#  sù   ù€ à×$Ñ$ UÔ+Øà×.Ñ.Ó0ˆØ�sœS Ó.Ñ.ˆØ×(Ñ(ò 	3ˆFØ×)Ñ)×-Ñ-¨fÓ5ˆEØˆ}àÜ˜eŸl™lÔ,?Ô@Ð@Ð@ô ÛR¤)¨LÓ"9ÔRØóˆHð Ðð Ø˜Ó" e§l¡lÑ2Ô"ð#	3ô* —7‘7×#Ñ#×,Ñ,ˆÜó 
ä),ØÑC°lÔCó*ô
ó 
ð 	
rd   c                ó´   — t        |t        «      r)ddj                  t        | j                  |«      «      › d�S | j                  | j                  |«      «      S )a�  
        Convert an index expr to a string that can be used in output code.
        e.g. a sympy expression "s2" may actually appear as "ks1" in the generated kernel.

        Index expressions often need to be passed in as arguments to the triton kernel.
        Rename_indexing and codegen_indexing keep track of the needed indices and add
        new parameters to the function signature.
        ú[r    ú])r  rL  ÚjoinÚmapÚindex_to_strrW  Úrename_indexingr"  s     rb   r7  zSIMDKernel.index_to_strG  sN   € ô �eœTÔ"Ø�t—y‘y¤ T×%6Ñ%6¸Ó!>Ó?Ð@ÀÐBÐBØ�z‰z˜$×.Ñ.¨uÓ5Ó6Ð6rd   c                ó~  — | j                  |«      }t        |t        j                  j                  j
                  «      }t        |j                  t        j                  «      «      s(t        |j                  t        j                  «      «      r3|j                  t        j                  j                  j
                  «      }t        |j                  t        j                  «      «      r‰|j                  t        j                  «      D ]g  }|j                  }t        |«      dkD  sŒt        d„ |D «       «      sŒ1|t        j                  j                  j                  |«      i}t        ||«      }Œi | j                  |«      }i }| j                  «       D ]e  }t        j                  j                  j!                  |j"                  d«      sŒ8|j$                  D ]  }t        j&                  j(                  ||<   Œ! Œg |rt        ||«      }t+        |t,        «      s|n|j.                  d   }| j1                  |«      S )Nr   c              3  óp   K  — | ].  }t        |t        j                  t        j                  f«      –— Œ0 y ­wr\   )r   r   r  ÚPRECOMPUTED_SIZEr  s     rb   r  z.SIMDKernel.prepare_indexing.<locals>.<genexpr>h  s.   è ø€ ò ,àô # 1¤t§y¡y´$×2GÑ2GÐ&H×Iñ,ùs   ‚46r@   )rb  r;   r?   rÏ   rÐ   Úprecomputed_replacementsr  Úatomsr�   ÚfloorÚceilingÚsubsrñ   r  Úlookup_precomputed_sizer,  rÑ   ro   rm   r‘   rý  r  r   r  Úcodegen_indexing)	rs   r˜   Úar!  ÚreplacementsÚ
simp_indexÚsingleton_replacementsra  rz   s	            rb   Úprepare_indexingzSIMDKernel.prepare_indexingT  sÀ  € ð ×&Ñ& uÓ-ˆÜ˜5¤!§'¡'×"2Ñ"2×"KÑ"KÓLˆäˆu�{‰{œ5Ÿ;™;Ó'Ô(¬C°·±¼E¿M¹MÓ0JÔ,KØ—J‘JœqŸw™w×/Ñ/×HÑHÓIˆEô ˆu�{‰{œ5Ÿ=™=Ó)Ô*Ø—[‘[¤§¡Ó/ò 	<�ð Ÿ.™.�Ü�w“< !Ó#¬ñ ,à$ô,õ )ð %&¤q§w¡w×'7Ñ'7×'OÑ'OÐPQÓ'RÐ#S�LÜ& u¨lÓ;‘Eð	<ð ×+Ñ+¨EÓ2ˆ
à!#ÐØ×+Ñ+Ó-ò 	>ˆDÜ—7‘7×#Ñ#×;Ñ;¸D¿J¹JÈÔJØð Ÿ-™-ò >�Ü16·±·±Ð& vÒ.ñ>ð	>ñ "Ü# JÐ0FÓGˆJô )¨´XÔ>‰JÀJÇOÁOÐTUÑDVð 	ð ×$Ñ$ ZÓ0Ð0rd   c                ór   — | j                   D �cg c]  }|j                  r| j                  sŒ|‘Œ c}S c c}w r\   )r^  rw   rk  )rs   Úts     rb   r,  zSIMDKernel.active_range_trees†  s5   € ð ×'Ñ'ö
Ø¨q¯~ª~À×AVÓAVŠAò
ð 	
ùò 
s   �4­4c              #  ó  K  — | j                   }t        |«      | _         | j                  j                  «        	 d–— || _         | j                  j                  «        y# || _         | j                  j                  «        w xY w­w)z;Temporarily codegen against an alternate range-tree family.N)r^  rL  rb  rÊ   )rs   r^  Úsaveds      rb   Úuse_range_treeszSIMDKernel.use_range_trees�  ss   è ø€ ð
 × Ñ ˆÜ Ó,ˆÔð 	×Ñ×*Ñ*Ô,ð	1Ûà$ˆDÔØ×"Ñ"×.Ñ.Õ0øð  %ˆDÔØ×"Ñ"×.Ñ.Õ0üs   ‚7BºA  ¾"BÁ #BÂBc                ó4  — t         j                  j                  j                  || j	                  «       «      }t        |j                  t        ¬«      D ]Á  }|| j                  v sŒi }| j                  |   j                  «       D ].  }t         j                  j                  j                  |«      ||<   Œ0 t        |«      dkD  r5t        | j                  |   j                  |«      | j                  |   _        | j                  |   j                  «        ŒÃ |S )Nrë   r   )r?   rÏ   rÐ   r]  rn   Úsortedrñ   r€   rÕ   r  rA  r  r;   r×   r  )rs   r×   ÚsymrD  Úpss        rb   rB  zSIMDKernel.codegen_indexing�  s÷   € Ü�w‰w×Ñ×4Ñ4°T¸4¿?¹?Ó;LÓMˆÜ˜$×+Ñ+´Ô5ò 	5ˆCØ�d×+Ñ+Ò+ð  "�Ø×/Ñ/°Ñ4×EÑEÓGò T�BÜ'(§w¡w×'7Ñ'7×'OÑ'OÐPRÓ'S�L Ò$ðTä�|Ó$ qÒ(Ü6@Ø×-Ñ-¨cÑ2×7Ñ7Ø$ó7�D×)Ñ)¨#Ñ.Ô3ð ×%Ñ% cÑ*×2Ñ2Õ4ð	5ð ˆrd   c                ó   — t        d«      ‚)NzNYI: codegen_nan_checkr…  rv   s    rb   Úcodegen_nan_checkzSIMDKernel.codegen_nan_check®  s   € Ü!Ð":Ó;Ð;rd   c                ó   — t        d«      ‚)Nz$NYI: iteration_ranges_codegen_headerr…  )rs   r&  Úcodes      rb   Úiteration_ranges_codegen_headerz*SIMDKernel.iteration_ranges_codegen_header±  s   € ô
 "Ð"HÓIÐIrd   c                ó¢   — t         j                  j                  }t        | j                  j
                  «      D ]  }|j                  |«       Œ y r\   )r?   rÏ   Úwrapper_coderÜ   r  Úworkspace_argsÚgenerate_workspace_deallocation)rs   ÚwrapperÚwss      rb   Údeallocate_workspacesz SIMDKernel.deallocate_workspaces¸  s=   € Ü—'‘'×&Ñ&ˆÜ˜4Ÿ9™9×3Ñ3Ó4ò 	8ˆBØ×3Ñ3°BÕ7ñ	8rd   c                ó   — t        d«      ‚)NzNYI: call_kernelr…  )rs   rl   rË   Údeallocate_wss       rb   Úcall_kernelzSIMDKernel.call_kernel½  s   € ô "Ð"4Ó5Ð5rd   c              #  óþ   K  — | j                   }| j                  }|rt        j                  ||«      }t	        j
                  |«      }|| _         || _        	 |–— || _         || _        y# || _         || _        w xY w­w)z:Context manager to add an additional mask to tl.load/storeN)Ú
_load_maskÚ_load_otherr=   Úlogical_andr>   Ú_unwrap)rs   r±   rA  rÂ  Ú	prior_vals        rb   Ú
mask_loadszSIMDKernel.mask_loadsÂ  sy   è ø€ ð —‘ˆØ×$Ñ$ˆ	ÙÜ—?‘? 4¨Ó/ˆDä×!Ñ! $Ó'ˆØˆŒØ ˆÔð	)àŠJà#ˆDŒOØ(ˆDÕøð $ˆDŒOØ(ˆDÕüs   ‚AA=ÁA* ÁA=Á*A:Á:A=c                ó(  — | j                   j                  «       D ��ci c]  \  }}||j                  “Œ }}}t        ||«      }i }| j                  D ]7  }t        |j                  «      }t        ||di«      t        ||di«      z
  ||<   Œ9 |S c c}}w )a\  
        This gets the stride of the index for each of the tiling variables
        (technically, it does it at index 0)

        For example, if
        xindex = x0 + 512*x1 + 1024*r0
        x0 = (xindex//512)
        x1 = (xindex % 512)
        r0 = rindex // 1024

        this function would return
        {xindex: 512, rindex: 1024}
        r@   r   )rÕ   r}   r×   r;   r^  r9   rl   )	rs   r˜   ÚkÚvÚindex_to_tile_indexesÚindex_in_tile_varsÚstridesÚ
range_treerô   s	            rb   Úget_strides_of_loadzSIMDKernel.get_strides_of_loadÔ  s¡   € ð 8<×7LÑ7L×7RÑ7RÓ7T× U©t¨q°!  A§F¡F¡Ð UÐÑ UÜ'¨Ð/DÓEÐØˆØ×*Ñ*ò 	ˆJÜ" :§?¡?Ó3ˆAÜ#Ð$6¸¸A¸Ó?Ä*Ø" Q¨ FóCñ ˆG�AŠJð	ð
 ˆùó !Vs   žBc                ó\   — t        |t        «      rt        t        | |«      «      S  | |«      S r\   )r  r¹   r6  )r  rA  s     rb   Ú_map_tuple_or_scalarzSIMDKernel._map_tuple_or_scalarì  s'   € ä�eœUÔ#Üœ˜R ›Ó(Ð(Ù�%‹yÐrd   c                ó¾   — t        j                  | j                  j                  «      D �cg c]  }|j	                  «       ‘Œ }}t        t        d |«      «      S c c}w r\   )rK   Ú
only_nodesrR  rM  Úestimate_flopsr�  Úfilter)rs   rË   Úflopss      rb   rs  zSIMDKernel.estimate_flopsò  sX   € ô +×5Ñ5°d·m±m×6QÑ6QÓRö
àð ×ÑÕ!ð
ˆð 
ô ”6˜$ Ó&Ó'Ð'ùò	
s   ¬Ac           	     ó  — g }t        t        | j                  j                  j	                  «       «      «      }| j                  j                  «       \  }}}}| j                  j                  «       }t        j                  j                  j                  t        | j                  j	                  «       «      «      }t        |«      D �]2  \  }}||vr|j                  d«       Œt        j                  j!                  |«      }	t        j                  j                  j                  |	«      }
|
|kD  rwt#        t$           «       }d}||   D ]M  }t'        |t(        t*        f«      r|j-                  d|› �«       |dz  }Œ3|j-                  |j.                  «       ŒO t        |«      |z  }n|
}t        j                  j1                  |«      }t3        |«      }|j                  ||z  dt5        ||k  «      z   z  «       �Œ5 t7        |«      S )a+  
        Try the best to estimate the total size (in bytes) of the
        kernel's inputs and outputs, which is used for estimating the memory
        throughput of this kernel. This information is used for checking how
        far we are from the peak memory bandwidth. It's important that
        we want to avoid overestimating the sizes of the inputs and outputs,
        because it can wrongfully give us a very large memory traffic value,
        which may be even larger than the theoretical bandwidth and thus
        become very misleading. This is particularly problematic for cases
        where we slice some inputs. In those cases, we should only count
        the size of the "slices" instead of the original inputs, because
        only the slices contribute to the real memory traffic.
        r   Úno_index_dep_r@   )r  r<   r  Úinplace_buffersrÉ   Úpython_argdefsrR  Úbuf_accessesr?   rÏ   rÐ   rå   r:   rh  r­  rÖ   Ú	get_numelr   r   r  r!   r"   rð   r˜   Ú	get_dtyper5   r÷   r�  )rs   ÚnbytesÚninplace_argsr/  Ú	call_argsrz  Ú	out_numelr“  r   Ú	arg_numelÚbuf_sizer¾  Úno_index_dep_countÚdepro   r   Ú
dtype_sizes                    rb   Úestimate_kernel_num_bytesz$SIMDKernel.estimate_kernel_num_bytesù  s¿  € ð ˆÜœF 4§9¡9×#<Ñ#<×#CÑ#CÓ#EÓFÓGˆØ!ŸY™Y×5Ñ5Ó7Ñˆˆ9�a˜Ø—}‘}×1Ñ1Ó3ˆô —G‘G×$Ñ$×6Ñ6Ü˜$Ÿ+™+×,Ñ,Ó.Ó/ó
ˆ	ô   	Ó*ó 	M‰FˆAˆsð ˜,Ñ&Ø—‘˜aÔ ØÜŸ™×)Ñ)¨#Ó.ˆIÜ—w‘w×'Ñ'×9Ñ9¸)ÓDˆHØ˜)Ò#ô %¤S™/Ó+�Ø%&Ð"Ø'¨Ñ,ò /�CÜ! #¬´Ð'9Ô:ØŸ™ mÐ4FÐ3GÐ$HÔIØ*¨aÑ/Ñ*àŸ™ C§I¡IÕ.ð/ô ˜G› yÑ0‘à �Ü—G‘G×%Ñ% cÓ*ˆEÜ'¨Ó.ˆJà�M‰M˜% *Ñ,°´C¸¸MÑ8IÓ4JÑ0JÑKÖLð;	Mô< �6‹{Ðrd   c           	     ó&  — t        | j                  j                  «      dk(  rEt        | j                  j                  «      dk(  r#t        | j                  j                  «      dk(  ry| j                  j                  «       \  }}}}d}|D �]F  }t        j                  j                  |«      }|sŒ&|j                  «       }	t        |	j                  «      dk(  sŒOt        |	j                  D �
cg c]
  }
|
dk(  sŒ	|
‘Œ c}
«      dk(  rŒ|t        j                  |	j                  «      }|€|}Œ ||k7  sŒ¦t        d|› d�d|› d	|› �z   «      }t        j!                  |«       |D �cg c]m  }t        j                  j                  |«      rJt        j                  t        j                  j#                  |«      j                  «       j                  «      nd‘Œo }}|D �cg c]Z  }t        j                  j                  |«      r7t        j                  j#                  |«      j                  «       j                  nd‘Œ\ }}|D �cg c]@  }|t        j                  j$                  v rd
n|t        j                  j&                  v rdnd‘ŒB }}|D �
cg c]  }
|
j(                  ‘Œ }}
t        d|› d|› d|› �d|› d|› d�z   «      }t        j!                  |«        y t+        d|› d�«      }t        j!                  |«       yc c}
w c c}w c c}w c c}w c c}
w )zr
        Print message if the kernel have mixed layout inputs.
        Only care about 4D tensor for now.
        r@   r   Né   r   zExpected stride order z, but found stride orderr   z for kernel Ú
GraphInputÚIntermediateBufferz  param names z
  buf names z
  strides z	
  sizes z
  sources ú
z%All the inputs for the triton kernel z have uniform layout)r  r  Úinput_buffersÚoutput_buffersrx  ry  r?   rÏ   Útry_get_bufferÚ
get_layoutr  r   Úget_stride_orderÚstrider.   ÚlogÚwarningÚ
get_bufferÚgraph_inputsÚname_to_bufferrl   r+   )rs   rY  Úargdefsr  Ú
_signaturer/  Úuniform_stride_orderÚarg_nameÚbufÚlayoutrX   Ústride_orderÚmsgrl   Ústride_order_listÚ	size_listÚsource_listÚargdef_namess                     rb   Úwarn_mix_layoutzSIMDKernel.warn_mix_layout;  s  € ô �—	‘	×'Ñ'Ó(¨AÒ-Ü�D—I‘I×,Ñ,Ó-°Ò2Ü�D—I‘I×-Ñ-Ó.°!Ò3ð
 à,0¯I©I×,DÑ,DÓ,FÑ)ˆ�˜J¨Ø#Ðà!ó 0	ˆHÜ—'‘'×(Ñ(¨Ó2ˆCÙØØ—^‘^Ó%ˆFÜ�6—;‘;Ó 1Ó$ä 6§;¡;Ö9˜a°!°q³&šÒ9Ó:¸aÒ?ØÜ!×2Ñ2°6·=±=ÓA�Ø'Ð/Ø+7Ñ(Ø)¨\Ó9Ü%Ø0Ð1EÐ0FÐF^Ð_Ø˜l˜^¨<¸°}ÐEñFó�Cô —K‘K Ô$ð %.ö)ð !ô Ÿ7™7×1Ñ1°$Ô7ô ×+Ñ+ÜŸG™G×.Ñ.¨tÓ4×?Ñ?ÓA×HÑHôð "ñ	"ð)Ð%ð )ð %.ö	!ð !ô Ÿ7™7×1Ñ1°$Ô7ô Ÿ™×*Ñ*¨4Ó0×;Ñ;Ó=×BÒBà!ñ"ð!�Ið !ð %.ö#ð !ð	  ¤1§7¡7×#7Ñ#7Ñ7ñ %ð  ¤1§7¡7×#9Ñ#9Ñ9ñ 2à!ñ	"ð#�Kð #ð 5<Ö#<¨q A§F£FÐ#<�LÐ#<Ü%Ø(¨¨°nÀYÀKÈ|Ð\mÐ[nÐoØ& y k°¸k¸]È"ÐMñNó�Cô —K‘K Ô$Ùða0	ôb Ø3°K°=Ð@TÐUó
ˆô 	�‰�CÕùò[ :ùò)ùò!ùò#ùò $=s'   Ã-
K:
Ã8K:
Å"A2K?ÇALÈ?AL	Ê
Lc                ó†  — t        j                  ||d|«      }d| _        t        j                  | j                  j
                  |«      }t        j                  ||«      }d| _        t        j                  ||«      }t        j                  ||«      }t        j                  ||d|«      }t        j                  |||f«      S )Nr�  FT)r=   Ú	reductionrk  Ú
index_exprrR  r  ÚtruedivÚsubÚmulr>   rd  )	rs   r   rA  Úsum_rQ  ÚmeanÚdxÚdx2Úm2s	            rb   Úwelford_reduce_fallbackz"SIMDKernel.welford_reduce_fallbackƒ  s›   € Ü�}‰}˜U E¨5°%Ó8ˆØ %ˆÔÜ—‘ §¡× =Ñ =¸uÓEˆÜ�{‰{˜4 Ó(ˆà $ˆÔÜ�W‰W�U˜DÓ!ˆÜ�g‰g�b˜"‹oˆÜ�]‰]˜5 %¨°Ó4ˆÜ×!Ñ! 4¨¨VÐ"4Ó5Ð5rd   c                óæ   — t        j                  ||d|«      }t        j                  ||«      }t        j                  |«      }t        j                  ||d|«      }t	        j
                  ||f«      S )NÚmaxr�  )r=   r¥  r¨  Úexpr>   rd  )rs   r   rA  Úvmaxr¨  r²  Úvsums          rb   Ú prepare_softmax_twopass_fallbackz+SIMDKernel.prepare_softmax_twopass_fallback�  s\   € Ü�}‰}˜U E¨5°%Ó8ˆÜ�g‰g�e˜TÓ"ˆÜ�g‰g�c‹lˆÜ�}‰}˜U E¨5°#Ó6ˆÜ×!Ñ! 4¨ ,Ó/Ð/rd   c                ó   — t         ‚r\   r…  rv   s    rb   Úcodegen_kernelzSIMDKernel.codegen_kernel–  r¢  rd   c                 ó   — y r\   r·   rv   s    rb   rá  zSIMDKernel.codegen_body™  ó   € Ørd   c                 ó   — y r\   r·   )rs   r&  s     rb   r  z)SIMDKernel.codegen_iteration_ranges_entryœ  r¹  rd   )NNNNF)rO  údict[str, sympy.Expr]rR  rL   rš   rø   r�  úbool | Noner‚  r¼  rP  údict[str, sympy.Expr] | Nonerp  r‰   r†   r‡   rú   )r†   ú	list[Any])r“  r÷   r†   r€   r(  )r   útorch.dtyper†   r€   )r†   r¿  rˆ   )rš   rø   rk  r‰   rw   r‰   rh  r»  rr  r‰   r†   úlist[IterationRangesRoot])rš   zdict[str, str]r†   r‡   )r¾  úSequence[sympy.Expr]r†   r‡   ©rl   r€   r˜   rƒ   rA  rB   r†   r‡   )r†   r‚   ©r˜   rƒ   r†   rƒ   )r˜   rƒ   ra  r…   r†   rƒ   )r†   z'contextlib.AbstractContextManager[None])rÝ   rƒ   r†   r�   )r  úIterable[sympy.Expr]rÝ   úSequence[Sequence[sympy.Expr]]r†   zStuple[list[list[sympy.Expr]], list[list[Callable[[list[sympy.Expr]], sympy.Expr]]]])r  rÄ  rÝ   rÅ  r  rƒ   r†   rÅ  )r  rÄ  rÝ   rÅ  r  rƒ   r†   r‰   )rÝ   rÅ  r†   úlist[list[sympy.Expr]])r  rÁ  rÝ   rÅ  r†   rÆ  )r˜   rƒ   r†   r‰   )r˜   rƒ   r†   r€   )r†   rÀ  )r^  zSequence[IterationRangesRoot]r†   úIterator[None])r×   rƒ   r†   rƒ   rþ   )r&  r…   rT  r6   r†   r‡   r¾   )rl   r€   rË   zIRNode | Noner^  r‰   r†   r‡   )r±   zstr | OpsWrapperrA  úint | floatr†   zIterator[str])r˜   rƒ   r†   r‚   )r†   rù   )r&  rÒ   )CrŒ   r�   rŽ   r�   ÚpexprrV  rI  rX  rk   rŽ  r�  r”  r™  r“   r2   rž  r¡  r¤  r¦  rq  rº  rz  r¿  rÃ  rl  rn  rn   r`  r_  rÓ  ræ  ré  Ústaticmethodr  Úclassmethodr�   r‘   r’   r  r  r  r  r#  r1  r7  rG  r,  rä  rå  rL  rB  rR  rU  r\  r_  rf  rn  rp  rs  r†  r£  r¯  rµ  r·  rá  r  r”   r•   s   @rb   r„   r„   ÿ  sÇ  ø… ñð */€EÐ&Ó.Ø&Ó&Ø!€O�TÓ!àÓð ,0Ø59Ø6:Ø6:Ø$)ðD?à%ðD?ð %ðD?ð )ð	D?ð
 (3ðD?ð )4ðD?ð 4ðD?ð "ðD?ð 
õD?ðL"ð 
ó"ó$
ó%òð
 ØòJó ó ðJó"ó!ð òHó ðHóð5à(ð5ð ð5ð ð	5ð
 &ð5ð ð5ð 
#ó5ón-óó*óóó
ó
ð>Øð>Ø':ð>à	ó>ðØðØ':ðà	óó(ó0
ð ðH1Ø$ðH1Ø/MðH1ð
òH1ó ðH1ðT ð
 ',§g¡g§k¡kð	à$ðð 0ðð $ð	ð
 
(òó ðð& ð
 ',§g¡g§k¡kð	à$ðð 0ðð $ð	ð
 
òó ðð VØ5ð Và	ó VðD ðOà$ðOð 0ðOð
 
 òOó ðOó84ó"
óH7ð01àð01ð 
ó01ód
ð ×Ñð1Ø8ð1à	ò1ó ð1óó"<ðJà"ðJð ðJð 
ó	Jò8ð LPð6Øð6Ø,ð6ØDHð6à	ó6ð
 ×Ñò)ó ð)ó"ð0 ñó ðó
(ò@òDFòP
6ò0ò"ò÷rd   r„   T)Úfrozenc                  ó&   — e Zd ZU dZded<   ded<   y)Ú_IterationSpacea…  Source domain used to remap a pointwise body.

    ``groups`` are source-domain extents. ``values`` are the matching
    codegen-side symbolic index expressions in the outer kernel.

    For a grouped reduction body with logical ranges [B, D // G, G],
    ``groups`` may be [B, D // G, G] while ``values`` are the concrete
    symbols the parent kernel is using for those coordinates.
    rÁ  r  rÉ   NrT  r·   rd   rb   rÎ  rÎ     s   … ñð !Ó Ø Ô rd   rÎ  c                  ó¾   — e Zd ZU dZded<    ej                  e¬«      Zded<   dZ	ded	<   dd
„Z
dd„Zej                  dd„«       Zdd„Zej                  dd„«       Zy)Ú_DerivedIterationFamilyaP  Iteration family for a nested-reduction consumer stage.

    Two configurations:
      - reduced-output: swaps in a derived grouped-axis tree; ``index_subs``
        rewrites body vars to the active tree symbols
      - parent-full: reuses outer trees; store_cache values are broadcast-lifted
        lazily through CSE when needed
    ztuple[IterationRangesRoot, ...]r^  )Údefault_factoryr‚   Ú
index_subsFr‰   Ú_headers_emittedc                ó8   — |j                  | j                  «      S r\   )r@  rÒ  r"  s     rb   Úremap_indexz#_DerivedIterationFamily.remap_indexÁ  s   € Ø�z‰z˜$Ÿ/™/Ó*Ð*rd   c                óº   — t        |j                  «      t        | j                  «      k7  ryt        d„ t        |j                  | j                  «      D «       «      S )NFc              3  ó*   K  — | ]  \  }}||u –— Œ y ­wr\   r·   )r  ÚactiveÚexpecteds      rb   r  z7_DerivedIterationFamily.is_active_on.<locals>.<genexpr>Ç  s!   è ø€ ò 
á �˜ð �hÔñ
ùs   ‚)r  r^  r  rÙ  ©rs   rq   s     rb   Úis_active_onz$_DerivedIterationFamily.is_active_onÄ  sP   € Üˆv×!Ñ!Ó"¤c¨$×*:Ñ*:Ó&;Ò;ØÜñ 
ä$'¨×(:Ñ(:¸D×<LÑ<LÓ$Mô
ó 
ð 	
rd   c              #  óŒ   K  — | j                  |«      rd–— y| j                  |«      5  d–— ddd«       y# 1 sw Y   yxY w­w)z>Activate this family if it isn't already active on ``kernel``.N)rÛ  ÚactivaterÚ  s     rb   Úensure_activez%_DerivedIterationFamily.ensure_activeÌ  s<   è ø€ ð ×Ñ˜VÔ$Üà—‘˜vÓ&ñ Û÷÷ ñ üs   ‚(Aª8¯	A¸A½Ac                óü   — | j                   ry | j                  D ]Y  }t        |t        «      sŒ|j                  xr |j
                  }|r|j                  n|j                  }|j                  ||«       Œ[ d| _         y r¾   )	rÓ  r^  r  r+  r›   rk  rf  re  rU  )rs   rq   ra  Úemit_in_loop_bodyÚtargets        rb   Úensure_headersz&_DerivedIterationFamily.ensure_headersÕ  sv   € ð
 × Ò ØØ×$Ñ$ò 	EˆDÜ˜$Ô :Õ;ð %)§L¡LÒ$L°V×5LÑ5LÐ!Ù1B˜×-Ò-ÈÏÉ�Ø×6Ñ6°t¸VÕDð	Eð !%ˆÕrd   c              #  ó–   K  — | j                  |«       |j                  | j                  «      5  d –— d d d «       y # 1 sw Y   y xY w­wr\   )râ  rL  r^  rÚ  s     rb   rÝ  z _DerivedIterationFamily.activateæ  sA   è ø€ ð 	×Ñ˜FÔ#Ø×#Ñ# D×$4Ñ$4Ó5ñ 	Û÷	÷ 	ñ 	üs   ‚-A	¯=´	A	½AÁA	NrÃ  )rq   úSIMDKernel[Any]r†   r‰   )rq   rä  )rq   rä  r†   r‡   )rq   rä  r†   rÇ  )rŒ   r�   rŽ   r�   rI  ÚdataclassesÚfieldrN  rÒ  rÓ  rÕ  rÛ  rä  rå  rÞ  râ  rÝ  r·   rd   rb   rÐ  rÐ  °  sv   … ñð 1Ó0à1B°×1BÑ1BÐSWÔ1X€JÐ.ÓXØ"Ð�dÓ"ó+ó
ð ×Ñòó ðó%ð" ×Ñòó ñrd   rÐ  c                  óN   — e Zd ZU dZded<   ded<   ded<   ded<   d	ed
<   d	ed<   y)Ú_GroupedReductionVarsaò  Variables used to call the grouped-reduction body.

    ``iter_remapped`` and ``reduce_remapped`` are the iteration/reduction
    arguments passed to the grouped reduction's loop body. ``passthrough`` is
    the non-grouped axis, while ``group_index`` and ``local_reduction`` are
    the two pieces of the grouped parent axis after it is split into
    [num_groups, local_reduction_size]. ``group_index_var`` is None only for
    the single-group case, where the group index is the constant 0.
    rÿ   Úiter_remappedÚreduce_remappedr‹   Úpassthrough_iter_varzsympy.Symbol | NoneÚgroup_index_varrƒ   Úgroup_index_exprÚlocal_reduction_varNrT  r·   rd   rb   rè  rè  ï  s-   … ñð $Ó#Ø%Ó%Ø&Ó&Ø(Ó(Ø Ó Ø#Ô#rd   rè  c                  óÔ  — e Zd ZU dZded<   ded<   ded<   ded<   e	 	 	 	 	 	 	 	 d d	„«       Zed!d
„«       Zed!d„«       Z	ed"d„«       Z
ed#d„«       Zed$d„«       Zed$d„«       Zed#d„«       Zed#d„«       Zed%d„«       Zed#d„«       Zed#d„«       Zed&d„«       Zed"d„«       Zed'd„«       Z	 	 	 	 d(d„Z	 	 	 	 d)d„Z	 	 d*d„Z	 	 	 	 d+d„Z	 	 	 	 	 	 d,d„Z	 	 	 	 	 	 d,d„Z	 	 	 	 	 	 	 	 d-d„Zy).Ú_GroupedReductionLayouta  Geometry of the grouped reduction (the consumer reduction in a
    nested-reduction pair). Describes how the outer kernel's tile is
    reshaped into [num_groups, local_reduction_size], which axis is reduced,
    and how reduced values are lifted back for epilogues.
    r…   Úx_treeÚr_treeúsympy.IntegerÚlocal_reduction_sizer‰   Úlocal_reduction_in_rc                óŽ   — t        |j                  «      dk(  sJ ‚|j                  \  }}|j                  dk(  sJ ‚ | ||||¬«      S )Nr   rX   )rñ  rò  rô  rõ  )r  r^  rp   )r  rq   rô  rõ  rñ  rò  s         rb   Úfrom_kernelz#_GroupedReductionLayout.from_kernel  sW   € ô �6×%Ñ%Ó&¨!Ò+Ð+Ð+Ø×+Ñ+‰ˆ�Ø�}‰} Ò#Ð#Ð#ÙØØØ!5Ø!5ô	
ð 	
rd   c                óJ   — | j                   r| j                  S | j                  S )z<The tree decomposed into [num_groups, local_reduction_size].)rõ  rò  rñ  rv   s    rb   Ú
group_treez"_GroupedReductionLayout.group_tree!  ó   € ð #×7Ò7ˆt�{‰{ÐH¸T¿[¹[ÐHrd   c                óJ   — | j                   r| j                  S | j                  S )z5The tree that passes through unchanged (not grouped).)rõ  rñ  rò  rv   s    rb   Úpassthrough_treez(_GroupedReductionLayout.passthrough_tree&  rú  rd   c                ó"   — | j                   rdS dS )Nr@   r   ©rõ  rv   s    rb   Úparent_axisz#_GroupedReductionLayout.parent_axis+  ó   € ð ×-Ò-ˆqÐ4°1Ð4rd   c                ón   — d| j                   j                  j                  «       j                  d«      z   S )NÚnested_r/  )rù  rp   r¨   Úremovesuffixrv   s    rb   Úgroup_prefixz$_GroupedReductionLayout.group_prefix0  s+   € à˜4Ÿ?™?×1Ñ1×7Ñ7Ó9×FÑFÀsÓKÑKÐKrd   c                óL   — t        j                  | j                  › d�dd¬«      S )NÚ_LOCAL_REDUCTION_SIZETr¤   ©r�   r§   r  rv   s    rb   Úlocal_reduction_size_symz0_GroupedReductionLayout.local_reduction_size_sym4  s+   € ä�|‰|Ø× Ñ Ð!Ð!6Ð7ØØô
ð 	
rd   c                óL   — t        j                  | j                  › d�dd¬«      S )NÚ_REDUCED_BLOCKTr¤   r  rv   s    rb   Úreduced_block_symz)_GroupedReductionLayout.reduced_block_sym<  s*   € ä�|‰|Ø× Ñ Ð! Ð0ØØô
ð 	
rd   c                ó6   — | j                   j                  «       S r\   )rù  r«   rv   s    rb   Úparent_blockz$_GroupedReductionLayout.parent_blockD  s   € à�‰×-Ñ-Ó/Ð/rd   c                ó6   — | j                   j                  «       S r\   )rü  r«   rv   s    rb   Úpassthrough_blockz)_GroupedReductionLayout.passthrough_blockH  s   € à×$Ñ$×3Ñ3Ó5Ð5rd   c                óV   — t        | j                  j                  | j                  «      S r\   )r   rù  ro   rô  rv   s    rb   Ú
num_groupsz"_GroupedReductionLayout.num_groupsL  s   € ä˜Ÿ™×-Ñ-¨t×/HÑ/HÓIÐIrd   c                ó,   — t        | j                  «      S r\   )r€   r  rv   s    rb   Únum_groups_strz&_GroupedReductionLayout.num_groups_strP  s   € ä�4×)Ñ)Ó*Ð*rd   c                ó,   — t        | j                  «      S r\   )r€   r  rv   s    rb   Úlocal_reduction_size_dimz0_GroupedReductionLayout.local_reduction_size_dimT  s   € ä�4×0Ñ0Ó1Ð1rd   c                ó¦   — | j                   r#| j                  | j                  | j                  fS | j                  | j                  | j                  fS )z,Shape used before the local group reduction.)rõ  r  r  r  rv   s    rb   Úreshape_shapez%_GroupedReductionLayout.reshape_shapeX  sY   € ð ×$Ò$à×&Ñ&Ø×#Ñ#Ø×-Ñ-ðð ð ×ÑØ×)Ñ)Ø×"Ñ"ð
ð 	
rd   c                ó"   — | j                   rdS dS )z@Axis reduced in [passthrough, num_groups, local_reduction_size].r   r@   rþ  rv   s    rb   Úreduce_axisz#_GroupedReductionLayout.reduce_axisg  r   rd   c                óz   — | j                   r| j                  | j                  fS | j                  | j                  fS r\   )rõ  r  r  rv   s    rb   Úoutput_shapez$_GroupedReductionLayout.output_shapel  s9   € à×$Ò$Ø×*Ñ*¨D×,?Ñ,?Ð@Ð@Ø×#Ñ# T×%;Ñ%;Ð<Ð<rd   c                ó  — t         j                  j                  j                  | j                  d«      rEt
        j                  j                  }d }| j                  j                  «       j                  «       }n6| j                  j                  | j                  | j                  g«      \  }}|}| j                  j                  | j                  j                  g«      d   }|g}t        |j                   «      dk(  r| j"                  r||gn||g}n_t        |j                   «      dk(  r<| j"                  r|| j                  z  |z   n|| j                  j                  z  |z   g}nt%        d«      ‚t'        ||||||«      S )Nr@   r   r   z1nested grouped reduction expects 1 or 2 iter vars)r?   rÏ   rÐ   rÑ   r  r�   r‘   rý  rù  rÚ   rz   râ   rô  rü  ro   r  Ú	iter_varsrõ  ÚAssertionErrorrè  )rs   re  rí  rì  rî  Úpassthrough_varrê  ré  s           rb   Úconstruct_group_reduction_varsz6_GroupedReductionLayout.construct_group_reduction_varsr  sv  € ô �7‰7×Ñ×3Ñ3°D·O±OÀQÔGô  %Ÿw™wŸ|™|ÐØ"ˆOØ"&§/¡/×"<Ñ"<Ó">×"EÑ"EÓ"GÑà37·?±?×3LÑ3LØ—‘ $×";Ñ";Ð<ó4Ñ0ˆOÐ0ð  /Ðð ×/Ñ/×9Ñ9Ø×"Ñ"×(Ñ(Ð)ó
à
ñˆð /Ð/ˆÜˆt�~‰~Ó !Ò#ð ×,Ò,ð !Ð"2Ñ3à&¨Ð8ñ ô
 �—‘Ó  AÒ%ð ×,Ò,ð   $§/¡/Ñ1Ð4DÒDà%¨×(=Ñ(=×(CÑ(CÑCÀoÑUð‰Mô !Ð!TÓUÐUä$ØØØØØØó
ð 	
rd   c                ó~  ‡ — dˆ fd„} |‰ j                   «      } |‰ j                  «      }i }‰ j                  r|n|}‰ j                  r|n|}|j                  «       j	                  «       ||j
                  <   |j                  �+|j                  «       j	                  «       ||j                  <   t        |||f¬«      S )Nc           	     óp  •— | ‰j                   ur| S ‰j                  ‰j                  df}‰j                  t	        | j                  «       ‰j                  «      df}t        | t	        | j                  ‰j                  «      ‰j                  t	        | j                  «       ‰j                  «      ||f¬«      S )NT)ro   r©   r¯   rÇ   )	rù  r  rô  r  r   r©   r+  ro   r¯   )ra  Úlocal_reduction_size_constÚreduced_block_constrs   s      €rb   ÚbuildzA_GroupedReductionLayout.make_reduced_output_family.<locals>.build§  s¶   ø€ Ø˜4Ÿ?™?Ñ*ð �à×-Ñ-Ø×)Ñ)Øð*Ð&ð ×&Ñ&Ü˜Ÿ™Ó*¨D×,IÑ,IÓJØð#Ðô
 .ØÜ˜tŸz™z¨4×+DÑ+DÓEØ×1Ñ1Ü%Ø×%Ñ%Ó'¨×)FÑ)Fóð "<Ð=PÐ Qôð rd   )rÒ  r^  )ra  r…   r†   r…   )rñ  rò  rõ  rÚ   rz   rë  rì  rÐ  )rs   Úgroup_reduction_varsr%  Úreduced_x_treeÚreduced_r_treerÒ  Úreduced_passthrough_treeÚreduced_group_trees   `       rb   Úmake_reduced_output_familyz2_GroupedReductionLayout.make_reduced_output_family£  sÉ   ø€ õ	ñ4 ˜tŸ{™{Ó+ˆÙ˜tŸ{™{Ó+ˆØ57ˆ
à"×7Ò7‰N¸^ð 	!ð #×7Ò7‰N¸^ð 	ð %×/Ñ/Ó1×8Ñ8Ó:ð 	Ð'×<Ñ<Ñ=ð  ×/Ñ/Ð;ð #×-Ñ-Ó/×6Ñ6Ó8ð Ð+×;Ñ;Ñ<ô 'Ø!Ø'¨Ð8ô
ð 	
rd   c                óF   — t        | j                  | j                  f¬«      S )N)r^  )rÐ  rñ  rò  rv   s    rb   Úmake_parent_full_familyz/_GroupedReductionLayout.make_parent_full_familyØ  s!   € ô 'ØŸ™ d§k¡kÐ2ô
ð 	
rd   c                óÚ  — | j                   s\| j                  | j                  | j                  j                  g}|j
                  |j                  |j                  g}t        ||«      S t        | j                  j                  | j                  j                  g| j                  j                  «       j                  «       | j                  j                  «       j                  «       g«      S r\   )rõ  r  rô  rü  ro   rí  rî  rë  rÎ  rñ  rò  rÚ   rz   )rs   r&  Úsource_groupsÚsource_valuess       rb   Úparent_full_iteration_valuesz4_GroupedReductionLayout.parent_full_iteration_valuesß  sË   € ð ×(Ò(à—‘Ø×)Ñ)Ø×%Ñ%×+Ñ+ðˆMð %×5Ñ5Ø$×8Ñ8Ø$×9Ñ9ðˆMô
 #ØØóð ô Ø�[‰[×Ñ §¡× 1Ñ 1Ð2Ø�[‰[×#Ñ#Ó%×,Ñ,Ó.°·±×0FÑ0FÓ0H×0OÑ0OÓ0QÐRó
ð 	
rd   c                ó’   — |j                   €J ‚|j                  �t        |j                  «      dk  r|S | j                  ||d¬«      S )Nr   F©Úmaterialize_singleton©r   r»   r  Ú%_broadcast_value_to_parent_resolution©rs   rq   rA  s      rb   Ú*maybe_broadcast_value_to_parent_resolutionzB_GroupedReductionLayout.maybe_broadcast_value_to_parent_resolutionö  sV   € ð
 �{‰{Ð&Ð&Ð&Ø�;‰;Ð¤# e§k¡kÓ"2°QÒ"6ð ˆLà×9Ñ9ØØØ"'ð :ó 
ð 	
rd   c                óœ   — |j                   €J ‚|j                  �t        |j                  «      dk\  sJ d«       ‚| j                  ||d¬«      S )Nr   z;grouped reduction input must have a known parent-tile shapeTr3  r5  r7  s      rb   Úensure_parent_tile_resolutionz5_GroupedReductionLayout.ensure_parent_tile_resolution  sa   € ð
 �{‰{Ð&Ð&Ð&Ø�{‰{Ð&¬3¨u¯{©{Ó+;¸qÒ+@ð 	
ØIó	
Ð@ð ×9Ñ9ØØØ"&ð :ó 
ð 	
rd   c               ó\  — |j                   €J ‚|j                  �t        |j                  «      dk\  sJ ‚t        |j                  | j                     «      }|| j
                  k(  s|dk(  r|s|S | j                  }| j
                  }|dk(  r&| j                  r|df}||f}||f}	nfd|f}||f}||f}	nY| j                  r'| j                  }
| j                  }||
df}||
|f}||f}	n&| j                  }
| j                  }|
d|f}|
||f}||f}	|j                  ||||	|j                   |	¬«      }|S )Nr   r¸   r@   )rA  Úpre_broadcast_shapeÚbroadcast_shapeÚfinal_shaper   Ú	out_shape)r   r»   r  r€   rÿ  r  r  rõ  r  r  Úemit_broadcast_via_reshape)rs   rq   rA  r4  Ú
parent_dimÚpassthrough_extentÚparent_extentr<  r=  r>  r  rô  s               rb   r6  z=_GroupedReductionLayout._broadcast_value_to_parent_resolution  sŠ  € ð �{‰{Ð&Ð&Ð&Ø�{‰{Ð&¬3¨u¯{©{Ó+;¸qÒ+@Ð@Ð@Ü˜Ÿ™ T×%5Ñ%5Ñ6Ó7ˆ
Ø˜×*Ñ*Ò*Ø˜#ÒÑ&;àˆLà!×3Ñ3ÐØ×)Ñ)ˆØ˜Òð ×(Ò(Ø'9¸1Ð&=Ð#Ø#5°}Ð"E�Ø1°=ÐA‘à'(Ð*<Ð&=Ð#Ø#0Ð2DÐ"E�Ø,Ð.@ÐA‘Ø×&Ò&Ø×,Ñ,ˆJØ#'×#@Ñ#@Ð à#5°zÀ1Ð"EÐØ1°:Ð?SÐTˆOØ-¨}Ð=‰Kà×,Ñ,ˆJØ#'×#@Ñ#@Ð à#-¨qÐ2DÐ"EÐØ)Ð+?ÐASÐTˆOØ(Ð*<Ð=ˆKØ×1Ñ1ØØ 3Ø+Ø#Ø—+‘+Ø!ð 2ó 
ˆð ˆrd   N)rq   rä  rô  ró  rõ  r‰   r†   rð  )r†   r…   r(  rú   rŠ   rû   )r†   ztuple[str, str, str])r†   ztuple[str, str])re  r   r†   rè  )r&  rè  r†   rÐ  )r†   rÐ  )r&  rè  r†   rÎ  )rq   rQ   rA  rB   r†   rB   )rq   rQ   rA  rB   r4  r‰   r†   rB   )rŒ   r�   rŽ   r�   rI  rË  r÷  r“   rù  rü  rÿ  r  r  r  r  r  r  r  r  r  r  r  r   r+  r-  r1  r8  r:  r6  r·   rd   rb   rð  rð    s8  … ñð  ÓØÓØ'Ó'ØÓàð
àð
ð ,ð
ð #ð	
ð
 
!ò
ó ð
ð  òIó ðIð òIó ðIð ò5ó ð5ð òLó ðLð ò
ó ð
ð ò
ó ð
ð ò0ó ð0ð ò6ó ð6ð òJó ðJð ò+ó ð+ð ò2ó ð2ð ò
ó ð
ð ò5ó ð5ð ò=ó ð=ð
/
àð/
ð 
ó/
ðb3
à3ð3
ð 
!ó3
ðj
à	 ó
ð
Ø$9ð
à	ó
ð.
àð
ð ð
ð 
ó	
ð$
àð
ð ð
ð 
ó	
ð 2àð2ð ð2ð
  $ð2ð 
ô2rd   rð  c                  ó.   — e Zd ZU dZded<   ded<   dd„Zy)	Ú_ParentFullLoadTransformz9Lift loaded values to parent-full resolution when needed.rQ   rq   rð  rœ  c                óN   — | j                   j                  | j                  |«      S r\   )rœ  r8  rq   )rs   rA  s     rb   Úapplyz_ParentFullLoadTransform.applyT  s#   € Ø�{‰{×EÑEØ�K‰KØó
ð 	
rd   N)rA  rB   r†   rB   )rŒ   r�   rŽ   r�   rI  rG  r·   rd   rb   rE  rE  M  s   … áCàÓØ#Ó#ô
rd   rE  c                  ód   ‡ — e Zd ZdZddœ	 	 	 	 	 	 	 dˆ fd„Zd	d„Z	 	 	 	 	 	 	 	 	 	 d
d„Zdd„Zˆ xZS )Ú_GroupedReductionOpsHandlera	  Group-reduction stage handler: reshape parent-full tile into
    [num_groups, local_reduction_size], reduce over the local axis, and store
    the result.
    Loads go through the normal CSE path, then may be lifted to parent
    resolution before the reshape.
    N©Úload_transformc               ó\   •— t         ‰| �  |«       || _        || _        || _        || _        y r\   )rj   rk   Ú_kernelÚ_layoutÚ_familyÚ_load_transform)rs   Úinnerrq   rœ  ÚfamilyrK  rt   s         €rb   rk   z$_GroupedReductionOpsHandler.__init__c  s/   ø€ ô 	‰Ñ˜ÔØˆŒØˆŒØˆŒØ-ˆÕrd   c                óŒ   — | j                   j                  ||«      }| j                  �| j                  j                  |«      }|S r\   )Ú_innerÚloadrP  rG  )rs   rl   r˜   rA  s       rb   rU  z _GroupedReductionOpsHandler.loadr  sA   € Ø—‘× Ñ   uÓ-ˆØ×ÑÐ+ð ×(Ñ(×.Ñ.¨uÓ5ˆEØˆrd   c                óz  — | j                   }| j                  j                  ||«      }| j                  j	                  |«       |j                  || j                  j                  |«      }|xj                  dz  c_        |j                  ||| j                  j                  || j                  j                  «      S )zAReshape the parent-full tile and reduce over the local reduction.r@   )rM  rN  r:  rO  râ  Úemit_reshaper  Únum_reductionÚemit_reducer  r  )rs   r   Ú	src_dtyperH  rA  rh  Úreshapeds          rb   r¥  z%_GroupedReductionOpsHandler.reductionz  s•   € ð �L‰LˆØ—‘×:Ñ:¸1¸eÓDˆð 	�‰×#Ñ# AÔ&Ø—>‘> %¨¯©×)CÑ)CÀYÓOˆØ	�Š˜1Ñ�Ø�}‰}ØØØ�L‰L×$Ñ$ØØ�L‰L×%Ñ%ó
ð 	
rd   c                óè   — | j                   j                  |«      }| j                   j                  | j                  «      5  | j                  j                  |||«       d d d «       y # 1 sw Y   y xY wr\   )rO  rÕ  rÞ  rM  rT  rÁ  )rs   rl   r˜   rA  Úremapped_indexs        rb   rÃ  z+_GroupedReductionOpsHandler.store_reduction’  sW   € ØŸ™×1Ñ1°%Ó8ˆØ�\‰\×'Ñ'¨¯©Ó5ñ 	;Ø�K‰K×Ñ˜d N°EÔ:÷	;÷ 	;ñ 	;ús   ÁA(Á(A1)rq   rQ   rœ  rð  rR  rÐ  rK  ú_ParentFullLoadTransform | None©rl   r€   r˜   rƒ   r†   rB   )
r   r¿  rZ  r¿  rH  r€   rA  rB   r†   rB   rÂ  )	rŒ   r�   rŽ   r�   rk   rU  r¥  rÃ  r”   r•   s   @rb   rI  rI  [  sy   ø„ ñð ;?ñ.ð ð.ð
 (ð.ð (ð.ð 8õ.óð
àð
ð ð
ð ð	
ð
 ð
ð 
ó
÷0;rd   rI  c                  óZ   ‡ — e Zd ZdZddœ	 	 	 	 	 dˆ fd„Zdd„Z	 d		 	 	 	 	 	 	 	 	 d
d„Zˆ xZS )Ú_PointwiseRemapHandlera@  Pointwise bodies at a remapped iteration range.

    Unlike _GroupedReductionOpsHandler (which performs reshape+reduce), this
    handler runs pure pointwise bodies for reduced/parent-full prologues
    and epilogues. Loads use the normal CSE path after index remapping, then
    may be lifted to parent resolution.
    NrJ  c               óN   •— t         ‰| �  |«       || _        || _        || _        y r\   )rj   rk   rM  rO  rP  )rs   rQ  rq   rR  rK  rt   s        €rb   rk   z_PointwiseRemapHandler.__init__¡  s(   ø€ ô 	‰Ñ˜ÔØˆŒØˆŒØ-ˆÕrd   c                ó6  — | j                   j                  |«      }| j                   j                  | j                  «      5  | j                  j                  ||«      }d d d «       | j                  �| j                  j                  «      }S # 1 sw Y   Œ2xY wr\   )rO  rÕ  rÞ  rM  rT  rU  rP  rG  )rs   rl   r˜   r]  rA  s        rb   rU  z_PointwiseRemapHandler.load®  s~   € ØŸ™×1Ñ1°%Ó8ˆØ�\‰\×'Ñ'¨¯©Ó5ñ 	;Ø—K‘K×$Ñ$ T¨>Ó:ˆE÷	;à×ÑÐ+Ø×(Ñ(×.Ñ.¨uÓ5ˆEØˆ÷		;ð 	;ús   ÁBÂBc                óð   — | j                   }| j                  j                  |«      }| j                  j                  |«      5  | j                  j                  ||||¬«       d d d «       y # 1 sw Y   y xY w)N)Úmode)rM  rO  rÕ  rÞ  rT  rÁ  )rs   rl   r˜   rA  re  rh  r]  s          rb   rÁ  z_PointwiseRemapHandler.store¶  sg   € ð �L‰LˆØŸ™×1Ñ1°%Ó8ˆØ�\‰\×'Ñ'¨Ó*ñ 	FØ�K‰K×Ñ˜d N°EÀÐÔE÷	F÷ 	Fñ 	Fús   Á A,Á,A5)rq   r„   rR  rÐ  rK  r^  r_  r\   )
rl   r€   r˜   rƒ   rA  rB   re  r   r†   r‡   )rŒ   r�   rŽ   r�   rk   rU  rÁ  r”   r•   s   @rb   ra  ra  ˜  su   ø„ ñð ;?ñ.ð ð.ð
 (ð.ð 8õ.óð ð
Fàð
Fð ð
Fð ð	
Fð
 ð
Fð 
÷
Frd   ra  c                  ó@  — e Zd ZU dZeZded<   d„ Zd„ ZeZ	eZ
d„ Zd„ Zd„ Zd	„ Z	 d?	 	 	 d@d„Zd„ Zd„ Z	 dA	 dBd„Z	 	 	 	 	 	 	 	 	 	 	 	 dCd„Z	 	 	 	 	 	 	 	 dDd„Ze	 	 	 	 	 	 dEd„«       Z	 	 	 	 	 	 dFd„Ze	 	 	 	 	 	 dGd„«       Z	 	 dHd„Zd
dœ	 	 	 	 	 	 	 dId„Z	 dA	 	 	 dJd„Z	 	 dKd„Ze	 	 	 	 	 	 dLd„«       Z	 dM	 	 	 	 	 	 	 dNd„ZdOd„Z dPd„Z!ddœ	 	 	 	 	 dQd „Z"	 	 	 	 dRd!„Z#d"„ Z$dd#œd$„Z%	 	 	 	 dSd%„Z&dTd&„Z'	 	 	 	 	 	 dUd'„Z(dd
d(œ	 	 	 dVd)„Z)d*„ Z*	 	 	 	 	 	 dWd+„Z+dXd,„Z,e	 	 	 	 	 	 dYd-„«       Z-	 	 dZ	 	 	 	 	 	 	 	 	 	 	 	 	 d[d.„Z.d/„ Z/e e0jb                  d0«      d\d1„«       «       Z2e	 	 	 	 	 	 d]d2„«       Z3e	 	 	 	 	 	 d^d3„«       Z4e	 	 	 	 	 	 	 	 d_d4„«       Z5e	 	 d`d5„«       Z6e	 	 	 	 	 	 	 	 	 	 dad6„«       Z7e	 	 	 	 	 	 	 	 dbd7„«       Z8e	 	 	 	 	 	 	 	 dcd8„«       Z9ee:jv                  jx                  d
f	 	 	 ddd9„«       Z=ee:jv                  jx                  d
f	 	 	 ded:„«       Z>d;„ Z?dfd<„Z@	 dg	 dhd=„ZAd>„ ZBy
)iÚSIMDSchedulingzo
    Single Instruction Multiple Data parent class used for fusion across
    multiple different backends.
    z	type[Any]Úkernel_typec                ó&   — t        d„ |D «       «      S )Nc              3  ó€   K  — | ]6  }t         j                  j                  j                  t	        |«      «      –— Œ8 y ­wr\   r  r  s     rb   r  z*SIMDScheduling.group_fn.<locals>.<genexpr>Ì  s*   è ø€ ÒPÀQ”Q—W‘W×%Ñ%×.Ñ.¬}¸QÓ/?×@ÑPùs   ‚<>©r¹   )rs   rï   s     rb   Úgroup_fnzSIMDScheduling.group_fnË  s   € ÜÑPÈ%ÔPÓPÐPrd   c                ó¶	  ‡‡‡‡— t        |t        j                  «      st        |t        j                  «      r t        j                  j                  ||«      S |j                  \  }\  }Š|j                  \  }\  ŠŠt        ||«      }|j                  «       r)|j                  «       s|j                  «       rB |d«       y|j                  «       r)|j                  «       s|j                  «       r	 |d«       y|j                  «       r÷|j                  «       rç|‰k(  xr ‰‰k(  }|sddlm	} |j                  ||«      }|s,ddlm
} |j                  ||«      xr |j                  ||«      }|s |d|‰‰‰«       |r�|j                  «       s|j                  «       ra|j                  «       s||}}| j                  |j                  «       |‰«      Št        ˆˆfd„|j                  «       D «       «      s	 |d«       y|S |j                  «       �s«|j                  «       �sš|‰k(  r‰‰k(  s�|j!                  «       s |d	|‰‰‰«       y|j                  «       D ]`  }	|	j!                  «       r nN|	j#                  «       |j%                  «       z  sŒ7|	j                  \  }\  }
}||
k(  r‰|k(  rŒT |d
||
‰|«        y ||fD ]  }|j!                  «       sŒ y | j                  |j                  «       |‰«      }| j                  |j                  «       |‰«      }| j                  |j                  «       |j                  «       z   |‰«      }t&        j(                  j*                  rVd}t-        |«      dkD  r%t-        |«      dkD  r||cxk(  xr |k(  nc }n||k(  }nt-        |«      dkD  r||k(  }|s |d|||«       yy|j                  «       sÙ|j                  «       rÉ‰dk(  r‰dk7  sJ ‚|‰‰z  k(  r£t        ˆˆfd„|j                  «       D «       «      s	 |d«       yt&        j(                  j.                  r\|j!                  «       sLt1        | j                  |j                  «       |«      j3                  «       «      |df‰‰dffv }|s |d«       |S y|‰k7  r |d«       |‰k(  S |j                  «       r|j                  «       rJ ‚| j5                  ||«      S )zª
        Hook called by Scheduler to determine if the Triton backend
        can fuse node1 and node2.  These nodes might already be
        FusedSchedulerNodes.
        z&Split scan cannot fuse with reductionsFr   )ÚMixOrderReduction)ÚNestedReductionz1numel/rnumel mismatch (reduce) (%s, %s), (%s, %s)c              3  ó†   •K  — | ]8  }t         j                  ‰j                  «       |j                  «       ‰¬ «      –— Œ: y­w©)r  N)r„   r  rÉ   Ú
get_ranges)r  Ún2Úrnumel1rO  s     €€rb   r  z*SIMDScheduling.can_fuse.<locals>.<genexpr>  s>   øè ø€ ò ð ô ×,Ñ,ØŸ™›¨¯©«È'ð -÷ ñùs   ƒ>Az/invalid loop order and tiling for native matmulz5numel/rnumel mismatch (non-reduce) (%s, %s), (%s, %s)z:numel/rnumel mismatch prologue mismatch (%s, %s), (%s, %s)Tr   ztiling mismatch (%s, %s, %s)r@   c              3  ój   •K  — | ]*  }t         j                  ‰‰f|j                  «       «      –— Œ, y ­wr\   )r„   r  rr  )r  rõ   Únumel2Úrnumel2s     €€rb   r  z*SIMDScheduling.can_fuse.<locals>.<genexpr>`  s1   øè ø€ ò àô ×,Ñ,¨f°gÐ->ÀÇÁÃ×Oñùs   ƒ03z"nodes numel/rnumel incompatibilityzinvalid tiling for reductionznodes numel incompatibility)r  r   ÚForeachKernelSchedulerNodeÚcan_fuser  r1   Úis_split_scanrw   Útorch._inductor.schedulerrn  ro  Ú_is_dependent_reduction_pairrt  Úselect_tilingÚ	get_nodesr  Úis_templateÚused_buffer_namesÚget_buffer_namesr   r_   Ú tiling_prevents_pointwise_fusionr  Ú tiling_prevents_reduction_fusionr¹   rÉ   Úcan_fuse_horizontal)rs   Únode1Únode2r/  Únumel1ÚwhyÚreduction_can_fusern  ro  rË   Ú	pro_numelÚ
pro_rnumelrõ   Útiling1Útiling2Útiling3ÚcondÚis_reduction_tiling_validrv  rt  rw  rO  s                     @@@@rb   ry  zSIMDScheduling.can_fuseÎ  sà  û€ ô �eœY×AÑAÔBÄjØ”9×7Ñ7ôG
ô ×7Ñ7×@Ñ@ÀÈÓNÐNà$Ÿ{™{ÑˆÑˆF�GØ$Ÿ{™{ÑˆÑˆF�GÜ˜˜uÓ%ˆà×ÑÔ ¨×)<Ñ)<Ô)>Ø×!Ñ!Ô#ÙÐ<Ô=ØØ× Ñ Ô"¨5×+>Ñ+>Ô+@Ø×!Ñ!Ô#ÙÐ<Ô=Øà×ÑÔ E×$6Ñ$6Ô$8Ø!'¨6Ñ!1Ò!H°gÀÑ6HÐÙ%ÝGà%6×%?Ñ%?ÀÀuÓ%MÐ"á%õ
 Fà%4×%QÑ%QØ˜5ó&ò &=à%×.Ñ.¨u°eÓ<ð #ñ &ÙØGØØØØôñ "Ø×&Ñ&Ô(¨E×,BÑ,BÔ,Dð ×-Ñ-Ô/Ø#(¨%˜5�Eð ×+Ñ+¨E¯O©OÓ,=¸vÀwÓO�Üô ð $Ÿo™oÓ/ô	ô ñ ÐIÔJØ à%Ð%à×!Ñ!Õ#¨E×,>Ñ,>Õ,@Ø˜fÒ$¨°GÒ);Ø×(Ñ(Ô*ÙØOØØØØôð !ð !&§¡Ó 1ò )˜à×+Ñ+Ô-Ù!ð  $×5Ñ5Ó7¸%×:PÑ:PÓ:RÒRØ$Ø59·Z±ZÑ2˜Ñ2˜I zØ &¨)Ò 3¸À:Ó8MÙØ \Ø &Ø )Ø 'Ø *ôñ $)ð#)ð& ˜U�^ò  �Ø—=‘=•?Ùð ð
 ×(Ñ(¨¯©Ó):¸FÀGÓLˆGØ×(Ñ(¨¯©Ó):¸FÀGÓLˆGØ×(Ñ(Ø—‘Ó! E§O¡OÓ$5Ñ5°v¸wóˆGô �}‰}×=Ò=Ø�Ü�w“< !Ò#Ü˜7“| aÒ'Ø&¨'Ö<°WÔ<™à&¨'Ñ1™Ü˜“\ AÒ%Ø" gÑ-�DÙÙØ6ØØØô	ð !àà×!Ñ!Ô#¨×(:Ñ(:Ô(<Ø˜a’< G¨q¢LÐ0Ð0Ø˜ 'Ñ)Ò)Üô à"Ÿ_™_Ó.ôô ñ Ð<Ô=Ø ä—M‘M×BÒBØ!×-Ñ-Ô/ä05Ø×*Ñ*¨5¯?©?Ó+<¸fÓE×LÑLÓNó1ð   ˜Ø ¨!Ð,ðð1Ð-ñ 5ÙÐ:Ô;Ø4Ð4Øà˜ÒÙÐ1Ô2Ø˜VÑ#Ð#à×!Ñ!Ô#¨E×,>Ñ,>Ô,@Ð@Ð@à×'Ñ'¨¨uÓ5Ð5rd   c           
     óz  ‡‡‡‡‡‡‡— g Št        t        j                     «       Št        «       Št        «       Šd Šˆˆfd„}ˆˆfd„}ˆfd„}ˆˆˆˆfd„}t        j                  ˆˆˆˆfd„«       }ˆˆfd„}	|D ]©  }
|
‰v rŒ‰j                  |
«        ||
«      r? |	|
‰«      r |«       5  	 d d d «       ‰r ||
«      s‰xs t        ‰«      Šnd Š ||
«       Œ` ||
«      r" |«       5  ‰j                  |
«       d d d «       ŒŠt        d‰› d‰› d	|
j                  d
   › �«      ‚ ‰S # 1 sw Y   Œ|xY w# 1 sw Y   ŒÃxY w)Nc                ób   •— | j                   \  }\  }}|‰k(  xr |‰k(  xs |‰‰z  k(  xr |dk(  S ©Nr@   ©r  ©rõ   r/  Ú
node_numelÚnode_rnumelro   rQ  s       €€rb   Úfits_in_main_bodyz@SIMDScheduling.generate_node_schedule.<locals>.fits_in_main_body‰  sH   ø€ Ø+,¯7©7Ñ(ˆAÑ(�
˜KØ %Ñ'ÒA¨K¸6Ñ,Aò Ø˜e f™nÑ,ÒA°ÀÑ1Aðrd   c                óN   •— | j                   \  }\  }}|‰k(  xr |dk(  xr ‰dk7  S r“  r”  r•  s       €€rb   Úfits_outside_reductionzESIMDScheduling.generate_node_schedule.<locals>.fits_outside_reduction�  s4   ø€ Ø+,¯7©7Ñ(ˆAÑ(�
˜KØ Ñ&ÒK¨;¸!Ñ+;ÒKÀÈ!ÁÐKrd   c                ó\   •— | j                   j                  D ]  }|j                  ‰v sŒ y yr  )Úread_writesÚreadsrl   )rõ   ÚreadÚcurrent_loop_buffer_usages     €rb   Úexpect_improved_memory_usagezKSIMDScheduling.generate_node_schedule.<locals>.expect_improved_memory_usage“  s1   ø€ ØŸ™×+Ñ+ò  �Ø—9‘9Ð 9Ò9Ùð ð rd   c                óŠ  •— ‰j                  | «       ‰j                  | «       ‰j                  | j                  j                  D �cg c]  }|j
                  ‘Œ c}«       | j                  «       rŒt        | t        j                  «      rrt        | j                  t        j                  «      rNt        | j                  j                  t        j                  «      s ‰j                  | j                  «       «       y ‰j                  | j                  j                   D �cg c]  }|j
                  ‘Œ c}«       y c c}w c c}w r\   )rð   rÖ   Úupdaterœ  r�  rl   rw   r  r   rv  rË   r   rw  ÚdataÚScanÚget_nameÚwrites)rõ   rX   rŸ  ÚdonerM  Únot_ready_yet_nodess     €€€€rb   Úschedule_node_in_loopzDSIMDScheduling.generate_node_schedule.<locals>.schedule_node_in_loop™  sÐ   ø€ Ø�H‰H�QŒKØ× Ñ  Ô#Ø%×,Ñ,¸a¿m¹m×>QÑ>QÖ-R¸¨a¯f«fÒ-RÔSð
 —‘Ô Ü˜q¤)×"9Ñ"9Ô:Ü˜qŸv™v¤r×'8Ñ'8Ô9Ü" 1§6¡6§;¡;´·±Ô8à#×'Ñ'¨¯
©
«Õ5à)×0Ñ0À!Ç-Á-×BVÑBVÖ1W¸Q°!·&³&Ò1WÕXùò .Sùò 2Xs   ÁD;Ä E c               3  óL  •K  — ‰r‰d   t         u r‰j                  «        n‰j                  t        «       ‰r1‰j	                  ‰t        «       ‰j	                  ‰dz   t         «       d Šd –— ‰j                  t         «       ‰j                  «        ‰ j                  «        y ­w)Nrß  r@   )rI   ÚpoprÖ   rH   ÚinsertÚclear)rŸ  Úmaybe_split_indexrM  r¨  s   €€€€rb   Úend_current_reduction_loopzISIMDScheduling.generate_node_schedule.<locals>.end_current_reduction_loopª  s�   øè ø€ ñ  ¨rÑ!2´oÑ!EØ×!Ñ!Õ#à×$Ñ$Ô%5Ô6Ù Ø×$Ñ$Ð%6Ô8HÔIØ×$Ñ$Ð%6¸Ñ%:¼OÔLØ$(Ð!ÛØ× Ñ ¤Ô1Ø×%Ñ%Ô'Ø%×+Ñ+Õ-ùs   ƒB!B$c                ó€   •— ‰dk(  ry‰| j                   z  sy|rt        |d   t        t        f«      rJ ‚t	        ‰«      S )Nr@   Frß  )Ú	ancestorsr  rI   rH   r‰   )rË   rM  r¨  rQ  s     €€rb   Ú#requires_closing_previous_reductionzRSIMDScheduling.generate_node_schedule.<locals>.requires_closing_previous_reductionº  sN   ø€ Ø˜Š{ØØ&¨¯©Ò7ØÙ ¬Ø˜bÑ!¤OÔ5EÐ#Fô*ð ð ô Ð+Ó,Ð,rd   zunexpected group: (r    z) != r@   )
r   r   r/   rä  rå  rð   r  rÖ   r†  r  )rs   r™   ro   rQ  r˜  rš  r   r©  r¯  r²  rË   rŸ  r§  r®  rM  r¨  s     ``       @@@@@rb   Úgenerate_node_schedulez%SIMDScheduling.generate_node_schedule€  sW  þ€ Ø#%ˆÜœ)×5Ñ5Ñ6Ó8ˆô 0:«|ÐÜ5?³\Ð!Ø(,Ðõ	õ	Lô	÷	Yô" 
×	"Ñ	"ö	.ó 
#ð	.õ	-ð ò 	ˆDØ�t‰|ØØ�H‰H�TŒNá  Ô&Ù6°t¸]ÔKÙ3Ó5ñ Ø÷ñ -Ñ5QÐRVÔ5Wà(9Ò(O¼SÀÓ=OÑ%ð )-Ð%á% dÕ+Ù'¨Ô-Ù/Ó1ñ /Ø!×(Ñ(¨Ô.÷/ð /ô *Ø)¨%¨°°6°(¸%ÀÇ
Á
È1Á¸ÐOóð ð-	ð4 Ð÷'ð ú÷/ð /ús   Â(D%Ã&D1Ä%D.	Ä1D:	c                óÐ   — |j                   |j                  }}|j                  |j                  «       z  s|j                  |j                  «       z  rJ ‚| j	                  ||«       y r\   )r…  r†  r±  Úget_operation_namesÚ_codegen_mix_order_reduction)rs   rË   r…  r†  s       rb   Úcodegen_mix_order_reductionz*SIMDScheduling.codegen_mix_order_reductionà  s[   € Ø—z‘z 4§:¡:ˆuˆð —O‘O e×&?Ñ&?Ó&AÒAØ�O‰O˜e×7Ñ7Ó9Ò9ð	
ð 
ð 	×)Ñ)¨%°Õ7rd   c                ó¦   — |j                  «       }g }g }|D ]5  }|j                  «       r|j                  |«       Œ%|j                  |«       Œ7 ||fS r\   )r~  rw   rÖ   )rs   rË   r™   Ú
reductionsÚ	epiloguess        rb   Ú#_split_mix_order_reduction_epiloguez2SIMDScheduling._split_mix_order_reduction_epilogueê  s\   € à—‘Ó ˆØˆ
Øˆ	Øò 	'ˆDØ× Ñ Ô"Ø×!Ñ! $Õ'à× Ñ  Õ&ð		'ð
 ˜9Ð$Ð$rd   c           	     óÖ  — |j                   |j                  }}|j                  }| j                  |||dœg|ddddœ«      d   }|j                  sJ ‚|j
                  sJ ‚||_        | j                  ||«       |j                  j                  t        |j                  «      |j                  d   z  |j                  d   |j                  z   dz
  |j                  z  z  d	t        j                  ¬
«      \  }}	}
|
dk(  s
J d|
›�«       ‚|5  |j                  «        ddd«       t!        j"                  «       }t%        j&                  |«      5  |5  |r%|j)                  t+        j,                  d¬«      «       |j/                  «       }ddd«       ddd«       |r)j1                  t3        t4        j6                  «      d«      }||	fS # 1 sw Y   Œ­xY w# 1 sw Y   ŒMxY w# 1 sw Y   ŒQxY w)zŸ
        for_benchmark:
            True if the generated code is for benchmarking. We need make
            sure benchmark harness code is generated.
        )rX   rY   NT)rR  rP  rp  r�  r   rY   rX   r@   F)r   zws_off=©Úbenchmark_kernelÚtriton_)ro   r  rM  Úcreate_kernel_choicesro  rp  r{  Ú!codegen_node_schedule_with_kernelr  Ú	workspacer  r~  rh  r]   r=  rá  rä  Ú	ExitStackr?   Úset_kernel_handlerÚenter_contextr   Úpatchr·  Úreplacer€   r7   ÚKERNEL_NAME)rs   Úkernel_featuresÚ
split_sizeÚfor_benchmarkro   rQ  rM  rq   r/  Úws_nameÚws_offÚstackÚsrc_codes                rb   Ú-_generate_kernel_code_for_mix_order_reductionz<SIMDScheduling._generate_kernel_code_for_mix_order_reductionö  sï  € ð (×-Ñ-¨×/NÑ/NˆvˆØ'×5Ñ5ˆà×+Ñ+ØØ Ñ(Ð)à+Ø!%Ø'+Ø15ñ	ó	
ð ñ	ˆð ×*Ò*Ð*Ð*Ø×)Ò)Ð)Ð)Ø'ˆÔØ×.Ñ.¨}¸fÔEð $Ÿ[™[×2Ñ2Ü�×/Ñ/Ó0Ø�m‰m˜EÑ"ñ#à—‘˜cÑ" V×%7Ñ%7Ñ7¸!Ñ;À×@RÑ@RÑRñTð Ü—+‘+ð 3ó 
Ñˆˆ7�Fð ˜Š{Ð(˜w˜v˜i˜LÓ(ˆ{Øñ 	"Ø×ÑÔ!÷	"ô ×$Ñ$Ó&ˆÜ×!Ñ! &Ó)ñ 	/¨5ñ 	/ÙØ×#Ñ#¤F§L¡LÀ$Ô$GÔHØ×,Ñ,Ó.ˆH÷	/÷ 	/ñ
 ð
  ×'Ñ'¬¬K×,CÑ,CÓ(DÀiÓPˆHØ�w Ð(Ð(÷	"ð 	"ú÷	/ð 	/ú÷ 	/ð 	/ús0   Ä
GÅGÅ8GÆGÇGÇG	ÇGÇG(Nc                ó   — t         ‚r\   r…  )rs   ÚmodÚn_spills_thresholdÚ
node_namess       rb   Úbenchmark_codegened_modulez)SIMDScheduling.benchmark_codegened_module+	  s
   € ô "Ð!rd   c                ó8  ‡ ‡‡%‡&— t         j                  j                  ‰«      \  Š&}ˆˆ&fd„} |«       }t        xj                  dz  c_        ‰ j                  |«      \  }}g }|D ]C  }	|	j                  «        |	j                  «       }
|
j                  «        |j                  |
«       ŒE ‰ j                  ‰j                  «       |z   ‰&|«      }t        |‰&|«      Š%t        j                  j                  j                   sqt        j"                  j$                  €Wt        j"                  j&                  s t        j(                  st        j*                  rˆ%ˆ fd„}t-        j.                  ||d«      }‰ j1                  ‰%|d¬«      \  }}}t3        |d   j4                  j6                  «      }i }|�rZ|D �]  }	|	j9                  «       d   j4                  j;                  «       }|	j9                  «       d   j<                  d   j4                  j9                  «       d   j4                  j;                  «       }|||<   ‰ j                   sJ ‚‰ j                   j>                  jA                  |	j9                  «       d   j<                  d   j4                  j;                  «       «       tB        jD                  jF                  jA                  |«       �Œ |jH                  D ]-  }|jK                  |jL                  |jL                  «      |_&        Œ/ ‰ jO                  |||«      }||_(        tS        |«      |_)        tC        jT                  |«      5  ‰%jW                  «       D ]@  }|j9                  «       d   j4                  j;                  «       |vsŒ1|jY                  «        ŒB 	 d d d «       tB        jD                  jZ                  j]                  d«       ‰ j_                  |d «       |ja                  |jP                  d¬	«       tB        jD                  xjF                  |jF                  z  c_#        tB        jD                  xjb                  |jb                  z  c_1        te        |«      te        |jH                  «      k(  sJ ‚tB        jD                  jZ                  jg                  ‰&|z   dz
  |z  «      }ti        |jH                  «      D �]�  \  }}|jL                  }d
|› d|› d�}|› d|› �}d
|› d|› �}dddœ}|jK                  |jj                  |jj                  «      }|› d|› d|› d|› d|› d|› d|› d�}tB        jD                  jm                  |«      } | �l| jo                  «       jp                  D �!cg c]+  }!tB        jD                  jZ                  jg                  |!«      ‘Œ- }"}!ddjs                  |"«      › d�}#|d|#› d�z  }tB        jD                  ju                  |«      x}$t        jv                  k7  r	|d|$› d�z  }tB        jD                  jZ                  jy                  |«       tB        jD                  jZ                  jz                  jA                  |«       �Œ“ |j}                  «        |r‰ j                  |«       ‰ j�                  «        y # 1 sw Y   �ŒîxY wc c}!w )Nc                 ór  •— t         j                  j                  �t         j                  j                  S t        j                  ‰j                  «       «      } | j                  }|dz  }t        j                  j                  j                  ‰«      }t        t        ||z  «      d«      }t        |d«      }|S )Né   é   é€   )r   r_   Úmix_order_reduction_split_sizer*   ÚcreateÚ
get_deviceÚmulti_processor_countr?   rÏ   rÐ   rå   r±  r,   Úmin)Údevice_propÚnum_smÚestimated_num_splitsÚ
numel_hintrÊ  r…  ro   s        €€rb   Ú_pick_split_sizezESIMDScheduling._codegen_mix_order_reduction.<locals>._pick_split_size3	  s™   ø€ ä�}‰}×;Ñ;ÐGÜ—}‘}×CÑCÐCô +×1Ñ1°%×2BÑ2BÓ2DÓEˆKØ ×6Ñ6ˆFØ#)¨A¡:Ð ô
 Ÿ™×)Ñ)×;Ñ;¸EÓBˆJÜœ_¨ZÐ;OÑ-OÓPÐRTÓUˆJÜ˜Z¨Ó-ˆJØÐrd   r@   c                óŠ   •— ‰j                  ‰| d¬«      \  }}}t        j                  |«      }‰j                  |«      \  }}|S )NT©rÊ  rË  )rÐ  r   rU  rÕ  )Úcandidate_split_sizer/  rÏ  rÒ  ÚmsrÉ  rs   s        €€rb   Ú_benchz;SIMDScheduling._codegen_mix_order_reduction.<locals>._benche	  sS   ø€ Ø!%×!SÑ!SØ#Ø3Ø"&ð "Tó "‘��1�hô
 "×&Ñ& xÓ0�Ø×7Ñ7¸Ó<‘��AØ�	rd   rØ  Fræ  r   z!# Call mix order reduction kernel)r^  ú(z) * (r  z * z + 1) * ÚaminÚamax)rß  r±  z = r3  z : z].view(r    z).z(dim=0)r4  z.view(z.to()Ar   rn  Úget_numel_rnumelr   r·  r»  Úcancel_reduction_splitÚextract_pw_from_reductionÚswap_pw_red_dimensionrÖ   r³  r~  rL   r]   r^   r   Údeterministicr_   rÛ  Ú'mix_order_reduction_autotune_split_sizeÚmax_autotuneÚcoordinate_descent_tuningr)   Úautotune_single_fieldrÐ  r‰   rË   Ú_split_sizeÚget_outputsr¥  ÚusersÚremoved_opsrð   r?   rÏ   Úremoved_buffersr~  rò   rG  Údefine_kernelrY  r   rÄ  Úscheduler_nodesÚmark_runrW  Úmake_commentÚcodegen_commentr_  Úinplaced_to_remover  Úcodegen_python_sizevarr­  rH  r”  r�  r  r5  r|  r=  Ú	writelineÚ	allocatedr\  Ú_codegen_nodesÚfree_buffers_in_scheduler)'rs   r…  r†  rQ  rä  rÊ  Únode2_reductionsÚnode2_epilogueÚconverted_nodesÚsubnodeÚ	convertedrM  ré  rq   rÌ  rÏ  Úis_split_reductionÚrenameÚbufnameÚusernameÚpartial_accumrY  rË   Únsplitr¯  rG  Ú
stride_strr–  ÚendÚreduction_type2opÚopnameÚfinal_reduceÚbufferrô   r>  Úfinal_shape_strÚbuffer_dtyperÉ  ro   s'   ``                                   @@rb   r¶  z+SIMDScheduling._codegen_mix_order_reduction0	  s   û€ Ü!×3Ñ3×DÑDÀUÓK‰ˆˆvõ	ñ$ &Ó'ˆ
ô 	×+Ò+¨qÑ0Õ+ð ,0×+SÑ+SØó,
Ñ(Ð˜.ð ˆØ'ò 	.ˆGØ×*Ñ*Ô,Ø×9Ñ9Ó;ˆIØ×+Ñ+Ô-Ø×"Ñ" 9Õ-ð		.ð
 ×3Ñ3Ø�O‰OÓ Ñ/°¸ó
ˆô -¨]¸EÀ6ÓJˆô —‘×&Ñ&×4Ò4Ü—‘×<Ñ<ÐDä—‘×EÒEÜ×&Ò&Ü×3Ò3õô '×<Ñ<ØØØóˆJð %)×$VÑ$VØØ!Øð %Wó %
Ñ!ˆ�˜ô "Ð"2°1Ñ"5×":Ñ":×"FÑ"FÓGÐØˆÚØ+ó 5�Ø!×-Ñ-Ó/°Ñ2×7Ñ7×@Ñ@ÓB�à×'Ñ'Ó)¨!Ñ,ß‘U˜1ñç‘TŸ+™+›-¨ñ+÷ ‘TŸ(™(›*ð	 ð #+��w‘Ø—~’~Ð%�~Ø—‘×*Ñ*×.Ñ.Ø×'Ñ'Ó)¨!Ñ,×2Ñ2°1Ñ5×:Ñ:×CÑCÓEôô —‘×'Ñ'×+Ñ+¨GÖ4ð5ð "(×!@Ñ!@ò �Ø,2¯J©JØ!×-Ñ-¨}×/HÑ/Hó-�Õ)ðð
 ×(Ñ(¨°=À&ÓIˆØ(ˆÔÜ$ XÓ.ˆÔä×!Ñ! &Ó)ñ 	$Ø'×7Ñ7Ó9ò $�ð ×#Ñ#Ó% aÑ(×-Ñ-×6Ñ6Ó8ÀÒFØ—M‘M•Oñ$÷	$ô 	
�‰×Ñ×)Ñ)Ð*MÔNØ×Ñ˜]¨DÔ1à×Ñ˜6×-Ñ-¸UÐÔCÜ	�‰×Ò 6×#9Ñ#9Ñ9ÕÜ	�‰×"Ò" f×&?Ñ&?Ñ?Õ"ô �?Ó#¤s¨6×+JÑ+JÓ'KÒKÐKÐKÜ—‘×%Ñ%×<Ñ<Ø�ZÑ !Ñ#¨
Ñ2ó
ˆô #,¨F×,KÑ,KÓ"Ló !	<ÑˆC�Ø'×3Ñ3ˆKà˜V˜H E¨&¨°Ð3ˆJØ�e˜3˜z˜lÐ+ˆEØ�c�U˜( : ,Ð/ˆCàØñ!Ðð '×*Ñ*Ø×,Ñ,¨m×.JÑ.JóˆFð *˜]¨#¨g¨Y°a¸°w¸cÀ#ÀÀgÈfÈXÐUWÐX^ÐW_Ð_aÐbhÐaiÐipÐqˆLô —W‘W×'Ñ'¨Ó4ˆFØÐ!ð $×.Ñ.Ó0×5Ñ5öàô —G‘G×(Ñ(×?Ñ?ÀÕBð�ð ð %& d§i¡i°Ó&<Ð%=¸QÐ"?�Ø &¨Ð(9¸Ð ;Ñ;�ô !"§¡× 1Ñ 1°+Ó >Ð>�Ä5Ç;Á;ÒNØ $ | n°AÐ 6Ñ6�Ü�G‰G× Ñ ×*Ñ*¨<Ô8ô �G‰G× Ñ ×*Ñ*×.Ñ.¨{Ö;ðC!	<ðF 	×$Ñ$Ô&áØ×Ñ Ô/à×&Ñ&Õ(÷y	$ñ 	$üòNs   Í	AZ
ÎZ
Õ,0ZÚ
Zc                ó>  ‡#— |j                   |j                  }}|j                  \  }\  }}|j                  \  }\  }}|j                  }	|j                  t
        j                  j                  j                  u }
t        |	«      }t
        j                  j                  |||j                  «      }|€J ‚t        |«      }t
        j                  j                  j                  }|D ��cg c]  \  }}||j                  «       v r||u r|‘Œ }}}t!        |«      }|j                  «       D �cg c]  }||vr|‘Œ
 }}|j"                  Š#| j%                  |j                  «       ||«      }t'        ˆ#fd„|D «       «      sJ ‚‰#j(                  }| j%                  |||«      }t*        j,                  j.                  j0                  j2                  r|j5                  «       nd}t7        ||||«      }| j9                  ||||«      \  }}d|vrd|vsJ d«       ‚t:        xj<                  dz  c_        |d|dœ}t?        d	| jA                  ||g|«      d
   «      }|
r||_!        n||_"        | jG                  ||«       |5  |jI                  «        tJ        jM                  ||	|
«      }|jO                  |«      }|jQ                  |«      }|jS                  «       } | jU                  ||jV                  |jX                  «      }!|j[                  |«      }"| j]                  ||| |!t_        ||«      ¬«       | ja                  ||‰#||jV                  |jX                  |!|"||| «       |jI                  «        ddd«       | jc                  |||g |¢|¢|¢«       yc c}}w c c}w # 1 sw Y   Œ/xY w)a�  
        Generate a single kernel with an outer reduction, a group
        reduction, and optional reduced/parent-full epilogues for
        dependent cross-axis reductions that share a large input.

        The local group may split either parent tree. For example,
        group-in-R reshapes [XBLOCK, RBLOCK] into [XBLOCK, RBLOCK/G, G],
        while group-in-X reshapes it into [XBLOCK/G, G, RBLOCK].
        Nc              3  ó&   •K  — | ]  }|‰u –— Œ
 y ­wr\   r·   )r  ÚsnÚgrouped_reductions     €rb   r  z:SIMDScheduling.codegen_nested_reduction.<locals>.<genexpr>	
  s   øè ø€ ÒF¨r�2Ð*Ô*ÑFùs   ƒrW   rV   z2nested reduction does not support tiled reductionsr@   F)rR  r‚  rP  rQ   r   rJ  )2r…  r†  r  Ú
group_sizeÚgrouped_axisr   ro  ÚGroupedAxisÚRr÷   Ú _classify_nested_pointwise_nodesÚdomain_contextrN  ÚPointwiseDomainÚLOCAL_REDUCTION_INPUTr~  r   r  r³  r-  Ú_bodyr]   r^   r   r_   Úcoalesce_tiling_analysisÚget_coalesce_analysisrL   Úget_tiling_and_scoresr   Úcodegen_nested_reductionr   rÀ  r}  r|  rÁ  rá  rð  r÷  r   r+  r-  Ú!_local_reduction_iteration_valuesré  rê  r1  Ú_codegen_remapped_pointwiserE  Ú _codegen_nested_grouped_scheduleÚ!_finalize_nested_reduction_kernel)$rs   rË   Ú
outer_nodeÚgrouped_noder/  Úouter_numelÚouter_rnumelÚgrouped_numelÚgrouped_rnumelrô  rõ  Úlocal_reduction_size_hintÚnested_pointwise_domainsÚpointwise_domain_by_nodeÚlocal_reduction_domainr  ÚdomainÚouter_local_reduction_pointwiseÚ#outer_local_reduction_pointwise_setÚouter_codegen_nodesÚgrouped_scheduleÚgrouped_reduction_bodyÚcombined_scheduleÚcoalesce_analysisrÉ  rO  Útiling_scoreÚkernel_kwargsrq   rœ  r&  Úreduced_output_familyÚparent_full_familyÚlocal_reduction_sourceÚparent_full_sourcer  s$                                      @rb   r)  z'SIMDScheduling.codegen_nested_reductionÖ	  sP  ø€ ð $(§:¡:¨t¯z©z�Lˆ
Ø)3×)9Ñ)9Ñ&ˆÑ&ˆK˜Ø-9×-?Ñ-?Ñ*ˆÑ*ˆM˜>Ø#Ÿ™Ðà×Ñ¤×!:Ñ!:×!FÑ!F×!HÑ!HÐHð 	ô %(Ð(<Ó$=Ð!ä×%Ñ%×FÑFØØØ×#Ñ#óð 	!ð (Ð3Ð3Ð3ô Ð)Ó*ð 	!ô ×%Ñ%×5Ñ5×KÑKð 	ð
 7÷+
á��FØ�Z×)Ñ)Ó+Ñ+°Ð:PÑ0Pò ð+
Ð'ñ +
ô
 /9Ø+ó/
Ð+ð
 !×*Ñ*Ó,ö
àØÐ<Ñ<ò ð
Ðð 
ð
 6:×5KÑ5KÐØ48×4OÑ4OØ×"Ñ"Ó$ØØó5
Ðô
 ÓFÐ5EÔFÔFÐFÐFØ!2×!8Ñ!8Ðà ×7Ñ7ØØØó
Ðô �‰×%Ñ%×,Ñ,×EÒEð ×,Ñ,Ô.àð 	ô
 -ØØØØó	
ˆð  $×9Ñ9ØØØØó	 
Ñˆ�ð ˜&Ñ  S°Ñ%6ð 	
Ø@ó	
Ð6ô 	×(Ò(¨AÑ-Õ(à'Ø.3Ø)ñ)
ˆô
 ØØ×&Ñ&ØØ�Øóð ñ	ó
ˆñ  Ø 9ˆFÕà 9ˆFÔð
 	×.Ñ.Ð/@À&ÔIàñ 5	"ð ×ÑÔ!ä.E×.QÑ.QØØ$Ø$ó/ˆFð ×5Ñ5Ð6LÓMð !ð ×1Ñ1Ð2FÓGð "ð ×.Ñ.Ó0ð ð ×6Ñ6Ø*Ø(×6Ñ6Ø(×8Ñ8óð #ð 39×2UÑ2UØ$ó3Ðð ×,Ñ,ØØ/Ø"Ø&Ü7¸ÀÓGð -ô ð ×1Ñ1ØØ Ø!ØØ$×2Ñ2Ø$×4Ñ4Ø&Ø"Ø(Ø%Ø"ôð ×ÑÔ!÷k5	"ðn 	×.Ñ.ØØØðØ"ðà0ðð "ðõ			
ùóG+
ùò
÷H5	"ð 5	"ús   Ã("NÄ*NÉ4C1NÎNc                óT  — | j                  |xs |«      }t        j                  |«      5  t        j                  di |¤Ž5  |j                  «       }d d d «       d d d «       | j                  ||«      |_        t        |«      |_        t        j                  |«      5  |j                  «       D ]  }|j                  «        Œ 	 d d d «       |D �cg c]  }t        |t        «      sŒ|‘Œ }}| j                  ||«       y # 1 sw Y   Œ°xY w# 1 sw Y   Œ´xY w# 1 sw Y   ŒTxY wc c}w rÆ   )Ú_collect_config_patchesr?   rÄ  r   rÆ  r·  rû  rY  r   r~  rý  r  r/   Ú_launch_kernel_and_cleanup)	rs   rq   r>  rË   Úconfig_patch_scheduleÚconfig_patchesrÏ  r  Úbase_scheduler_nodess	            rb   r-  z0SIMDScheduling._finalize_nested_reduction_kernel„
  s  € ð ×5Ñ5Ø!Ò6Ð%6ó
ˆô ×!Ñ! &Ó)ñ 	/¬6¯<©<Ñ+I¸.Ñ+Iñ 	/Ø×,Ñ,Ó.ˆH÷	/÷ 	/à!×/Ñ/ØØØó
ˆÔô
 % XÓ.ˆÔä×!Ñ! &Ó)ñ 	Ø—n‘nÓ&ò �Ø—‘•ñ÷	ð
 /ö 
Ø´*¸TÔCTÕ2UŠDð 
Ðð  
ð 	×'Ñ'¨Ð0DÕE÷!	/ð 	/ú÷ 	/ð 	/ú÷	ð 	üò 
s;   «DÁDÁDÂ&DÃD%Ã(D%ÄD
	ÄDÄDÄD"c           	     óð  — |j                   }t        |j                  D �cg c]  }|j                  |   ‘Œ c}|«      }t	        ||«      }|D �]"  }|t
        t        fv rŒt        |t        j                  «      sJ ‚||u r| j                  ||||||
«       ŒJ|	j                  |«      }|€J ‚|t        j                  j                  j                  u r| j                  ||g|
|«       Œ›|t        j                  j                  j                   u r|}nH|t        j                  j                  j"                  u r| j%                  |||«      }nt'        d|› �«      ‚| j                  ||g|||¬«       �Œ% yc c}w )a/  Interpret the local reduction schedule with nested emitters.

        ``local_reduction_source`` is the pre-reduction domain, including the
        local reduction lane. ``parent_full_source`` is the outer tile domain
        used by pointwise consumers after reduced values are broadcast back.
        Nz$unexpected nested pointwise domain: rJ  )r%  rÎ  r  rn   rE  rH   rI   r  r   rv  Ú_codegen_grouped_reductionrò   ro  r#  ÚREDUCEDr+  r$  ÚPARENT_FULLÚ(_select_full_resolution_pointwise_sourcer  )rs   rq   r<  r  rœ  ré  rê  rD  rE  r6  rB  rC  r=  ri  Úreduced_sourceÚparent_full_load_transformr  r8  Úsources                      rb   r,  z/SIMDScheduling._codegen_nested_grouped_schedule 
  s«  € ð, "3×!8Ñ!8ÐÜ(ð 0×9Ñ9öàð '×1Ñ1°!Ó4òð ó
ˆô &>¸fÀfÓ%MÐ"Ø"ó /	ˆBØÔ&¬Ð8Ñ8ð
 Ü˜b¤)×"9Ñ"9Ô:Ð:Ð:ØÐ&Ñ&Ø×/Ñ/ØØØØ!Ø#Ø)ôð Ø-×1Ñ1°"Ó5ˆFØÐ%Ð%Ð%Øœ×2Ñ2×BÑB×JÑJÑJØ×0Ñ0ØØ�DØ)Ø"ô	ð àÜ×,Ñ,×<Ñ<×RÑRñSð 0‘Øœ9×4Ñ4×DÑD×PÑPÑPØ×FÑFØØ&Ø*ó‘ô %Ð'KÈFÈ8Ð%TÓUÐUØ×,Ñ,ØØ�Ø"ØØ9ð -ö ñS/	ùòs    E3c                ó>  — |j                  «       }| j                  |j                  |«      r|S | j                  |j                  |«      r|S t        j	                  |j                  |«      r|S t        j	                  |j                  |«      r|S t        d|› �«      ‚)Nz5unsupported full-resolution nested pointwise ranges: )rr  Ú_ranges_exactly_match_groupsr  r„   r  r  )rs   r  rE  rD  Ú	sn_rangess        rb   rP  z7SIMDScheduling._select_full_resolution_pointwise_sourceð
  sš   € ð —M‘M“Oˆ	Ø×,Ñ,Ð-?×-FÑ-FÈ	ÔRØ%Ð%Ø×,Ñ,Ð-C×-JÑ-JÈIÔVØ)Ð)Ü×#Ñ#Ð$6×$=Ñ$=¸yÔIØ%Ð%Ü×#Ñ#Ð$:×$AÑ$AÀ9ÔMØ)Ð)ÜØCÀIÀ;ÐOó
ð 	
rd   c                ó€   — |\  }}| xr5 t        |«      t        | «      k(  xr t        d„ t        || «      D «       «      S )Nc              3  óv   K  — | ]1  \  }}t         j                  j                  j                  ||«      –— Œ3 y ­wr\   )r?   rÏ   rÐ   rÑ   )r  r)  r  s      rb   r  z>SIMDScheduling._ranges_exactly_match_groups.<locals>.<genexpr>  s3   è ø€ ò á%�J ô —‘× Ñ ×8Ñ8¸ÀU×Kñùs   ‚79)r  r  rÙ  )r  rè  Úiter_rangesÚreduction_rangess       rb   rU  z+SIMDScheduling._ranges_exactly_match_groups  sQ   € ð
 )/Ñ%ˆÐ%à Ð ò Ü�KÓ ¤C¨£KÑ/òäñ ä),¨[¸&Ó)Aôó ð	
rd   c                ó.  — |j                   }t        ||«      }t        t        j                  «       ||||¬«      }	t        j
                  |	«      5  |j                  |«      5   |||d¬«       d d d «       d d d «       y # 1 sw Y   ŒxY w# 1 sw Y   y xY w)N)rq   rœ  rR  rK  T)Úallow_same_symbol_in_index)r%  rE  rI  r?   Úget_ops_handlerÚset_ops_handlerÚset_current_node)
rs   rq   r  rœ  ré  rê  rB  r=  rK  Úhandlers
             rb   rM  z)SIMDScheduling._codegen_grouped_reduction  s—   € ð "3×!8Ñ!8ÐÜ1°&¸&ÓAˆÜ-Ü×ÑÓØØØ(Ø)ô
ˆô ×Ñ˜wÓ'ñ 	¨×)@Ñ)@ÐARÓ)Sñ 	Ù"ØØØ+/õ÷	÷ 	÷ 	ñ 	ú÷ 	ð 	ús$   ÁBÁ"A?Á.BÁ?B	ÂBÂBc                ó¨   ‡— |j                   }|j                  Št        |«      t        ‰«      k(  sJ ‚	 	 	 	 dˆfd„}t        j	                  |||«      S )Nc                 ób   •— t        ‰| «      D ��cg c]  \  }}t        ||«      ‘Œ c}}S c c}}w r\   )rÙ  r3   )rñ  rA  rè  rÉ   s      €rb   Úsplit_values_by_rangeszRSIMDScheduling._map_iteration_values_to_node_sizes.<locals>.split_values_by_ranges5  s7   ø€ ô
 &)¨°Ó%<÷á!�E˜6ô    vÕ.óð ùó s   ‘+)rñ  rÁ  r†   rÆ  )r  rÉ   r  r„   r  )r  rS  rÝ   r  rc  rÉ   s        @rb   Ú#_map_iteration_values_to_node_sizesz2SIMDScheduling._map_iteration_values_to_node_sizes+  s`   ø€ ð —‘ˆØ—‘ˆÜ�6‹{œc &›kÒ)Ð)Ð)ð	Ø-ð	à#õ	ô ×9Ñ9ØØØ"ó
ð 	
rd   c                ó˜   — g |j                   ¢|j                  ¢}|D �cg c]  }|j                  |   ‘Œ }}t        |g |¢|¢«      S c c}w r\   )r  Úreduce_varsrn   rÎ  )rs   re  ré  rê  Ú	body_varsri  r  s          rb   r*  z0SIMDScheduling._local_reduction_iteration_valuesC  sV   € ð 9�d—n‘nÐ8 t×'7Ñ'7Ð8ˆ	Ø.7Ö8¨�$—/‘/ !Ó$Ð8ˆÐ8Ü˜vÐ'I¨Ð'I¸Ð'IÓJÐJùò 9s   ŸArJ  c          	     ó²  — |j                  |«      5  |D ]“  }| j                  ||j                  «       «      \  }}|rJ ‚t        t	        j
                  «       |||¬«      }	t	        j                  |	«      5  |j                  |«      5  |j                  |«       ddd«       ddd«       Œ• 	 ddd«       y# 1 sw Y   ŒxY w# 1 sw Y   ŒµxY w# 1 sw Y   yxY w)zÇEmit pointwise nodes under an explicit nested iteration family.

        `source` describes the coordinate space whose values are remapped into
        each pointwise node's own loop ranges.
        )rq   rR  rK  N)	rÝ  rd  rr  ra  r?   r]  r^  r_  r%  )
rs   rq   Úpointwise_nodesrR  rS  rK  r  r  Úreduction_varsr`  s
             rb   r+  z*SIMDScheduling._codegen_remapped_pointwiseM  sÜ   € ð �_‰_˜VÓ$ñ 	(Ø%ò (�Ø,0×,TÑ,TØØ—M‘M“Oó-Ñ)�	˜>ñ *Ð)Ð)Ü0Ü×%Ñ%Ó'Ø!Ø!Ø#1ô	�ô ×&Ñ& wÓ/ñ (°×1HÑ1HÈÓ1Lñ (Ø—H‘H˜YÔ'÷(÷ (ð (ñ(÷	(ð 	(÷(ð (ú÷ (ð (ú÷	(ð 	(ús<   ’A#CÁ5CÂB5	ÂCÂ!
CÂ5B>Â:CÃC
ÃCÃCc                ób  — | j                   sJ ‚|D �cg c]+  }|j                  «       | j                   j                  vsŒ*|‘Œ- }}|sy t        |d„ ¬«      j                  \  }\  }}| j                  |||«      }t        j                  d|«       | j                  t        ||||«      «      S c c}w )Nc                ó4   — t        | j                  «       «      S r\   ©r÷   rw   ©rX   s    rb   rê   z/SIMDScheduling._codegen_nodes.<locals>.<lambda>v  s   € ´c¸!¿.¹.Ó:JÓ6K€ rd   rë   zSchedule:
 %s)
r   r¥  rù  r±  r  r³  Úschedule_logÚdebugÚcodegen_node_schedulerL   )rs   r™   r?  rË   r/  ro   rQ  rM  s           rb   r  zSIMDScheduling._codegen_nodesk  s±   € ð
 �~Š~Ðˆ~à"ö
Ø d§m¡m£o¸T¿^¹^×=WÑ=WÒ&WŠDð
ˆð 
ñ ØÜ  Ñ,KÔL×RÑRÑˆ‰?ˆE�6à×3Ñ3°E¸5À&ÓIˆÜ×ÑÐ+¨]Ô;à×)Ñ)Ü˜}¨e°VÐ=NÓOó
ð 	
ùò
s
   “+B,¿B,c                ó  — | j                   sJ ‚|j                  «       D �cg c]*  }|j                  «       | j                   j                  vr|‘Œ, }}t	        |«      dk(  ryt
        j                  j                  j                  j                  rdt	        |«      t	        j                  «       «      k7  r.| j                   sJ ‚t        j                  | j                   |«      }|j                  «       }nd}| j                  ||«      S c c}w )zI
        Given a set of pre-fused nodes, generate a SIMD kernel.
        r   N)r   r~  r¥  rù  r  r]   r^   r   r_   r&  ÚFusedSchedulerNoder'  r  )rs   rË   r™   r?  s       rb   Úcodegen_nodezSIMDScheduling.codegen_node  sß   € ð �~Š~Ðˆ~ð Ÿ™Ó(ö
àØ�}‰}‹ d§n¡n×&@Ñ&@Ñ@ò ð
ˆð 
ô
 ˆu‹:˜Š?Øä�?‰?×!Ñ!×(Ñ(×AÒAÜ�5‹zœS §¡Ó!1Ó2Ò2Ø—~’~Ð%�~Ü ×3Ñ3°D·N±NÀEÓJ�Ø $× :Ñ :Ó <Ñà $Ðà×"Ñ" 5Ð*;Ó<Ð<ùò!
s   ¡/Dc                óà  — t        j                  t         j                  «      j                  }t	        | «      sy|D �cg c]0  }|j                  «       r|j                  «       j                  «       ‘Œ2 }}|D ]}  }|j                  «       rŒt        |t        j                  «      sŒ/|j                  «       }||D �cg c]0  }|j                  «       r|j                  «       j                  «       ‘Œ2 c}z  }Œ t        d„ |D «       «      syt        j                  j                  j!                  | |«       |D ],  }t        j                  j                  j!                  ||«       Œ. yc c}w c c}w )NFc              3  ó2   K  — | ]  }t        |«      –— Œ y ­wr\   )r4   )r  r  s     rb   r  z8SIMDScheduling.can_use_32bit_indexing.<locals>.<genexpr>³  s   è ø€ ÒF°DÔ)¨$×/ÑFùrœ  T)r]   ÚiinfoÚint32r±  r4   Úhas_tensor_outputr�  Ústorage_sizer  r   ÚMutationOutputÚget_mutation_buffersr  r?   rÏ   rÐ   Ú	check_leq)ro   ÚbuffersÚint_maxr›  Ú	buf_sizesÚmutated_bufsr  s          rb   Úcan_use_32bit_indexingz%SIMDScheduling.can_use_32bit_indexing˜  s=  € ô
 —+‘+œeŸk™kÓ*×.Ñ.ˆä% eÔ,Øð ö
àØ×$Ñ$Ô&ð �N‰NÓ×)Ñ)Õ+ð
ˆ	ð 
ð ò 	ˆCØ×(Ñ(Õ*¬z¸#¼r×?PÑ?PÕ/QØ"×7Ñ7Ó9�Øà+öàØ×,Ñ,Ô.ð —N‘NÓ$×1Ñ1Õ3òñ ‘	ð	ô ÑF¸IÔFÔFØô 	
�‰×Ñ×"Ñ" 5¨'Ô2Øò 	6ˆDÜ�G‰G×Ñ×&Ñ& t¨WÕ5ð	6àùò/
ùòs   ¾5E&Â;5E+Fc                óŠ  — | j                  ||«       |sIt        j                  |«      5  t        j                  |«      D ]  }|j                  «        Œ 	 ddd«       t        j                  xj                  |j                  z  c_        t        j                  xj                  |j                  z  c_        y# 1 sw Y   ŒdxY w)ze
        Process a kernel by generating code for its node schedule and updating graph state.
        N)	rÁ  r?   rÄ  rK   rr  rý  rÏ   rú  r   )rs   rq   rM  Úonly_gen_src_coderË   s        rb   Úprocess_kernelzSIMDScheduling.process_kernel½  s•   € ð 	×.Ñ.¨}¸fÔEÙ Ü×%Ñ% fÓ-ñ $Ü.×9Ñ9¸-ÓHò $�DØ—M‘M•Oñ$÷$ô 	
�‰×Ò 6×#9Ñ#9Ñ9ÕÜ	�‰×"Ò" f×&?Ñ&?Ñ?Ö"÷	$ð $ús   ª+B9Â9Cc                ó°   — i }|D ]N  }t        |t        «      sŒ|j                  €Œ!|j                  j                  «       }|sŒ>|j	                  |«       ŒP |S )zÔCollect and merge config_patches from all operations in the node schedule.

        This enables scoped config (e.g., coordinate_descent_tuning) for kernels
        that contain decomposition operations.
        )r  r/   rË   Úget_config_patchesr¢  )rs   rM  Úmerged_patchesrË   Úpatchess        rb   rG  z&SIMDScheduling._collect_config_patchesÎ  sW   € ð *,ˆØ!ò 	3ˆDÜ˜$Ô 1Õ2°t·y±yÑ7LØŸ)™)×6Ñ6Ó8�ÚØ"×)Ñ)¨'Õ2ð		3ð
 Ðrd   c                óæ  — |j                   }| j                  ||j                  |j                  |j                  «      \  }}| j                  ||g||dœ«      }|D ]  }| j                  ||«       Œ t        j                  |«       | j                  |«      }|D ]Ž  }t        j                  |«      5  t        j                  di |¤Ž5  |j                  «       }ddd«       ddd«       | j                  ||«      }	t         j#                  d|	«       |	|_        t'        |«      |_        Œ� ~t)        |«      dkD  rt        |«      }
n|\  }
t        j                  |
«      5  |j+                  «       D ]  }|j-                  «        Œ 	 ddd«       |D �cg c]  }t/        |t0        «      sŒ|‘Œ }}| j3                  |
|d¬«       t        j4                  j6                  j8                  rÐt        j:                  rÀ|d   j<                  j?                  «       }|j+                  «       D ]�  }|jA                  «       }||vrŒ|jB                  €J ‚|jB                  jE                  «       }|€ŒCtF        d   d	xx   dz  cc<   t        j4                  j6                  jI                  d
|jJ                  ›d|› d�«       Œ’ | jM                  «        y# 1 sw Y   �ŒðxY w# 1 sw Y   �ŒõxY w# 1 sw Y   �Œ]xY wc c}w )z<
        Generate code for nodes in kernel_features
        ©rR  rP  Nz+Generating kernel code with kernel_name: %sr@   F©Úfree_buffersr   ÚinductorÚintermediate_hookszrun_intermediate_hooks(r    r  r·   )'rM  r(  ro   r  r?  rÀ  rÁ  rF   Úmerge_workspaces_inplacerG  r?   rÄ  r   rÆ  r·  rû  r’  rp  rY  r   r  rü  rý  r  r/   rH  rÏ   rW  Úsupports_intermediate_hooksÚgenerate_intermediate_hooksr  Úlive_output_buffersr¥  rË   Úget_origin_noder   r  rl   r  )rs   rÉ  rM  rO  r@  Úkernelsrq   rJ  rÏ  rY  Úfinal_kernelrË   rK  Ú	live_outsrl   Úorigin_nodes                   rb   rq  z$SIMDScheduling.codegen_node_scheduleÜ  sÕ  € ð (×5Ñ5ˆà#×9Ñ9ØØ×!Ñ!Ø×+Ñ+Ø×-Ñ-ó	 
Ñˆ�ð ×,Ñ,ØØˆHØ(¸<ÑHó
ˆð
 ò 	JˆFØ×2Ñ2°=À&ÕIð	Jä×,Ñ,¨WÔ5ð ×5Ñ5°mÓDˆàò 	3ˆFÜ×%Ñ% fÓ-ñ 3¬v¯|©|Ñ/M¸nÑ/Mñ 3Ø!×0Ñ0Ó2�÷3÷ 3à×,Ñ,¨X°}ÀfÓMˆKÜ�I‰IÐCÀ[ÔQØ!,ˆFÔÜ(¨Ó2ˆFÕð	3ð ô ˆw‹<˜!ÒÜ& wÓ/‰Là%‰Oˆ\ä×!Ñ! ,Ó/ñ 	 Ø'×7Ñ7Ó9ò  �Ø—‘•ñ ÷	 ð +ö9
Ø¬j¸Ô?PÕ.QŠDð9
Ðð 9
ð 	×'Ñ'ØØ Øð 	(ô 	
ô �G‰G× Ñ ×<Ò<Ü×2Ò2ð   ™
Ÿ™×;Ñ;Ó=ˆIØ'×7Ñ7Ó9ò 
�Ø—}‘}“�Ø˜yÑ(ØØ—y‘yÐ,Ð,Ð,Ø"Ÿi™i×7Ñ7Ó9�ØÑ*Ü˜ZÑ(Ð)=Ó>À!ÑCÓ>Ü—G‘G×(Ñ(×2Ñ2Ø1°+×2BÑ2BÐ1EÀRÈÀvÈQÐOõð
ð 	×&Ñ&Õ(÷_3ñ 3ú÷ 3ñ 3ú÷	 ñ 	 üò
9
s<   Â2KÃKÃKÅ&K!ÆK.Æ(K.ËKËKËK	Ë!K+TrŒ  c               óR  — | j                  ||j                  «       t        j                  j                  r\t
        j                  j                  j                  «        t
        j                  j                  j                  |j                  |«       |j                  |j                  «       t        j                  j                  r(t
        j                  j                  j                  «        t        j                  r|j                  «        t        j                  r|j                  |j                  «       t
        j                  xj                  |j                  z  c_        t
        j                  xj                   |j                   z  c_        |r| j#                  «        y y r\   )rÿ  rY  r   ÚcppÚenable_kernel_profiler?   rÏ   rW  Ú write_kernel_context_guard_beginÚwrite_kernel_context_guardr_  Úwrite_kernel_context_guard_endÚnan_assertsrR  r£  rú  r   r  )rs   rq   rK  r�  s       rb   rH  z)SIMDScheduling._launch_kernel_and_cleanup'  s  € ð 	×ÑÐ1°6×3EÑ3EÔFÜ�:‰:×+Ò+Ü�G‰G× Ñ ×AÑAÔCÜ�G‰G× Ñ ×;Ñ;Ø×"Ñ"Ø$ôð 	×Ñ˜6×-Ñ-Ô.Ü�:‰:×+Ò+Ü�G‰G× Ñ ×?Ñ?ÔAä×ÒØ×$Ñ$Ô&Ü×!Ò!Ø×"Ñ" 6×#5Ñ#5Ô6ä	�‰×Ò 6×#9Ñ#9Ñ9ÕÜ	�‰×"Ò" f×&?Ñ&?Ñ?Õ"áØ×*Ñ*Õ,ð rd   c                ó(   —  | j                   |i |¤ŽgS r\   )rh  )rs   rÉ  Úkernel_argsrA  s       rb   rÀ  z$SIMDScheduling.create_kernel_choicesD  s)   € ð ˆD×ÑØðàñð
ð 	
rd   c           	     óf  — |5  t        j                  «       }i }|D ]½  }|t        u r |j                  |j	                  «       «       Œ+|t
        u r|j                  «        ŒD|j                  «        |j                  |j                  «       «      }|j                  t        j                  |j                  j                  |«      j                  «       «      «       Œ¿ |j!                  |j#                  «       «       |D ]�  }|t        u r |j                  |j	                  «       «       Œ+|t
        u r|j                  «        ŒDt%        |j                  «       t'        |j                  «       |j                  |j                  «       «      }|j)                  |«       ŒŸ 	 d d d «       y # 1 sw Y   y xY wr\   )rä  rÃ  rH   rÅ  ræ  rI   ÚcloseÚdecide_inplace_updater  rr  r¢  rN  Úfromkeysr%  Úindexing_from_argsrÉ   r¿  Úkeysr(   r'   r  )rs   rM  rq   rÎ  Úall_indexingrË   rî   s          rb   rÁ  z0SIMDScheduling.codegen_node_schedule_with_kernelN  s_  € Øñ  	-Ü×(Ñ(Ó*ˆEØˆLð &ò �ØÔ+Ñ+Ø×'Ñ'¨×(@Ñ(@Ó(BÕCØœ_Ñ,Ø—K‘K•Mà×.Ñ.Ô0Ø!'×!<Ñ!<¸T¿_¹_Ó=NÓ!O�JØ ×'Ñ'ÜŸ™Ø ŸJ™J×9Ñ9¸*ÓE×LÑLÓNóõðð ×$Ñ$ \×%6Ñ%6Ó%8Ô9ð &ò 
-�ØÔ+Ñ+Ø×'Ñ'¨×(@Ñ(@Ó(BÕCØœ_Ñ,Ø—K‘K•Mô 6°d·j±jÔAÜ4°T·Z±ZÔ@Ø!'×!<Ñ!<¸T¿_¹_Ó=NÓ!O�JØ—L‘L Õ,ñ
-÷- 	-÷  	-ñ  	-ús   ƒFF'Æ'F0©r„  c          	     óJ  — i }|j                  «       }g }	|D ]  }
|
j                  «       }|	j                  |
«       ||z  sŒ*t        |«      dk(  sJ ‚|	|t	        t        |«      «      <   |j                  j                  t	        t        |«      «      «       g }	Œ� t        |	«      dk(  sJ ‚|j                  D ](  }|j                  j                  j                  |d«       Œ* |j                  | ||||t        |«      }t        j                  rH|j                  «       dz  }|j!                  «       › d|› d|j#                  |«      j%                  «       › �}g |¢|‘|¢}|r|S t'        |j)                  «       D �cg c]  }t+        |«      ‘Œ c}«      }t-        j.                  |«      5  |j1                  «        |D ]   }t+        |«      |vsŒ|j1                  «        Œ" |D ]  }|j1                  «        Œ 	 ddd«       | j3                  |||«      |_        |S c c}w # 1 sw Y   Œ(xY w)zK
        Helper method to codegen a single template kernel variant
        r@   r   Ng    eÍÍAr‹  )r€  r�  rÖ   r  rÓ   ÚiterÚprologue_fused_inputsrð   r  rŒ  r«  rŽ  r   r   r¾  r†  Úimports_for_benchmark_kernelÚcodegen_kernel_benchmarkÚgetvaluer   r�  Úidr?   rÄ  rý  rû  rY  )rs   rq   r�  rˆ  r‰  rŠ  r„  r‹  Útemplate_readsÚprologue_groupÚprologueÚnamesÚbuf_namerÏ  Únum_gbrM  rõ   Úunfused_setrË   s                      rb   Ú_codegen_single_templatez'SIMDScheduling._codegen_single_templateq  s2  € ð &(Ð"Ø&×8Ñ8Ó:ˆØˆØ&ò 	$ˆHØ×-Ñ-Ó/ˆEØ×!Ñ! (Ô+à�~Ó%Ü˜5“z Q’Ð&�Ø@NÐ*¬4´°U³Ó+<Ñ=Ø×,Ñ,×0Ñ0´´d¸5³kÓ1BÔCØ!#‘ð	$ô �>Ó" aÒ'Ð'Ð'ð ×4Ñ4ò 	:ˆHØ�K‰K×%Ñ%×)Ñ)¨(°DÕ9ð	:ð ×/Ñ/ØØØØØ&Ü(Øó
ˆô ×"Ò"Ø×5Ñ5Ó7¸#Ñ=ˆFà×6Ñ6Ó8Ð9¸Ø�*˜BØ×2Ñ2°6Ó:×CÑCÓEÐFðHð ð J˜.ÐI¨-ÐI¸.ÐIˆáØˆOô !°×1MÑ1MÓ1OÖ!P¨A¤" Q¥%Ò!PÓQˆÜ×!Ñ! &Ó)ñ 	 Ø×"Ñ"Ô$Ø&ò $�Ü�d“8 ;Ò.Ø—M‘M•Oð$ð 'ò  �Ø—‘•ñ ÷	 ð "×/Ñ/°¸-ÈÓPˆÔàˆùò "Q÷	 ð 	 ús   Å6HÆ$#HÇ)HÈH"c                ó  ‡‡— ddl mŠ ˆfd„Šg }t        |j                  «      |gz   D ]S  }t	        |t        t
        f«      r$|j                  t        ˆfd„|D «       «      «       Œ=|j                   ‰|«      «       ŒU t        |«      S )Nr   r$   c                ó¼   •— t        | ‰«      sy t        | t        j                  «      r| j                  «       } | j	                  «       x}€y t        d„ |D «       «      S )Nc              3  ó    K  — | ]  }|–— Œ y ­wr\   r·   r  s     rb   r  zKSIMDScheduling._get_multikernel_shapes.<locals>.get_size.<locals>.<genexpr>È  s   è ø€ Ò)˜qœÑ)ùs   ‚)r  r   ÚBaseViewÚunwrap_viewÚmaybe_get_sizer¹   )r   r  r%   s     €rb   Úget_sizez8SIMDScheduling._get_multikernel_shapes.<locals>.get_sizeÁ  sR   ø€ Ü˜c 6Ô*ØÜ˜#œrŸ{™{Ô+Ø—o‘oÓ'�Ø×*Ñ*Ó,Ð,�Ð5ØÜÑ) DÔ)Ó)Ð)rd   c              3  ó.   •K  — | ]  } ‰|«      –— Œ y ­wr\   r·   )r  Ú_argr¿  s     €rb   r  z9SIMDScheduling._get_multikernel_shapes.<locals>.<genexpr>Í  s   øè ø€ Ò @°D¡¨$§Ñ @ùs   ƒ)r   r%   rL  Úinputsr  r¹   rÖ   )rs   rË   Úoutr   r%   r¿  s       @@rb   Ú_get_multikernel_shapesz&SIMDScheduling._get_multikernel_shapes¼  st   ù€ õ 	 ô	*ð ˆÜ˜Ÿ™Ó$¨ vÑ-ò 	*ˆCÜ˜#¤¤e˜}Ô-Ø—
‘
œ5Ó @¸CÔ @Ó@ÕAà—
‘
™8 C›=Õ)ð		*ô
 �S‹zÐrd   c                óH   — | j                  |«      }t        d„ |D «       «      S )Nc              3  ó@   K  — | ]  }t        d „ |D «       «      –— Œ y­w)c              3  óŠ   K  — | ];  }t        |t        j                  «      xr t        |t        j                  «       –— Œ= y ­wr\   ©r  r�   ÚExprr  r  s     rb   r  zFSIMDScheduling._kernel_has_dynamic_shapes.<locals>.<genexpr>.<genexpr>Õ  s9   è ø€ ò àô ˜1œeŸj™jÓ)ÒN´*¸QÄÇÁÓ2NÐ.NÓNñùs   ‚AAN)r-  )r  r»   s     rb   r  z<SIMDScheduling._kernel_has_dynamic_shapes.<locals>.<genexpr>Ô  s,   è ø€ ò 
ð
 ô	 ñ àô÷ ñ
ùs   ‚)rÄ  r-  )rs   rË   Úshapess      rb   Ú_kernel_has_dynamic_shapesz)SIMDScheduling._kernel_has_dynamic_shapesÒ  s.   € Ø×-Ñ-¨dÓ3ˆÜñ 
ð
  ô
ó 
ð 	
rd   c                óN   ‡— | j                  |«      }t        ˆfd„|D «       «      S )zk
        Returns cache key for hint-based multi-graph; key is tuple of shapes with hint filled in.
        c              3  óF   •K  — | ]  }t        ˆfd „|D «       «      –— Œ y­w)c              3  óŽ   •K  — | ]<  }t        |t        j                  «      rt        |t        j                  «      s‰n|–— Œ> y ­wr\   rÈ  )r  rô   Úhints     €rb   r  zASIMDScheduling._make_shape_cache_key.<locals>.<genexpr>.<genexpr>ä  s@   øè ø€ ò ð ô ˜a¤§¡Ô,´ZÀÄ5Ç=Á=Ô5Qñ àóñùs   ƒAANrk  )r  r»   rÏ  s     €rb   r  z7SIMDScheduling._make_shape_cache_key.<locals>.<genexpr>ã  s/   øè ø€ ò 
ð ô ó ð ô	÷ ñ
ùr*  )rÄ  r¹   )rs   rË   rÏ  rÊ  s     ` rb   Ú_make_shape_cache_keyz$SIMDScheduling._make_shape_cache_keyÜ  s1   ø€ ð ×-Ñ-¨dÓ3ˆÜó 
ð  ô
ó 
ð 	
rd   ©r„  Úhint_overridec          	     ó¢  — |j                   \  }\  }}|dk(  sJ ‚t        |j                  t        «      �r|j                  j                  �röt        |j                  j                  «      dkD  �rÓ| j                  |j                  «      �r·i }	g }
|j                  j                  j                  «       D ]˜  \  }} ||j                  |¬«      \  }}|r;| j                  |||||d¬«      }t        |t        «      sJ ‚|
j                  |«       ŒZ|€Œ]| j                  |||||d¬«      }|€dn| j                  |j                  |«      }||	|<   Œš |rdj                  |
«      S t        j                  t        |	j!                  «       «      «       t#        |	«      }g |¢|‘|¢}| j%                  ||j&                  «       |j)                  |j&                  «       t*        j,                  xj.                  |j.                  z  c_        t*        j,                  xj0                  |j0                  z  c_        | j3                  «        y|j                  j5                  |j                  |¬«      \  }}|r| j                  |||||d¬«      S | j                  |||||d¬«      }g |¢|‘|¢}| j%                  ||j&                  «       |j)                  |j&                  |j                  «       t*        j,                  xj.                  |j.                  z  c_        t*        j,                  xj0                  |j0                  z  c_        | j3                  «        y)zÆ
        Codegen a triton template with multi-kernel dispatch support

        If `only_gen_src_code=True` the src code will be returned instead of being
        codegenned into the wrapper
        r@   )rÒ  Tr©  NFz

)r  r  rË   r   Ú_make_kernel_rendersr  rË  r}   r¸  r€   rÖ   rÐ  r5  rF   r�  rL  rÉ   rG   rÿ  rY  r_  r?   rÏ   rú  r   r  Úmake_kernel_render)rs   rˆ  r‰  rŠ  r„  rÒ  r/  Ú_numelrQ  r•  Ú	src_codesÚ	size_hintrÕ  rq   r�  rÏ  Úshape_cache_keyÚmulti_kernelrM  s                      rb   Úcodegen_templatezSIMDScheduling.codegen_templateí  sD  € ð  ,×1Ñ1ÑˆÑˆF�FØ˜Š{Ðˆ{ô �}×)Ñ)Ô+>Õ?Ø×"Ñ"×7Ó7Ü�M×&Ñ&×;Ñ;Ó<¸qÓ@Ø×/Ñ/°×0BÑ0BÕCàˆGØˆIð
 ×#Ñ#×8Ñ8×>Ñ>Ó@ò%6ñ ØØ"á!3Ø!×&Ñ&°mô"‘�˜ñ %Ø#×<Ñ<ØØØ%Ø&Ø&Ø*.ð  =ó  �Hô & h´Ô4Ð4Ð4à×$Ñ$ XÕ.à Ð(Ø Ø!×:Ñ:ØØØ%Ø&Ø&Ø*/ð ;ó �Fð %Ð,ñ à!×7Ñ7¸×8JÑ8JÈIÓVð $ð 06�G˜OÒ,ðK%6ñN !Ø—{‘{ 9Ó-Ð-ä×0Ñ0´°g·n±nÓ6FÓ1GÔHÜ.¨wÓ7ˆLØM˜nÐM¨mÐM¸nÐMˆMØ× Ñ  °×0HÑ0HÔIØ×$Ñ$ \×%=Ñ%=Ô>Ü�G‰G×#Ò# |×'CÑ'CÑCÕ#Ü�G‰G×&Ò&¨,×*IÑ*IÑIÕ&Ø×*Ñ*Ô,Øà*×/Ñ/×BÑBØ×"Ñ"°-ð Có ‰NˆF�Fñ !Ø×4Ñ4ØØØ!Ø"Ø"Ø&*ð 5ó ð ð ×6Ñ6ØØØ!Ø"Ø"Ø&+ð 7ó �ð !R .Ð Q°-Ð QÀ.Ð Q�Ø×$Ñ$ ]°F×4FÑ4FÔGØ×"Ñ" 6×#5Ñ#5°}×7IÑ7IÔJä—‘×'Ò'¨6×+AÑ+AÑAÕ'Ü—‘×*Ò*¨f×.GÑ.GÑGÕ*Ø×.Ñ.Ô0Ørd   c                ó¢   — t         j                  j                  j                  t         j                  j                  j                  «       «       y r\   )r?   rÏ   rW  r  Ú
device_opsÚsynchronizerv   s    rb   Úcodegen_synczSIMDScheduling.codegen_sync]  s-   € Ü	�‰×Ñ×&Ñ&¤q§w¡w×'9Ñ'9×'EÑ'EÓ'GÕHrd   c                óv  — i }| j                   j                  |j                  |«        | j                   |j                  f|j                  |j                  dœ|¤Ž}| j                  ||j                  |«       t        j                  |«      5  |j                  «       }d d d «       ||fS # 1 sw Y   |fS xY w)Nr‹  )
rh  Ú apply_feature_required_overridesrR  rO  rP  r…  rM  r?   rÄ  r·  )rs   Ú	node_infor„  rA  rq   rÏ  s         rb   Ú_codegen_standalone_kernelz)SIMDScheduling._codegen_standalone_kernel`  sÃ   € ð )+ˆØ×Ñ×9Ñ9Ø×Ñ ô	
ð "�×!Ñ!Ø×Ñð
à×'Ñ'Ø#×1Ñ1ñ
ð ñ	
ˆð 	×Ñ˜F I×$;Ñ$;Ð=NÔOÜ×!Ñ! &Ó)ñ 	/Ø×,Ñ,Ó.ˆH÷	/à˜ÐÐ÷	/à˜ÐÐús   ÂB,Â,B8c                óê  — ddl m}m}m} ddlm} t        dd¬«      }| j                  j                  |j                  |«        | j                  |j                  f|j                  |j                  dœ|¤Ž}t        j                  d¬	«      5  t        j                  |«      5  | j!                  |j"                  |«       |j%                  «       }d d d «       d d d «       |j&                  j)                  «       D �	�
ci c]X  \  }	}
t+        |	«      r|j,                  r<|	t/        t1        t        j2                  j4                  j7                  |
«      «      «      “ŒZ }}	}
i |j9                  «       ¥|j;                  «       ¥}|j                  r> |||j                  j=                  |j                  «      |j>                  |d¬
«      }||fS |j,                  r> |||j                  j=                  |j                  «      |j>                  |d¬
«      }||fS |j@                  jC                  «       \  }}}} |||j>                   |||«      |d¬«      }||fS # 1 sw Y   �Œ†xY w# 1 sw Y   �Œ‹xY wc c}
}	w )Nr   )ro  Ú	pointwiser¥  r@   )Úselect_tile_hintTF)Úis_combo_kernelr‚  r‹  r½  )Úreduction_hintÚtriton_metaÚinductor_metaÚreturn_configs)ré  Ú	tile_hintrê  rë  )"Úruntime.triton_heuristicsro  rå  r¥  Útriton_utilsræ  rN  rh  rá  rR  rO  rP  r   rÆ  r?   rÄ  rÁ  rM  r·  rh  r}   r8   rk  r-   r÷   rÏ   rÐ   rå   Úinductor_meta_commonÚinductor_meta_per_kernelÚget_reduction_hintré  r  ry  )rs   râ  ro  rå  r¥  ræ  rA  rq   r/  rp   ro   Ú
size_hintsrê  ÚconfigsÚ	signatures                  rb   Ú_probe_subkernel_heuristicz)SIMDScheduling._probe_subkernel_heuristicr  s`  € ÷	
ñ 	
õ
 	3ä(,Ø Ø+0ô)
ˆð 	×Ñ×9Ñ9Ø×Ñ ô	
ð "�×!Ñ!Ø×Ñð
à×'Ñ'Ø#×1Ñ1ñ
ð ñ	
ˆô �\‰\¨5Ô1ñ 	(´1×3GÑ3GÈÓ3Oñ 	(Ø×2Ñ2°9×3JÑ3JÈFÔSØ×%Ñ%Ó'ˆA÷	(÷ 	(ð "(§¡×!4Ñ!4Ó!6÷
á�˜Ü& vÔ.°&×2IÒ2Ið ”O¤C¬¯©×(8Ñ(8×(JÑ(JÈ5Ó(QÓ$RÓSÑSð
ˆ
ñ 
ð

Ø×)Ñ)Ó+ð
à×-Ñ-Ó/ð
ˆð ×&Ò&Ù*ØØ%Ÿ™×AÑAÀ&×BVÑBVÓWØ"×.Ñ.Ø+Ø#ôˆGð0 ˜ˆÐð# ×$Ò$ÙØØ%Ÿ™×AÑAÀ&×BVÑBVÓWØ"×.Ñ.Ø+Ø#ôˆGð  ˜ˆÐð "(§¡×!;Ñ!;Ó!=ÑˆAˆq�)˜QÙØØ"×.Ñ.Ù*¨:°yÓAØ+Ø#ôˆGð ˜ˆÐ÷M	(ñ 	(ú÷ 	(ñ 	(üó
s+   ÂI"Â$-IÃI"Ã>AI/ÉI	ÉI"É"I,c                ó(  ‡‡— dd l }d	d„}d
d„Št        t        t        | «      «      ˆˆfd„¬«      }| |   }t	        | «      D ���ci c]0  \  }} ||«      D ]   }|› d|› �t        |j                  |   «      “Œ" Œ2 }	}}}|	j                  |j                  j                  «       D ��
ci c]  \  }}
|j                  d«      s||
“Œ c}
}«       |j                  |	t        |j                  «      t        |j                  «      ¬«      S c c}}}w c c}
}w )Nr   c                ód   — | j                   D �cg c]  }|j                  d«      sŒ|‘Œ c}S c c}w )Nr£   )ÚkwargsÚendswith)Úcfgrh  s     rb   Ú
block_keysz@SIMDScheduling._stitch_no_bench_combo_config.<locals>.block_keysµ  s%   € Ø"Ÿz™zÖA˜!¨Q¯Z©Z¸Õ-@’AÒAÐAùÒAs   �-¦-c                óž   — t        d„ t        j                  | j                  j                  | j                  j
                  «      D «       «      S )Nc              3  ó^   K  — | ]%  }t         j                  j                  |d ¬«      –— Œ' y­w)T)Úcount_bytesN)r?   rÏ   Úget_dep_size_hint)r  Úds     rb   r  zYSIMDScheduling._stitch_no_bench_combo_config.<locals>.node_total_bytes.<locals>.<genexpr>¹  s,   è ø€ ò àô —‘×)Ñ)¨!¸Ð)×>ñùs   ‚+-)r�  ri  rÉ  rœ  r�  r¦  )Úsnodes    rb   Únode_total_byteszFSIMDScheduling._stitch_no_bench_combo_config.<locals>.node_total_bytes¸  sB   € Üñ ä"Ÿ™Ø×%Ñ%×+Ñ+¨U×->Ñ->×-EÑ-Eóôó ð rd   c                ó   •—  ‰‰|    «      S r\   r·   )r“  Úchosen_nodesr  s    €€rb   rê   z>SIMDScheduling._stitch_no_bench_combo_config.<locals>.<lambda>Æ  s   ø€ Ñ*¨<¸©?Ó;€ rd   rë   r/  r£   )Ú	num_warpsÚ
num_stages)rú  r   r†   z	list[str])r  r   r†   r÷   )r_   r±  Úranger  r­  r÷   rø  r¢  r}   rù  ÚConfigr  r  )Úchosen_configsr  r_   rû  Ú
winner_idxÚ
winner_cfgr“  rú  rì   Ústitched_kwargsrA  r  s    `         @rb   Ú_stitch_no_bench_combo_configz,SIMDScheduling._stitch_no_bench_combo_config¯  s)  ù€ ó 	ó	Bó	ô Ü”#�nÓ%Ó&Ü;ô
ˆ
ð $ JÑ/ˆ
ô $ NÓ3÷+
ð +
á��3Ù! #“ò+
ð ð ˆe�1�Q�CˆLœ#˜cŸj™j¨™oÓ.Ñ.ð+
Øð+
ˆò +
ð
 	×Ñð #-×"3Ñ"3×"9Ñ"9Ó";÷á�C˜Ø—|‘| GÔ,ð �U‘
óô	
ð �}‰}ØÜ˜*×.Ñ.Ó/Ü˜:×0Ñ0Ó1ð ó 
ð 	
ùô+
ùós   Á5DÂ'D
c           
     ó~  — ddl m} ddlm} t	        | j
                  |«      sJ ‚|D �	cg c]  }	|	j                  «       ‘Œ }
}	i }t        ||
«      D �]A  \  }}t        |d„ ¬«      j                  \  }\  }}| j                  |||«      }d}|ržt        j                  j                  j                   j                  r=t        |t         j"                  t         j$                  f«      sJ ‚|j'                  «       }nd}t)        ||||¬«      }| j+                  ||||j,                  «      \  }}n t)        |||«      }| j/                  |||«      }|j1                  «       xr! t2        j4                  j7                  |d¬	«      }t9        |||||||¬
«      ||<   �ŒD |j;                  || ||¬«      }t<        j?                  dtA        |«      |D �cg c]  }tA        |«      ‘Œ c}«       g }|D �]J  }tA        |«      dk(  rŒtA        |«      dk(  rI||d      }|r|jC                  dd|f«       Œ@| jE                  ||«      \  }}|jC                  |||f«       Œj|xr | xr | }g }g } g }!|r³|D ]�  }| jG                  ||   «      \  }"}#t        jH                  jJ                  r|"xr |#jL                   }$n|"xr tA        |"«      dk  }$|$r&|jC                  |«       | jC                  |"d   «       Œ€|!jC                  |«       Œ’ tA        |«      dk  rg }g } tO        |«      }!ntO        |«      }tA        |«      dk\  rÍ || j
                  |||¬«      }|D ]s  }||   }|jQ                  |jR                  |jT                  | | j
                  |jV                  |¬«      }%| jY                  |j[                  |%«      |j\                  |«       Œu |r| r| j_                  | |«      |_0        |jc                  «       }|jC                  |||f«       |!D ]/  }| jE                  ||   |«      \  }&}'|jC                  |&|'|gf«       Œ1 �ŒM |S c c}	w c c}w )al  
        Generate kernel code for combo kernel partitions.

        Partitions subkernel_nodes using horizontal_partition(), then generates
        kernel code for each partition. Single-node partitions are generated as
        regular kernels, while multi-node partitions use ComboKernel.

        Returns a list of (src_code, kernel, node_group) tuples.
        r@   rP   )ÚComboKernelc                ó4   — t        | j                  «       «      S r\   rm  rn  s    rb   rê   z;SIMDScheduling.generate_combo_kernel_code.<locals>.<lambda>ø  ó   € ¼#¸a¿n¹nÓ>NÓ:O€ rd   rë   N)r?  F)rm  )rM  rO  rP  ro   rQ  rR  rS  )r™   Útriton_schedulingÚcustom_algorithmÚnode_info_mapz1ComboKernels: %d nodes partitioned into %s groupsr   r   )Útriton_kernel_clsÚenable_autotuneÚmixed_sizesÚper_subkernel_blocks)rR  Úoptimize_maskr  rP  r  )2r_   rQ   Útriton_combo_kernelr  Ú
issubclassrh  r~  rÙ  r±  r  r³  r]   r^   r   r&  r  r   rs  rv  r'  rL   r(  r?  r}  rw   r?   Úchoicesrn  rK  Úhorizontal_partitionr’  rp  r  rÖ   rã  rõ  ÚversionÚhipÚautotune_hintsrL  Úcreate_triton_kernelrO  rR  rP  r…  Úcreate_sub_kernelrM  r  Úno_bench_stitched_configr·  )(rs   Úsubkernel_nodesÚcustom_part_algorithmr  r  r„  r  rQ   r  rË   Úfused_node_listsÚnode_schedule_mapÚpnr™   r/  ro   rQ  rM  rP  r?  rR  rO  rS  Ú
partitionsÚpÚkernel_code_listÚ
node_grouprâ  rÏ  rq   Úno_bench_modeÚ
fusion_pnsÚfusion_configsÚcarve_out_pnsró  Úprobe_kernelÚfuse_okÚ	subkernelÚco_src_codeÚ	co_kernels(                                           rb   Úgenerate_combo_kernel_codez)SIMDScheduling.generate_combo_kernel_codeÝ  sÎ  € õ$ 	)Ý4ô ˜$×*Ñ*¨LÔ9Ð9Ð9à9HÖI°˜DŸN™NÕ,ÐIÐÐIØ13ÐÜ˜_Ð.>Ó?ó &	‰IˆB�Ü!$ UÑ0OÔ!P×!VÑ!VÑˆA‰��vØ ×7Ñ7¸¸uÀfÓMˆMØ ˆMÙ#Ü—?‘?×)Ñ)×0Ñ0×IÒIÜ%ØœY×9Ñ9¼9×;RÑ;RÐSôð ð ð )+×(@Ñ(@Ó(BÑ%à(,Ð%Ü-Ø! 5¨&ÐDUô�ð )-×(BÑ(BØ!ØØØ×.Ñ.ó	)Ñ%�™ô .¨m¸UÀFÓK�Ø×+Ñ+¨M¸5À&ÓI�à×%Ñ%Ó'ò Ü—I‘I×=Ñ=Ø°Eð >ó ð $ô %-Ø+ØØ+ØØØ!Ø(?ô%Ð˜bÓ!ð=&	ðP !×5Ñ5Ø!Ø"Ø2Ø+ð	 6ó 
ˆ
ô 	�	‰	Ø?Ü�Ó Ø'Ö(˜ŒS��VÒ(ô	
ð
 ÐØ$ó U	LˆJÜ�:‹ !Ò#Øä�:‹ !Ò#à-¨j¸©mÑ<�	Ù$à$×+Ñ+¨T°4¸Ð,DÕEà'+×'FÑ'FØ!Ð#4ó(Ñ$�H˜fð %×+Ñ+¨X°v¸zÐ,JÕKð )ò .Ø+Ð+ò.à-Ð-ð ð
 )+�
Ø,.�Ø+-�Ù Ø(ò 5˜Ø04×0OÑ0OØ-¨bÑ1ó1Ñ-˜ ô !Ÿ=™=×,Ò,Ø&-Ò&Q°l×6QÑ6QÐ2Q™Gà&-Ò&C´#°g³,À!Ñ2C˜GÙ"Ø&×-Ñ-¨bÔ1Ø*×1Ñ1°'¸!±*Õ=à)×0Ñ0°Õ4ð5ô ˜:“¨Ò*Ø%'˜
Ø)+˜Ü(,¨ZÓ(8™ä!% jÓ!1�Jä�z“? aÒ'Ù(Ø*.×*:Ñ*:Ø(7Ø$/Ø-Aô	�Fð )ò ˜Ø$5°bÑ$9˜	Ø$/×$DÑ$DØ%×,Ñ,Ø%.×%7Ñ%7Ø.9¨/Ø.2×.>Ñ.>Ø*3×*AÑ*AØ1Eð %Eó %˜	ð ×+Ñ+Ø"×4Ñ4°YÓ?Ø%×3Ñ3Ø-õðñ  %©à ×>Ñ>Ø .°
óð Ô7ð  &×4Ñ4Ó6�Hà$×+Ñ+¨X°v¸zÐ,JÔKà'ò L�BØ-1×-LÑ-LØ)¨"Ñ-Ð/@ó.Ñ*�K ð %×+Ñ+¨[¸)ÀbÀTÐ,JÕKòLðaU	Lðn  Ðùò[ Jùòf )s   ©P5ÇP:c                óÔ  — |j                  «       }|j                  }|j                  }t        j                  dkD  xs t        j                  dk(  xr |}|j
                  }| j                  |||||¬«      }|D ]]  \  }}	}
| j                  ||g|	«      }| j                  |j                  |«       t        j                  d|«       |	j                  |«       Œ_ | j                  «        y )Nr@   )r  z"ComboKernels: generated kernel %s.)Úget_subkernel_nodesÚuse_custom_partition_algor  r   Úcombo_kernel_allow_mixed_sizesr  r6  rû  rÿ  Úsnodesr’  rp  r_  r  )rs   Úcombo_kernel_noder$  r%  r  r  r  r+  rÏ  rq   r/  rY  s               rb   Úcodegen_combo_kernelz#SIMDScheduling.codegen_combo_kernel„  sø   € Ø+×?Ñ?ÓAˆØ 1× KÑ KÐØ+×;Ñ;ˆÜ×;Ñ;¸aÑ?ò 
Ü×1Ñ1°QÑ6ÒPÐ;Pð 	ð  1×EÑEÐà×:Ñ:ØØ!ØØØ!5ð ;ó 
Ðð $4ò 	,ÑˆH�f˜aØ×,Ñ,¨XÐ8IÐ7JÈFÓSˆKØ× Ñ Ð!2×!9Ñ!9¸;ÔGÜ�I‰IÐ:¸KÔHØ×Ñ˜{Õ+ð		,ð 	×&Ñ&Õ(rd   é    c           
     ó¢  ‡ ‡‡
— ‰dk(  }dˆ ˆˆ
fd„}|j                  «       \  }Š
t        |«      dk  rt        ‰
«      dk  st        |‰
z   «      rg S |j                  «       \  }Š
 |||r|n‰
|j                  |«      «      }|D �cg c]?  }t	        ‰ j                  |j                  |‰«      |j                  |j                  ¬«      ‘ŒA }	}|	S c c}w )Nr@   c                ód  •— t        |j                  «      t        |«      k(  sJ d|j                  ›d|›�«       ‚|j                  |j                  g}t	        d„ t
        j                  j                  |«      D «       «      sJ ‚t
        j                  j                  |«      D �cg c]:  }|j                  t        j                  j                  vrt        |t        «      r|‘Œ< }}t        |j                  D �cg c]  }|j                  ‘Œ c}«      }dd„}t        ‰j!                   ||«      g| «      dd¬«      g}|D �]Ô  }t        j                  j"                  j%                  |j&                  |j                  «      }	t        |	«      t        |«      k(  sJ ‚	 |	j'                  d«      dz   }
|
t        |«      k(  rŒt	        d	„ |	|
d
 D «       «      rŒ•	  ||d
|
 «       |||
d
 «      f}t        j                  j"                  j+                  t-        d„ t/        ||	«      D «       «      «      }|j                  |v r|dz  }t        j1                  |d   «      r|dz  }t        j1                  |d   «      r|dz  }t        j                  j"                  j+                  |t-        t        j                  |‰«      «      z
  «      dk\  s�ŒŠ|j3                  t        ‰j!                   ||d
|
 «       |||
d
 «      g‰«      ||j                  ¬«      «       �Œ× |S c c}w c c}w # t(        $ r Y �Œðw xY w)zX
            Compute tiling candidates by dividing up the iteration ranges.
            zrw.range_vars=z ranges=c              3  óH   K  — | ]  }t        |t        t        f«      –— Œ y ­wr\   )r  r    r!   )r  r„  s     rb   r  zHSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.<genexpr>®  s$   è ø€ ò àô ˜3¤¬GÐ 4×5ñùs   ‚ "c                óf   — t         j                  j                  j                  t	        | «      «      S r\   r  )rè  s    rb   Úcollapse_rangeszNSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.collapse_rangesº  s"   € Ü—w‘w×'Ñ'×0Ñ0´¸vÓ1FÓGÐGrd   Únoner   )rO  rl   Úscorer@   c              3  ó&   K  — | ]	  }|d k(  –— Œ y­wrì  r·   r  s     rb   r  zHSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.<genexpr>Ð  s   è ø€ Ò; a˜1 �6Ñ;ùs   ‚Nc              3  ó2   K  — | ]  \  }}|d k7  sŒ|–— Œ y­wrì  r·   )r  r  r‘  s      rb   r  zHSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.<genexpr>ß  s    è ø€ ò "Ù!-  vÈÐSTËœñ"ùs   ‚�r   ©rO  rE  rl   )rè  rÁ  r†   rƒ   )r  Ú
range_varsr�  r¦  r  ri  rÉ  rÊ  rl   r?   rÏ   rú  r  r    r   ÚCandidateTilingÚcreate_partial_tilingrÐ   Ústride_hintsr˜   Ú
ValueErrorrå   r:   rÙ  Úis_good_sizerÖ   )Úis_pointwiserè  ÚrwÚdep_sourcesr„  ÚdepsÚwrite_namesrC  Útilingsrl  ÚsplitÚtiled_groupsrE  r  r  rZ  s                €€€rb   Útile_rangesz5SIMDScheduling.candidate_tilings.<locals>.tile_ranges£  s  ø€ ô �r—}‘}Ó%¬¨V«Ò4ÐS¸¸¿¹Ð8HÈ	È&ÈÐ6SÓSÐ4ð Ÿ8™8 R§Y¡YÐ/ˆKÜñ ä$Ÿ?™?×8Ñ8¸ÓEôô ð ð ô %Ÿ?™?×8Ñ8¸ÓEöàØ—8‘8¤1§7¡7×#:Ñ#:Ñ:Ü˜s¤IÔ.ò ðˆDð ô %¸"¿)¹)Ö%D°3 c§h£hÒ%DÓEˆKóHô
  Ø×4Ñ4Ù(¨Ó0Ð1°<óð  ØôðˆGð ó 4�ÜŸ'™'×*Ñ*×7Ñ7¸¿	¹	À2Ç=Á=ÓQ�Ü˜7“|¤s¨6£{Ò2Ð2Ð2ð
Ø#ŸM™M¨!Ó,¨qÑ0�EØ¤ F£Ò+Ø ÜÑ;¨7°5°6¨?Ô;Ô;ð !ð <ñ $ F¨6¨E NÓ3Ù# F¨5¨6 NÓ3ð �ô Ÿ™×(Ñ(×:Ñ:Ü!ñ "Ü14°V¸WÓ1Eô"ó ó�ð
 —8‘8˜{Ñ*à˜Q‘J�EÜ"×/Ñ/°¸Q±Ô@Ø˜Q‘J�EÜ"×/Ñ/°¸Q±Ô@Ø˜Q‘J�Eô —G‘G×$Ñ$×6Ñ6Ø¤¬i¯o©o¸fÐFVÓ.WÓ XÑXóð ôð
 —N‘NÜ'Ø#&×#<Ñ#<á$3°F¸6¸E°NÓ$CÙ$3°F¸5¸6°NÓ$Cð!"ð !0ó$ð #(Ø!$§¡ô
öðQ4ðl ˆNùò[ùò &Eøô: "ò Úðús$   Â$?LÃ8LÆ"L"Æ=L"Ì"	L/Ì.L/rH  )rO  r‰   r†   úlist[CandidateTiling])	rr  r  r   Ú"pointwise_or_reduction_read_writesrJ  Úcomplete_partial_tilingrO  rE  rl   )r  rË   ro   r  rO  rW  Úpointwise_rangesÚpartial_tilingsrO  Úfull_tilingsrZ  s   `  `      @rb   Úcandidate_tilingsz SIMDScheduling.candidate_tilingsž  sð   ú€ ð '¨!Ñ+ˆ÷\	ð| .2¯_©_Ó->Ñ*ÐÐ*äÐ Ó! QÒ&ÜÐ$Ó%¨Ò*Ü$Ð%5Ð8HÑ%HÔIàˆIð .2¯_©_Ó->Ñ*ÐÐ*Ù%ØÙ ,ÑÐ2BØ×3Ñ3°LÓAó
ˆð *ö	
ð ô Ø×2Ñ2Ø—M‘M 5¨/óð —l‘lØ—[‘[öð	
ˆð 	
ð Ðùò	
s   ÂACc           	     óÖ   — d}d}t        |«      t        |«      k  sJ ‚t        |«      t        |«      k  sJ ‚t        g t        |t        |«       d |d¬«      ¢t        ||d¬«      ¢«      S )zK
        Create a tiling dict from pointwise and reduction splits.
        )rV   rW   rX   )rY   rZ   NF)Ústrict)r  r   rÙ  )r  Ú	pw_tilingÚreduction_tilingÚpw_prefixesÚreduction_prefixess        rb   Úcreate_tilingzSIMDScheduling.create_tiling  s„   € ð &ˆØ+ÐÜ�9‹~¤ [Ó!1Ò1Ð1Ð1ÜÐ#Ó$¬Ð,>Ó(?Ò?Ð?Ð?äðÜ�[¤# i£. Ð!2Ð3°YÀuÔMðäÐ'Ð)9À%ÔHðó
ð 	
rd   c                ó>   — | j                  |r|ng |s|«      S g «      S r\   )re  )r  rO  rO  s      rb   rK  z$SIMDScheduling.create_partial_tiling2  s0   € ð × Ñ Ù"‰F¨Ù&ˆFó
ð 	
à,.ó
ð 	
rd   c                ó–   — t        |j                  «       «      }d|v }||z  }|t        |«      z  g}|r||fn||f} | j                  |Ž S )zb
        Given a tiling for only pointwise or reduction dimensions, adds the missing one.
        rX   )rL  rÉ   r:   re  )	r  rO  ro   r  ÚsplitsrO  Útotal_numelÚmissing_tilingÚtiling_argss	            rb   rZ  z&SIMDScheduling.complete_partial_tiling=  sf   € ô �f—m‘m“oÓ&ˆØ˜f�}ˆà˜oÑ-ˆØ%¬°fÓ(=Ñ=Ð>ˆñ )5ˆV�^Ñ$¸>È6Ð:Rð 	ð !ˆs× Ñ  +Ð.Ð.rd   c                ó   ‡‡‡‡— 	 	 	 	 	 	 dˆˆfd„Š	 	 d	ˆˆfd„}‰dk(  }t        t        t        t        j                  f      «       }t        j                  |«      D �]  }t        |t        j                  «      sŒ|j                  «       }|st        |d   «      dk(  rŒC ‰|d   ‰«      }	 ‰|d   ‰«      }
|	|
fg}|j                  j                  «       D �cg c],  }t        |t        «      rt        |j                  «      dkD  r|‘Œ. }}|D �]  Šg ‰j                  j!                  «       ¢}t        j"                  j$                  }t&        j(                  j*                  }d}t-        |«      D ]#  \  }\  }}||z  }|}|j/                  |‰«      sŒ# n |j1                  |‰«      sŒš|dz   }|d| }|rdn||d } |||d   ‰«      }|rt        j"                  j$                  fn |||d   ‰«      }t        |«      sŒðt        |«      dkD  sŒÿ|j3                  ||f«       �Œ t5        j6                  t9        |Ž Ž D ]&  \  }}|j;                  | j=                  ||«      «       Œ( �Œ	 t?        |t        d¬«      }|S c c}w )
zÐ
        Creates N-dimensional tiling candidates, attempting to simplify loads/stores
        by tiling the kernel into higher dimensions.

        Returns a list of tilings ranked by dimensionality.
        c                ó^  •— | s|fS t        d«      }t        j                  j                  j	                  ‰d«      r6t        j                  j                  j                  ‰d«      rt        |d«      }t        dt        | «      |z
  «      }|dz   }t        | d| «      }|ft        | |d «      z   S )zU
            Collapse dimensions to the maximum allowed number of tiles.
            r   r@   r   N)rc   r?   rÏ   rÐ   rÑ   r  rß  r±  r  r:   r¹   )ÚdimsÚfallback_numelr`   Únum_leading_dimsÚfirst_trailing_dimÚcollapsed_leading_dimÚpointwise_numelr  s         €€rb   Úcollapse_dimsz4SIMDScheduling.get_nd_tilings.<locals>.collapse_dims`  s±   ø€ ñ Ø&Ð(Ð(Ü% aÓ(ˆIÜ�w‰w×Ñ×7Ñ7Ø ôä—'‘'×"Ñ"×6Ñ6°ÈÔJô   	¨1Ó-�	Ü" 1¤c¨$£i°)Ñ&;Ó<ÐØ!1°AÑ!5ÐÜ$1°$Ð7JÐ8JÐ2KÓ$LÐ!Ø)Ð+¬e°DÐ9KÐ9LÐ4MÓ.NÑNÐNrd   c                óþ  •— g }| D ]—  \  }}t        j                  ‰j                  |«      }t        d|j	                  t
        «      |j	                  t        «      z   t        |«      «      }t        j                  ||||«      }|�|d   n|g}	|j                  |	«       Œ™ |D �
cg c]F  }
t        j                  j                  j                  |
t        j                  j                   «      s|
‘ŒH }}
 ‰||«      S c c}
w )Nr   r   )rA   Úget_subexpr_involving_symbolr˜   r±  rj  r   r   r  Úmatch_mod_div_block_exprr¼  r?   rÏ   rÐ   rÑ   r�   r‘   r’   )rn   Úranges_to_tileri  rO  Úvarro   r˜   Únum_dimsÚmatch_resultrn  Údimrt  r„  s              €€rb   Útile_var_rangesz6SIMDScheduling.get_nd_tilings.<locals>.tile_var_rangest  sõ   ø€ ð ˆFØ(ò $‘
��UÜ+×HÑHÈÏÉÐTWÓX�ô ØØ—K‘K¤Ó)¨E¯K©K¼Ó,HÑHÜ˜Ó'ó�ô  3×KÑKØ˜3  xó �ð +7Ð*B�| A’ÈÈ�Ø—‘˜dÕ#ð!$ð* "öàÜ—w‘w×'Ñ'×?Ñ?ÀÄUÇWÁWÇ[Á[ÔQò ðˆFð ñ ! ¨Ó5Ð5ùòs   Â$AC:r@   r   NT)rì   Úreverse)rn  rÁ  ro  rƒ   r†   útuple[sympy.Expr, ...])r†   r  ) r   r   r€   r�   rÉ  rI   rt  r  r   rv  rr  r  rœ  Úreads_and_writesr    rè  r}   r‘   r’   r?   rÏ   rÐ   r­  Ústatically_known_geqrÑ   rÖ   ri  ÚproductrÙ  rð   re  rN  )r  rM  rs  r  r}  rO  rT  rË   Únode_rangesÚdefault_pointwise_tilingÚdefault_reduction_tilingÚnode_tilingsr„  Úmemory_depsÚall_var_rangesÚpointwise_vars_numelrÐ   Úpointwise_end_idxr¯  Ú_varro   Úreduction_start_idxÚpointwise_var_rangesÚreduction_var_rangesÚpointwise_tilingrb  Úranked_tilingsrt  s     ``        `              @rb   Úget_nd_tilingszSIMDScheduling.get_nd_tilingsR  sé  û€ ð	OØ&ð	OØ8Bð	Oà#ö	Oð(	6à#ö	6ð@ '¨!Ñ+ˆÜœ^¬C´·±¨OÑ<Ñ=Ó?ˆÜ#×*Ñ*¨=Ó9ó F	SˆDÜ˜d¤I×$;Ñ$;Ô<Øð Ÿ/™/Ó+ˆKÙ¤C¨°A©Ó$7¸1Ò$<Øñ (5°[À±^À_Ó'UÐ$Ù'4°[À±^À_Ó'UÐ$Ø5Ð7OÐPÐQˆLð  ×+Ñ+×<Ñ<Ó>öàÜ˜c¤9Ô-´#°c·j±j³/ÀAÒ2Eò ðˆKð ð
 #ó (N�ð "7 3§:¡:×#3Ñ#3Ó#5Ð!6�Ü',§w¡w§{¡{Ð$ÜŸ7™7×+Ñ+�Ø$%Ð!Ü*3°NÓ*Cò Ñ&�C™˜$ Ø(¨EÑ1Ð(Ø(+Ð%Ø×4Ñ4Ø,¨oõñ ðð  ×7Ñ7Ø(¨/ôð ð '8¸!Ñ&;Ð#Ø'5Ð6JÐ7JÐ'KÐ$á(‘D¨nÐ=PÐ=QÐ.Rð %ñ $3Ø(¨+°a©.¸/ó$Ð ñ
 $ô —W‘W—[‘[‘Ná(Ø,¨k¸!©n¸oóð !ô Ð'Õ(¬SÐ1AÓ-BÀQÓ-FØ ×'Ñ'Ð)9Ð;KÐ(LÖMðQ(NôX 7@×6GÑ6GÜ�lÐ#ð7ò SÑ2Ð Ð"2ð —‘˜C×-Ñ-Ð.>Ð@PÓQÕRòSðGF	SôT  ØÜØô
ˆð Ðùò}s   Ã1I;c                óx  ‡‡‡‡‡‡‡‡‡‡‡‡‡— ‰j                   sdn‰j                   j                  Š‰j                  j                  Š‰j                  j                  Š‰j                  j
                  }‰D �cg c]  }||   ‘Œ	 c}Š‰D �cg c]  }||   ‘Œ	 c}Št        j                  j                  j                  }t        j                   |t        ‰«      «       |‰«      k(  ˆˆˆfd„«       t        j                   |t        ‰«      «       |‰«      k(  ˆˆˆfd„«       i Šg }	 	 	 d	 	 	 	 	 	 	 dˆˆˆˆˆˆˆˆˆf	d„}	|j                   |	d¬«       |	d¬«      f«       ‰r$|j                   |	‰fdd¬«       |	d¬«      f«       ‰‰j                  j                  «       z  }
|
D ]%  }|j                   |	|fd¬«       |	d¬«      f«       Œ' t!        d	¬
«      d	k(  rB‰dk(  r=t#        j$                  |
d«      D ]$  }|j                   |	|d¬«       |	d¬«      f«       Œ& g }|D ]b  \  \  }}\  }}t'        | j)                  ||«      t+        |«      t+        |«      z   ¬«      }| j)                  ||«      }|j                  ||f«       Œd | j)                  ‰g‰g«      }dŠdŠt+        ‰j,                  j/                  «       «      Šˆˆˆfd„}t1        ||¬«      D ]Ó  \  }}| j3                  ‰‰‰|j4                  «      s|j4                  |k(  r‚t7        |j4                  «      ‰dk(  rdndz
  }|t!        d	¬
«      kD  rDt8        j;                  d|t        j<                  j>                  j@                  jB                  «       Œ¥|j4                  |fc S |j4                  |k(  sŒÅ|j4                  |fc S  |dfS c c}w c c}w )zr
        Generates a tiling, and a score of each tile according to each tile's coalesced memory accesses.
        Nc                 ó   •— ‰› d‰› d‰ › �S ©Nr    r·   )rM  rs  Ú	pw_rangess   €€€rb   rê   z8SIMDScheduling.compute_tiling_strategy.<locals>.<lambda>  s   ø€ �y�k  OÐ#4°B°}°oÐF€ rd   c                 ó   •— ‰› d‰› d‰ › �S r”  r·   )rM  Ú
red_rangesr  s   €€€rb   rê   z8SIMDScheduling.compute_tiling_strategy.<locals>.<lambda>	  s   ø€ �z�l " _Ð$5°R¸°ÐG€ rd   Fc                ó@  •	— |r‰n‰}|r‰n‰}|s|r|gg fS g g fS t        | «      ||f}‰j                  |«      x}r|S |r‰n‰}g }g }	d}
d}t        ||«      D �]  \  }}|| vr"|
|z  }
‰j                  j                  |d«      }Œ-|r§|‰k(  r¢‰j                  }|€J ‚|j
                  }t        ||j
                  «      }|j                  |
|z  «       |	j                  |j                  «       |j                  |«       |	j                  ‰j                  j                  |d«      «       d}
d}ŒÖ|
|z  }
|j                  |
«       |	j                  ‰j                  j                  |d«      «       d}
�Œ |
dk7  s|r0t        |«      dk(  r"|j                  |
«       |	j                  |«       t        t        |«      «      D ]S  }t        j                  j                  j                  ||   d¬«      }t        |d«      }t!        |	|   |z  dz  «      |	|<   ŒU ||	f‰|<   ||	fS )z]
            Generate a tiling, and a tiling score, given vars to use as splits.
            r@   r   r>  ©ÚfallbackrØ  )r@  rò   rÙ  Úcoalesced_by_varÚsuggested_splitÚtiling_factorr   rÖ   rE  r  r  r?   rÏ   rÐ   rå   rß  r÷   )Úvars_to_useÚuse_split_varrO  rè  Útarget_numelrì   rÃ  Úsplitting_varsrh  Úsplit_scoresÚprodÚprev_var_coalesced_scoreri  Úv_rangeÚ
var_tilingÚtileÚ	remainderr“  rô   Úall_iter_varsÚall_red_varsr?  rs  r•  r—  r  Úscored_sub_splitÚ
tiling_vars                      €€€€€€€€€rb   Úprocess_node_varszASIMDScheduling.compute_tiling_strategy.<locals>.process_node_vars  sX  ø€ ñ #/‘Y°JˆFÙ.:™?ÀˆLáÙØ)˜N¨BÐ/Ð/à ˜8�Oä˜Ó$ m°\ÐBˆCØ&×*Ñ*¨3Ó/Ð/ˆsÐ/Ø�
á.:™]ÀˆNàˆFØˆLØˆDØ'(Ð$ô " .°&Ó9ó ‘
��7Ø˜KÑ'Ø˜G‘O�DØ/@×/QÑ/Q×/UÑ/UØ˜1ó0Ð,ð á  Q¨*¢_Ø!2×!BÑ!B�JØ%Ð1Ð1Ð1à%×3Ñ3�DÜ (¨°*×2JÑ2JÓ K�Ià—M‘M $¨Ñ"2Ô3Ø ×'Ñ'¨
×(8Ñ(8Ô9à—M‘M $Ô'Ø ×'Ñ'Ð(9×(JÑ(J×(NÑ(NÈqÐRSÓ(TÔUà�DØ/0Ð,àà˜‘�Ø—‘˜dÔ#Ø×#Ñ#Ð$5×$FÑ$F×$JÑ$JÈ1ÈaÓ$PÔQØ’ð;ð> �qŠy™\¬c°&«k¸QÒ.>Ø—‘˜dÔ#Ø×#Ñ#Ð$<Ô=ô œ3˜v›;Ó'ò ?�Ü—G‘G×$Ñ$×6Ñ6°v¸a±yÈ2Ð6ÓN�Ü˜˜1“I�Ü"% l°1¡o¸Ñ&9¸AÑ&=Ó">�˜Q’ð?ð
 &,¨\Ð$:Ð˜SÑ!Ø˜LÐ)Ð)rd   T)rO  )rŸ  rO  r   ©ra   r@   r   )rE  gffffffð?g®Gázð?c                óÂ   •— d}| d   j                   j                  «       D ]"  }t        j                  |«      s|‰z  }Œ|‰z  }Œ$ ‰dz  }| d   j                  |z    |z  S )Ng      ð?r   gš™™™™™©?)rO  rÉ   rJ  rN  rE  )rI  Úscore_factorÚ	tile_sizeÚuncoalesced_penaltyÚ"bad_size_additional_tiling_penaltyÚgood_size_tiling_penaltyÚtotal_uncoalesceds       €€€rb   Ú	score_modz9SIMDScheduling.compute_tiling_strategy.<locals>.score_modš  sw   ø€ ØˆLØ˜q™TŸ[™[×/Ñ/Ó1ò K�	Ü&×3Ñ3°IÔ>Ø#/Ð2TÑ#T‘Là#/Ð2JÑ#J‘Lð	Kð #4°dÑ":Ðà�q‘T—Z‘ZÐ"5Ñ5Ð6¸ÑEÐErd   rë   r   zmFound optimal tiling with %s tiles but torch._inductor.config.triton.max_tiles set to %s. Consider increasing)r·   FF)rž  r  rŸ  r‰   rO  r‰   r†   ztuple[list[int], list[int]])"rœ  ry  Únorm_read_writesrî   rf  rn   r?   rÏ   rÐ   rå   r]   Ú_checkr:   rÖ   r›  r§  rc   ri  ÚcombinationsrJ  re  r�  Úuncoalesced_addrsrÉ   rN  Útiling_is_compatiblerO  r  Úperf_hint_logÚinfor^   r   r_   r`   ) r  rM  rs  r  r?  rè  ri  Úget_hintÚscore_splitr­  Úoverlapping_iter_varsrž  rT  Úpw_splitÚpw_scoreÚ	red_splitÚ	red_scoreÚ	candidater@  Údefault_tilingr¶  ÚcandÚ
tiling_lenr©  rª  r³  r´  r•  r—  r«  r¬  rµ  s     ````                  @@@@@@@@@rb   Úcompute_tiling_strategyz&SIMDScheduling.compute_tiling_strategyè  sì  ÿü€ ð %×4Ò4ñ à"×2Ñ2×6Ñ6ð 	ð *×:Ñ:×EÑEˆØ(×9Ñ9×EÑEˆØ"×3Ñ3×>Ñ>ˆà(5Ö6 1�V˜A“YÒ6ˆ	Ø)5Ö6 A�f˜Q“iÒ6ˆ
ô —7‘7×#Ñ#×5Ñ5ˆÜ�‰Ù”] 9Ó-Ó.±(¸?Ó2KÑKÝFô	
ô
 	�‰Ù”] :Ó.Ó/±8¸OÓ3LÑLÝGô	
ð DFÐð ð 	ð
 35Ø"'Ø!&ðK	*Ø/ðK	*àðK	*ð ðK	*ð )÷	K	*õ K	*ð\ 	×Ñá!¨tÔ4Ù!¨uÔ5ðô	
ñ Ø×Ñá%Ø#˜°TÈôñ &°5Ô9ð	ôð Ð-×>Ñ>×CÑCÓEÑEð 	ð 'ò 	ˆAØ×Ñá% q d¸Ô>Ù%°5Ô9ðõð	ô  Ô# qÒ(¨_ÀÒ-AÜ(×5Ñ5Ð6KÈQÓOò �Ø×"Ñ"á)¨+ÀDÔIÙ)°uÔ=ðõðð RTˆØ<Gò 	6Ñ8Ñ ˆX�xÑ"8 9¨iÜ'Ø×!Ñ! (¨IÓ6Ü˜(“m¤c¨)£nÑ4ôˆIð ×,Ñ,¨X°yÓAˆLØ�N‰N˜I |Ð4Õ5ð	6ð ×*Ñ*¨OÐ+<¸Ð>OÓPˆð .3Ð*Ø#(Ð äÐ 1× CÑ C× JÑ JÓ LÓMÐö	Fô #)¨°iÔ"@ò 	1ÑˆD�,à×(Ñ(Ø! ?°OÀTÇ[Á[ôð —;‘; .Ò0ô ! §¡Ó-°oÈÒ6J±ÐPQÑR�
Ø¤°aÔ 8Ò8Ü!×&Ñ&ð9à"ÜŸ™×.Ñ.×5Ñ5×?Ñ?ô	ð à—{‘{ LÐ0Ò0ð �{‰{˜nÓ,Ø—{‘{ LÐ0Ò0ð/	1ð2 ˜tÐ#Ð#ùòK 7ùÚ6s   Á8N2ÂN7c                óT   ‡‡— t        ‰t        «      sJ ‚t        ˆˆfd„|D «       «      S )Nc              3  óº   •K  — | ]R  }t        |t        j                  «      r6t        j	                  ‰j                  «       |j                  «       ‰¬ «      –— ŒT y­wrq  )r  r   rv  r„   r  rÉ   rr  )r  rË   r  rO  s     €€rb   r  z6SIMDScheduling.tiling_is_compatible.<locals>.<genexpr>Í  sO   øè ø€ ò 
ð Ü˜$¤	× 7Ñ 7Ô8ô	 ×$Ñ$Ø—‘“ §¡Ó!2ÀOð %÷ ñ
ùs   ƒAA)r  rN  r  )r  rM  ro   r  rO  s      ``rb   r»  z#SIMDScheduling.tiling_is_compatibleÄ  s1   ù€ ô ˜&¤$Ô'Ð'Ð'Üô 
ð &ô	
ó 
ð 	
rd   c                óB   — |D ]  }| j                  ||||«      sŒ|c S  y r\   )r»  )r  rM  ro   r  r�  rO  s         rb   Úget_first_compatible_tilingz*SIMDScheduling.get_first_compatible_tilingÕ  s1   € ð %ò 	ˆFØ×'Ñ'¨°u¸oÈvÕVØ’ð	ð rd   c                ó0   — | j                  ||||«      d   S rÌ  )r(  )r  rM  ro   r  r?  s        rb   r}  zSIMDScheduling.select_tilingã  s)   € ð ×(Ñ(Ø˜5 /Ð3Dó
à
ñð 	rd   c                ó  — |dk(  }| j                  |g|g«      }t        j                  |«      D ]‘  }t        |j                  t
        j                  «      sŒ(|j                  j                  «       dk(  sŒFt        j                  j                  sŒa|j                  «       }|d   }	|d   }
| j                  |	|
«      }|dfc S  t        j                  j                  j                  j                  r0|r.t        j                  j                  s| j!                  ||||«      S |st        j                  j"                  rt%        d¬«      dk  r¦t&        j(                  t*        j,                  k  r�t        j                  |«      D ]i  }t        j                  j"                  rŒt/        | j1                  |||«      «      dkD  sŒ>t&        j3                  t5        j6                  d«      «        |dfS  |dfS t9        «       }t;        j<                  «       }t        j                  |«      D ]g  }| j1                  |||«      D ]O  }|j>                  |v rŒ|j>                  �|jA                  |j>                  «       ||xx   |jB                  z  cc<   ŒQ Œi |jE                  «       D ��cg c]  \  }}|jF                  ‘Œ }}}t%        d¬«      dk\  r?|r=	 	 	 	 	 	 dd	„}tI        dt/        |«      «      D ]  } ||d   ||   «      }|€Œ|g|z   } n t/        |«      dkD  rt&        j3                  d
|«       t        j                  j                  r| jK                  |||«      |z   }| jM                  ||||«      x}r|dfS |dfS c c}}w )zÓ
        Heuristics to decide how to tile kernels.
        Currently, we tile based on stride-1 dimensions.

        Returns:
            `(tile1, tile2, reduction_numel)` s.t. `tile1 * tile2 == numel`

        r@   r[  r   Nr   r®  z½
                                Reduction over non-contiguous dims.
                                Consider setting config.triton.tile_reductions to True.
                                r   c                ó   — | d   | j                  dd«      }}|d   |j                  dd«      }}t        j                  j                  j                  } |||z
  «      dk(  ry  |||z
  «      dk  r||f||fc\  }}\  }} |||z
  «      dkD  sJ ‚t        j                  j                  j                  ||«      sy |t        ||«      || d   dœ}|S )NrX   rW   r@   r   rY   )rV   rW   rX   rY   )rò   r?   rÏ   rÐ   rÿ  rï  r   )Útiling0rŒ  Úa0Úa1Úb0Úb1rÏ  Ú
new_tilings           rb   Úconvert_tiling_to_3dzBSIMDScheduling.get_tiling_and_scores.<locals>.convert_tiling_to_3dK  så   € ð ! ™ w§{¡{°3¸Ó':�B�Ø  ™ w§{¡{°3¸Ó':�B�ô —w‘w×'Ñ'×>Ñ>�Ù˜˜R™“= AÒ%ØÙ˜˜R™“= 1Ò$à*,¨b¨°B¸°8Ð&‘H�R˜™h˜r 2á˜B ™G“} qÒ(Ð(Ð(Ü—w‘w×'Ñ'×DÑDÀRÈÔLØð Ü! " bÓ)ØØ" 5™>ñ	�
ð "Ð!rd   zpossibly bad tiling: %s)rÑ  r»  rŒ  r»  r†   r½  )'re  rI   rt  r  rË   r   rw  rx  r   r_   ru  rr  r]   r^   r&  Úprefer_nd_tilingrÉ  Útile_reductionsrc   r¼  ÚlevelÚloggingÚWARNINGr  r^  r½  ÚtextwrapÚdedentr   Úcollectionsr   rl   rð   rE  Úmost_commonrO  r  r‘  rÍ  )r  rM  ro   r  r?  rO  rÆ  rË   rƒ  Ú	range_y_xÚrange_rrO  Ú
seen_namesÚcandidate_tilesÚcandidate_tilingrE  r�  r×  r“  Únew_3d_tilings                       rb   r(  z$SIMDScheduling.get_tiling_and_scoresï  s�  € ð" '¨!Ñ+ˆð ×*Ñ*¨E¨7°_Ð4EÓFˆô $×*Ñ*¨=Ó9ò 	(ˆDÜ˜$Ÿ)™)¤R×%6Ñ%6Õ7à—I‘I×0Ñ0Ó2°eÓ;ÜŸ™×3Ó3ð #'§/¡/Ó"3�KØ +¨A¡�IØ)¨!™n�GØ ×.Ñ.¨y¸'ÓB�FØ! 4˜<Ò'ð	(ô  �O‰O×"Ñ"×)Ñ)×BÒBÙ!Ü—M‘M×2Ò2à×.Ñ.Ø˜u oÐ7Hóð ñ ¤V§]¡]×%BÒ%BÄ}ØôH
àòHô ×"Ñ"¤g§o¡oÒ5Ü+×2Ñ2°=ÓAò �Dä"ŸM™M×9Ó9Ü × 5Ñ 5°d¸EÀ?Ó SÓTÐWXÓXä%×*Ñ*Ü$ŸO™Oð!$óôð à! 4Ð'Ð'ðð " 4Ð'Ð'ä&0£lˆ
Ü4?×4GÑ4GÓ4IˆÜ#×*Ñ*¨=Ó9ò 	LˆDØ$'×$9Ñ$9¸$ÀÀÓ$Wò LÐ Ø#×(Ñ(¨JÑ6ØØ%×*Ñ*Ð6Ø—N‘NÐ#3×#8Ñ#8Ô9ØÐ 0Ó1Ð5E×5KÑ5KÑKÔ1ñLð	Lð ,;×+FÑ+FÓ+H÷7
á'Ð  %ð ×#Ó#ð7
ˆñ 7
ô
  Ô# qÒ(©\ð"Ø.ð"Ø9Nð"à-ó"ô@ ˜1œc .Ó1Ó2ò �Ù 4Ø" 1Ñ% ~°aÑ'8ó!�ð !Ñ,Ø&3 _°~Ñ%E�NÙðô ˆ~Ó Ò"Ü×ÑÐ8¸.ÔIô �=‰=×)Ò)à×"Ñ" =°%¸ÓIØ ñ!ð ð
 ×4Ñ4Ø˜5 /°>ó
ð 
ˆ6ð 
ð ˜4�<Ðà˜tÐ#Ð#ùóI7
s   Ê"M;c                 ó   — y r\   r·   rv   s    rb   ÚflushzSIMDScheduling.flush…  r¹  rd   c                 ó   — yrÂ   r·   rv   s    rb   Úready_to_flushzSIMDScheduling.ready_to_flushˆ  rÄ   rd   c                ó  — t        d„ |D «       «      s×t        |d„ ¬«      j                  \  }\  }}| j                  |||«      }| j	                  |||«      }| j                  |t        |||«      ¬«      }	| j                  ||	«       | j                  |«      }
||
d<   t        j                  d
i |
¤Ž5  t        j                  |	«      5  |	j                  «       }d d d «       d d d «       nM|d   j                  |«      \  }}}t        j                  d|«      5  | j                  |||d|¬«      }d d d «       j!                  t#        t$        j&                  «      d	«      }|S # 1 sw Y   ŒŠxY w# 1 sw Y   Œ@xY w# 1 sw Y   ŒLxY w)Nc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wr\   )r  )r  rõ   s     rb   r  zASIMDScheduling.generate_kernel_code_from_nodes.<locals>.<genexpr>Ž  s   è ø€ Ò2 q�1—=‘=—?Ñ2ùs   ‚c                ó4   — t        | j                  «       «      S r\   rm  rn  s    rb   rê   z@SIMDScheduling.generate_kernel_code_from_nodes.<locals>.<lambda>�  r  rd   rë   )rR  r¾  r   TrÑ  r¿  r·   )r-  r±  r  r³  r}  rh  rL   rÁ  rG  r   rÆ  r?   rÄ  r·  Úget_prologue_template_epiloguerÛ  rÇ  r€   r7   rÈ  )rs   r™   r¾  rÒ  r/  ro   rQ  rM  rO  rq   rJ  rÏ  r³  ÚtemplateÚepilogues                  rb   Úgenerate_kernel_code_from_nodesz.SIMDScheduling.generate_kernel_code_from_nodes‹  s’  € ô Ñ2¨EÔ2Ô2Ü!$ UÑ0OÔ!P×!VÑ!VÑˆA‰��vØ ×7Ñ7¸¸uÀfÓMˆMØ×'Ñ'¨°u¸fÓEˆFØ×%Ñ%ØÜ+¨M¸5À&ÓIð &ó ˆFð ×2Ñ2°=À&ÔIà!×9Ñ9¸-ÓHˆNØ1AˆNÐ-Ñ.ä—‘Ñ.˜~Ñ.ñ3ä×$Ñ$ VÓ,ñ3ð "×0Ñ0Ó2�÷	3÷ 3ð 3ð ,1°©8×+RÑ+RØó,Ñ(ˆH�h ô —‘Ð0Ð2BÓCñ Ø×0Ñ0ØØØØ&*Ø"/ð 1ó �÷ð ×#Ñ#¤C¬×(?Ñ(?Ó$@À)ÓLˆØˆ÷)3ð 3ú÷ 3ð 3ú÷ð ús0   Â2E.ÃE"ÃE.ÄE:Å"E+	Å'E.Å.E7Å:Fc                ó   — t         ‚r\   r…  )rs   rÏ  rM  rq   s       rb   rû  zSIMDScheduling.define_kernel°  r¢  rd   )rØ  N)rÔ  zOrderedSet[str] | Noner†   ztuple[float, str]r\   rþ   )r  úscheduler.SchedulerNoderœ  rð  rD  rÎ  rE  rÎ  r6  zBdict[BaseSchedulerNode, scheduler.NestedReduction.PointwiseDomain]r†   r‡   )r  ró  rE  rÎ  rD  rÎ  r†   rÎ  )r  rÁ  rè  rÅ  r†   r‰   )r  ró  rœ  rð  r†   r‡   )rS  rÎ  rÝ   rÅ  r†   rÆ  )r†   rÎ  )rR  rÐ  rS  rÎ  rK  r^  r†   r‡   )r™   z!Sequence[scheduler.SchedulerNode]r?  úCoalesceVarAnalysis | None)rË   z6scheduler.FusedSchedulerNode | scheduler.SchedulerNode)ro   rƒ   r~  zCIterable[ir.Buffer | ir.TensorBox | ir.TorchBindObject | ir.IRNode]r†   r‰   )F)rq   r   rM  úlist[NodeScheduleEntry]r„  r‰   r†   r‡   )rM  r¾  r†   zdict[str, Any])rÉ  rL   )rK  úlist[BaseSchedulerNode]r�  r‰   r†   r‡   )rÉ  rL   r†   zlist[SIMDKernel])rË   r   r†   útuple[tuple[int, ...], ...])rË   r   r†   r‰   )rË   r   rÏ  r÷   r†   r÷  )rÒ  rù   r†   ú
str | None)râ  rK  r„  r‰   r†   ztuple[str, TritonKernel])râ  rK  r†   ztuple[list[Any], Any])r	  r¾  r  r¾  r†   r   )FF)r$  rö  r%  r‰   r  r‰   r  r‰   r„  r‰   r  r‰   r†   z!list[tuple[str | None, Any, Any]])r†   rX  )ra  rÁ  rb  rÁ  r†   úimmutable_dict[str, sympy.Expr])rO  rÁ  rO  r‰   r†   rù  )rO  r»  ro   rƒ   r  rƒ   r†   rù  )r†   z%list[immutable_dict[str, sympy.Expr]])
rM  rõ  rs  rƒ   r  rƒ   r?  rR   r†   ú:tuple[dict[str, sympy.Expr], dict[str, sympy.Expr] | None])rM  rõ  ro   rƒ   r  rƒ   rO  r»  )rM  rõ  ro   rƒ   r  rƒ   r�  zlist[dict[str, sympy.Expr]])r?  rô  r†   r»  )r?  rô  r†   rú  rˆ   )FN)rÒ  rù   )CrŒ   r�   rŽ   r�   r„   rh  rI  rl  ry  Úcan_fuse_verticalr„  r³  r·  r»  rÐ  rÕ  r¶  r)  r-  r,  rP  rÊ  rU  rM  rË  rd  r*  r+  r  rt  r‚  r…  rG  rq  rH  rÀ  rÁ  r¸  rÄ  rË  rÐ  rÛ  rß  rã  rõ  r  r6  r=  r  r  r^  re  rK  rZ  r‘  rÉ  r»  rÍ  r�   r‘   r’   r}  r(  rè  rê  rñ  rû  r·   rd   rb   rg  rg  Ã  sö  … ñð
 (€K�Ó'òQòm6ð^ !ÐØ"Ðò^ò@8ò
%ò2)ðl OSð"Ø5Kð"à	ó"ò
d)òLl
ðf #ðFð 
óFð8Nð 3ð	Nð
 (ðNð !0ðNð ,ðNð#
ðNð 
óNð`
à#ð
ð ,ð
ð !0ð	
ð
 
ó
ð& ð
Ø$ð
à.ð
ð 
ò
ó ð
ðð 3ðð (ð	ð 
óð2 ð
àð
ð 0ð
ð 
 ò	
ó ð
ð.Kð
 
óKð" ;?ñ(ð (ð	(ð
  ð(ð 8ð(ð 
ó(ðB 9=ð
à0ð
ð 6ó
ð(=ØJó=ð2 ð"Øð"àTð"ð 
ò"ó ð"ðP #(ð	@àð@ð /ð@ð  ð	@ð
 
ó@ó"óI)ð` "ñ-ð 6ð-ð
 ð-ð 
ó-ð:
Ø1ð
à	ó
ò!-ðV  ôIðVØ'ðà	$óó,
ð
Ø'ð
Ø/2ð
à	$ó
ð.  Ø$(ñnð "ðnð 
ónò`Ið Ø!ð Ø6:ð à	!ó ó$;ðz ð+
Ø!ð+
Ø1:ð+
à	ò+
ó ð+
ðf #(Ø%*ðe à0ðe ð  $ðe ð ð	e ð
 ðe ð  ðe ð #ðe ð 
+óe òN)ð4 Ø€Y×Ñ˜Óò}ó ó ð}ð~ ð
Ø,ð
Ø@Tð
à	(ò
ó ð
ð$ ð
à$ð
ð ð
ð 
)ò	
ó ð
ð ð/à%ð/ð ð/ð $ð	/ð
 
)ò/ó ð/ð( ðSð
 
/òSó ðSðj ðY$à.ðY$ð $ðY$ð $ð	Y$ð
 /ðY$ð 
DòY$ó ðY$ðv ð
à.ð
ð ð
ð $ð	
ð
 &ò
ó ð
ð  ðà.ðð ðð $ð	ð
 4òó ðð ð
 Ÿ™Ÿ™Ø8<ð	ð
 6ð	ð 
ò	ó ð	ð ð
 Ÿ™Ÿ™Ø8<ðS$ð
 6ðS$ð 
DòS$ó ðS$òjóð JNð#Ø<Fó#óJ"rd   rg  c                  ó@   — e Zd ZU ded<   ded<   dZded<   ed„ «       Zy)	rJ  r»  rO  r÷   rE  Nrø  rl   c                óv   — t         j                  j                  j                  | d¬«      } | dk\  xr | dz  dk(  S )z@Somewhat arbitrary heuristic used to boost scores for some sizesi    r™  r>  r   )r?   rÏ   rÐ   rå   )rô   s    rb   rN  zCandidateTiling.is_good_sizeº  s:   € ô �G‰G×Ñ×.Ñ.¨q¸4Ð.Ó@ˆØ�B‰wÒ(˜A ™F a™KÐ(rd   )rŒ   r�   rŽ   rI  rl   rÊ  rN  r·   rd   rb   rJ  rJ  ´  s)   … à!Ó!ØƒJØ€Dˆ*Óàñ)ó ñ)rd   rJ  c                  ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )rð  c                ó>   •— t         ‰| �  «        || _        || _        y r\   )rj   rk   r×   rò  )rs   r×   rò  rt   s      €rb   rk   zCantSplit.__init__Â  s   ø€ Ü‰ÑÔØˆŒ	Ø"ˆ�rd   c                ó8   — | j                   › d| j                  › �S )Nz not divisible by )r×   rò  rv   s    rb   Ú__str__zCantSplit.__str__Ç  s   € Ø—)‘)�Ð.¨t¯~©~Ð.>Ð?Ð?rd   )rŒ   r�   rŽ   rk   r  r”   r•   s   @rb   rð  rð  Á  s   ø„ ô#ö
@rd   rð  )r   )ra   r÷   r†   r÷   )rA  rÈ  r†   r€   )•Ú
__future__r   rß  rä  rå  r  ri  rÛ  r>  r  rÝ  r   Útypingr   r   r   r   r	   Útyping_extensionsr
   r�   r]   Útorch._loggingÚtorch._inductorr   Útorch._inductor.irr   Ú%torch.fx.experimental.symbolic_shapesr   Útorch.fx.immutable_collectionsr   Útorch.utils._ordered_setr   Útorch.utils._sympy.functionsr   r   r   Útorch.utils._sympy.symbolr   r   r   r   Ú_dynamo.utilsr   Ú r   r   r   Úanalyze_preserves_zero_maskr   Ú	codecacher   r   Údependenciesr    r!   r"   Úcollections.abcr#   r%   Úops_handlerr&   Úoptimize_indexingr'   r(   Ú runtime.coordinate_descent_tunerr)   Úruntime.hintsr*   Úruntime.runtime_utilsr+   r,   r-   r.   r/   r0   r1   Úutilsr2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   Úvirtualizedr=   r>   r?   Úblock_analysisrA   ÚcommonrB   rC   rD   rE   rÚ  rF   rG   Úsimd_kernel_featuresrH   rI   rJ   rK   rL   rM   rN   rO   Útorch._inductor.codegen.tritonrQ   Útorch._inductor.tiling_utilsrR   Ú	getLoggerrŒ   r’  Ú_loggingÚgetArtifactLoggerr¼  ro  Ú
fusion_logÚdoprintrÉ  r±  rc   Ú	dataclassrf   r…   rÒ   r+  rB  rC  rF  rK  r„   rÎ  rÐ  rè  rð  rE  rI  ra  rg  rJ  Ú	Exceptionrð  r·   rd   rb   ú<module>r&     s  ðå "ã Û Û Û Û Û Û Û Û Ý ß @Õ @Ý %ã ã Û Ý #Ý 2Ý GÝ 9Ý /ß LÑ L÷ó õ &ß $Ñ $Ý Fß .ß 6Ñ 6ñ Ý(åå (÷õ =Ý ,÷ó ÷ EÑ D÷÷ ÷ ñ ÷ -Ñ ,Ý /ß PÓ Pß :÷õ ñ ß<Ñ<å;Ý@ð €g×Ñ˜Ó!€Ø—‘×0Ñ0°¸<ÓH€Ø�~‰~×/Ñ/°¸*ÓE€Ø�^‰^×-Ñ-¨h¸ÓA€
ñ 	‹×Ñ€áÒ7Ó8€ô;ð
 ×Ò÷3+ð 3+ó ð3+ôlr;˜/ô r;ôj;'˜?ô ;'ô|CÐ!4ô CóLñ Ð+°;ÈÔT€ð ×Ò÷ð ó ðô"ˆzô "ô^�˜Ñ(¨'°/Ñ*Bô ^ðB €×Ò˜dÔ#÷!ð !ó $ð!ð ×Ò÷;ð ;ó ð;ð| €×Ò˜dÔ#÷$ð $ó $ð$ð& €×Ò˜dÔ#÷Fð Fó $ðFðR
 €×Ò˜dÔ#÷

ð 

ó $ð

ô:; .ô :;ôz(F˜^ô (FôVn'"�^ô n'"ðbO €×Ò˜dÔ#÷	)ð 	)ó $ð	)ô@�	õ @rd   