^jP  UddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlmZmZddlmZmZddlmZmZmZmZmZmZmZmZddl m!Z!ddl"m#Z#dd l$m%Z%m&Z&er"dd l'm(Z(m)Z)m*Z*dd l+m,Z,dd l-m.Z.dd l/m0Z0ddl1m2Z2ddl3Z3ddl4Z4ddl5Z4ddl6m7cm8Z9ddl:m;Z;mZ>ddl?m@Z@mAZAddlBmCZCddlDmEZEmFZFddlGmHZHddlImJZJddlKmLZLddlMmNZNmOZOmPZPddlQmRZRddlSmTZTmUZUmVZVmWZWm$Z$mXZXddlYmZZZddl[m\Z\m]Z]m^Z^ddl_m`Z`maZaddlWmbZbmcZcmdZdmeZeddlfmgZgmhZhddlimjZjdd l$mkZkmlZlmmZmmnZnmoZompZpdd!lqmrZrdd"lsmtZtmuZudd#lvmwZwmxZxdd$lymzZzm{Z{m|Z|dd%l}m~Z~dd&l7mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd'lmZe j(eZe4j.j1ed(Ze4j.j1ed)Ze4j.j1ed*Ze4j.j1ed+Zed,Zd-ed.<ed/Ze!d0ZejDGd1d2ZejDGd3d4ZGd5d6eZejDd78Gd9d:Z d}d;ZGd<d=ZGd>d?ZejDGd@dAZejDGdBdCeZGdDd,ZejZd~dEZddFZ ddGZddHZejDd78GdIdJZddKZGdLdMZ ddNZGdOdPeZGdQdReZGdSdTeZ ddUZ ddWZGdXdYeZGdZd[eZGd\d]eZGd^d_eZGd`daeZGdbdceZ d dddZ ddeZddfZ ddgZ ddhZejDGdidjZejZddkZ ddlZddmZd7dn ddoZ ddpZddqZddrZddsZddtZdduZejDGdvdwZejDGdxdyZGdzdVZGd{d|Zy)) annotationsN)Counter defaultdict) as_completedFuture)AnyGenericLiteral NamedTupleoverload TYPE_CHECKING TypeAliasTypeVar) ParamSpec OrderedSet)ComputedBuffer Pointwise)CallableIteratorSequence) ModuleType)EnterCudaStreamContextLine)PythonWrapperCodegen)CoalesceVarAnalysis)counters dynamo_timed)use_pipelined_autotuning) LambdaFuture PyCodeCache)TritonTemplateCallerBase)get_metric_tableis_metric_table_enabled)get_stream_name) free_symbols)FloorDiv)free_symbol_is_typesymbol_is_typeSymT) has_triton)commsconfig config_comms dependenciesirmetrics)can_codegen_without_upcasts)BackendFeatureget_scheduling_for_deviceKernel) estimate_nccl_collective_runtime/estimate_nccl_collective_runtime_nccl_estimator)Dep MemoryDepStarDepWeakDep)GPUTooOldForTriton TritonMissing)count_flops_fx)assign_origin_nodeget_device_typeGraphPartitionSignature MultiOutputMultiOutputLayout NoneLayout)LoopBody)MemoryPlanningInfoForBufferMemoryPlanningInfoForNode)DeviceProperties ReductionHint) green_text is_power_of_2red_text)SimplifyIndexing)&_unstable_customized_partition_wrapper cache_on_selfcmpdevice_need_guardget_current_backendget_device_tflopsget_dtype_sizeget_gpu_dram_gbps get_op_namesGraphPartitionMapIndentedBuffer is_collectiveis_cudagraph_unsafe_opis_gpuis_multi_outputs_template#is_output_of_multi_outputs_templateis_wait sympy_product sympy_subs)Vfusion loop_orderingcompute_dependencies cudagraphsBaseSchedulerNoder PartitionType_T_PcleZdZUdZded<dZded<dZded<dZed d Z e d d d Z y) FusionResultNz bool | None should_fusezCallable[[], bool] | None callable_fnLambdaFuture | NonefuturecL|jdu|jduz sJdy)NzLFusion result should contain either fusion decision or callable_fn, not both)rlrmselfs d/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/scheduler.py __post_init__zFusionResult.__post_init__s0  ,1A1A1MN Z Nct|S)N)rlrk)clsrls rsfusezFusionResult.fuses  44ruct||S)Nrmrorw)rxrmros rs from_callablezFusionResult.from_callables FCCru)rlboolN)rmCallable[[], bool]rorn) __name__ __module__ __qualname__rl__annotations__rmrort classmethodryr|rursrkrk|sf#K#-1K*1"&F & 55LPD,D6IDDrurkcBeZdZUded<ded<ded<dZded<d d Zy) PendingFusionrrmrfnode1node2Nrnroc2|j|jfSr~rrrqs rsget_fusion_nodeszPendingFusion.get_fusion_nodess DJJ''ru)return+tuple[BaseSchedulerNode, BaseSchedulerNode])rrrrrorrrursrrs$##  "&F &(rurc0eZdZUdZded<ded<ded<y) _LocalEntryzOne row of the post-rewrite slice the gate builds. `cur` is the step the node currently wants to run at, `baseline` is its original index (used to break ties when sorting), and `node` is the slice member itself. intcurbaselinerfnodeN)rrr__doc__rrrursrrs HM rurT)slotscreZdZUdZded<ded<dZded<dZded <eje Z d ed <y )ComboKernelMemoryContextaShared state used by the memory-aware combo gate. Candidate windows are evaluated independently against the original schedule. Earlier accepted combos only contribute through `running_peak`, which caps the cumulative peak drift from the original graph. OrderedSet[str] graph_outputsdict[BaseSchedulerNode, int] node_to_idxrr baseline_peak running_peakdefault_factory list[int]baseline_live_beforeN) rrrrrrr dataclassesfieldlistrrrursrrsE#"--M3L#'8k&7&7&M)Mrurc|jr|jsy|jj}|dvxrt|dk(S)NF)cudaxputriton)r[ get_devicetyperR)rr device_types rs_is_gpu_triton_backendrsI <<>""$))K&W+>{+Kx+Wruc$eZdZdZeddZeddZe ddZeddZ e ddZ e ddZ eddZ e dd Z edd Ze dd Zedd Zedd Zy)MixOrderReductionz This class contains utility functions to decide if we should fuse reductions reducing across different dimensions of the same input tensor. cf|jxr td|jDS)Nc3K|]V}t|trD|jr4t|jtr|jj duXywr~) isinstance SchedulerNode is_reductionrr _split_size.0subnodes rs z7MixOrderReduction.is_split_reduction..sK+ '=1$$&7<<8 LL $ $D 0+ sAA)rall get_nodesrs rsis_split_reductionz$MixOrderReduction.is_split_reductions3  " s+ >>++ (  ruc`|j|rd}d}|jD]n}t|tr*|j rt|j t s?|j jJtjjjt|j j}|j jJtjjjt|j j}||}|}tjjj||s J|d|tjjj||reJ|d||J||fS|jdS)N v.s. r)rrrrrrr_original_rangesragraphsizevarssimplifyr__original_reduction_rangesstatically_known_equalsgroup)rxrxnumelrnumelr curxnumel currnumels rsget_numel_rnumelz"MixOrderReduction.get_numel_rnumels  ! !$ 'FF>>+ 4w 6,,."7<<@||44@@@GG,,55!',,"?"?@ ||>>JJJGG,,55!',,"I"IJ >&F&F77++CC 4  {3477++CC 4  {341 48% %%F# #::a= ruc|j|}|j|}t|dk7st|dk7s||k(ryt|tt|k(S)NF)rlentuplereversed)rxrrg1g2s rshas_mix_reduction_ordersz*MixOrderReduction.has_mix_reduction_orderssZ ! !% (  ! !% ( r7a<3r7a<28RyE(2,///rucRd}|jjD]&}t|ts|j|k(s$|}n|sy|j }|jj }|sDt|tsJt||jdjj }|sJt|t|jz sytjjjt!|j"t!|j%ryy)z@ The access to 'buf' is not a broadcast access. NFrT) read_writesreadsrr9nameindex var_rangesFusedSchedulerNodersnodesrr&rarrrr_sizevalues)rxbufr found_depdeprrs rs_is_full_accessz!MixOrderReduction._is_full_accesss  ##)) C#y)chh#o   %%00 d$67 HDJ< H7Q33>>Jz:&E4F4F)GG 77   3 3 ).. )=9J9J9L+M rucg}|j|jz}|D]9}|j||s|j||s)|j|;|Sr~)used_buffer_namesrappend)rxrrout common_readsrs rsget_common_readz!MixOrderReduction.get_common_read0se..053J3J3LL  C""3.33F3FsE3R 3  ruc>t|j||dkDSNr)rrrxrrs rshas_common_readz!MixOrderReduction.has_common_read<s!3&&ue4599ruc|j|}tjjj |d|dzdS)Nrrfallback)rrarroptimization_hint)rxrrs rs get_numelzMixOrderReduction.get_numelBs>  ! !$ 'ww11"Q%"Q%-!1LLruc$|j|Sr~)rrs rsget_fusion_scorez"MixOrderReduction.get_fusion_scoreGs }}U##ructjjsytjj ryt ||sy|jr|jsy|j|jzs|j|jzry|j||sytj||}t|dk(ry|j|r||}}n|j|r||}}ny|j|}|\}}tjj sd} tjj"j%t'j(||z| ddsytjj"j%t'j(||dzddsytjj"j%t'j(|dddsyt+d|j-Drytjj"j/|d sytj1|ryt3d |j-D} | S) zP Check whether we can fuse two reductions with mix loop orders. FriPT)size_obliviousfallback_valueric3K|]T}|jrB|jjjtj tj fvVywr~)rrdatareduction_hintrIINNERDEFAULTrs rsrz-MixOrderReduction.can_fuse..sR ##% LL   , ,##%%  sAAi@c3tK|]0}|jr|jjdv2yw)>sumprodN)rrget_reduction_typers rsrz-MixOrderReduction.can_fuse..s= ##% LL + + -  s68)r-rmix_order_reductionrar cpp_wrapperrr ancestorsget_operation_namesrrrris_contiguous_noder#mix_order_reduction_non_strict_moder evaluate_exprsympyGeanyrstatically_known_leqrr) rxrrrcontiguous_node other_nodernrowncol size_thresrs rscan_fusezMixOrderReduction.can_fuseOsG }}00 77  %eU3!!#5+=+=+? OOe779 9 OOe779 9++E59)88F |  !  ! !% (*/ZO  # #E **/ZO  ! !/ 2 d}}@@#J 77##11j1#$2  77##11tax(#$2  77##11t$#$2   +446    ww44T9E  / / @ &//1    ruc&|j||Sr~)rrs rsare_mix_order_reductionsz*MixOrderReduction.are_mix_order_reductionss||E5))ruc\tfdjjDsyy)Nc3VK|] }j|j"ywr~)is_contiguous_loadr)rrrxrs rsrz7MixOrderReduction.is_contiguous_node..s' 7:C " "388T 2 &)FT)rrr)rxrs``rsrz$MixOrderReduction.is_contiguous_nodes, >B>N>N>T>T  rucddlm}|jD]}t|tsJ|j }|j |j}|Dcgc]}|j|k(s|j }}t|dk(rr|D]u} |j| } |j} t| j} tj j"j%| | | } | ddk(rk| ddk(rtyycc}w)NrMemoryUsageTyperFT)torch._inductor.loop_bodyrrrr_body memory_usageLOAD buffer_name index_namerindexing_exprsrrkeysrarr stride_vars)rxr parent_noderr loop_bodyentriese index_namesr index_exprr var_symbolsr"s rsrz$MixOrderReduction.is_contiguous_loads=))+ !DdM2 22 I,,_-A-ABG18QAAMMSrmaxminrrxor_sum)rrrrrrr get_rangesrrarrrrrInteger) rxr7rTsn reductions reduction iter_ranges reduce_ranges group_sizes rs_get_grouped_reduction_and_sizez/NestedReduction._get_grouped_reduction_and_size?s ((*#/#9#9#;QRr?PbQ Q z?a qM )]3: NNN< %.%9%9%;" ] { 6 )S-?1-D >> , , .7  WW%%..~> *sEMM&:;s:QR?R%-- 333ARs D:D:cR|j|||}|y|j||S)a\Classify pointwise nodes by nested stage and validate their ranges. Each pointwise node must run either on the grouped reduction output, on the grouped reduction's full local-group domain, or on the outer reduction's full parent domain, based on its producer/consumer relationship to the grouped reduction. F) _classify_nested_pointwise_nodes!_pointwise_domains_are_compatible)rxr6r7domain_contextpointwise_domainss rs%_pointwise_nodes_match_nested_domainsz5NestedReduction._pointwise_nodes_match_nested_domainshs= @@      $44^EVWWrucg}t}|jD]&}|js||jz}(|jD]\}|jr||jzs$t |t sy|j||jjf^|j||j}|yg||Sr~) rrrrrrrrrHrF!_classify_grouped_pointwise_nodes)rxr6r7rhouter_pointwise_domainsouter_reduction_namesr^grouped_pointwise_domainss rsrfz0NestedReduction._classify_nested_pointwise_nodess  2<&&( BB %)?)?)AA% B &&( B $r||3!"m4'..,,BBC  %($I$I   " " $% ! % ,E(E+DEErucL|j}|j}tjjj |j |jz}g}|D]?}|jrt|tsy|j}t||jz} t||jz} | r| ry| s| sy| r|jjn|jj} |j \} \} } tjjj#| |j r|jj$}n/tjjj#| |r| }ny|j'||fB|S)aVClassify pointwise nodes relative to the grouped reduction. A node must be on exactly one side of the grouped reduction: either a producer feeding its local-group body, or a consumer of its reduced output. Its numel then determines whether it runs at reduced, grouped-full, or parent-full resolution. N)rQrrarrrrSrTrrrr}rrHrFrGrrrEr)rxrhnodesrQreduction_names full_numelrir^sn_names is_producer is_consumer full_domainr=sn_numeldomains rsrlz1NestedReduction._classify_grouped_pointwise_nodess~+<<+??AWW%%..  ( (>+H+H H   3B b-0--/Hx*;*E*EEFK=>K{{##99((44  "xx A}!ww77.66,,44!!99(JO$  $ $b&\ 2A 3B! ruc0tfd|DS)Nc3JK|]\}}j||ywr~)_pointwise_domain_is_compatible)rr^ryrxrhs rsrzDNestedReduction._pointwise_domains_are_compatible..s, F  / /FN K  #)r)rxrhris`` rsrgz1NestedReduction._pointwise_domains_are_compatibles  /   rucddlm}|jj\}}|j\}\}}||j j ur|j}t|} n||j jurMtjjj|j|jz}|j} nf||j j usJtjjj|j|jz}|j"} tjjj%||xr |j'| |jS)Nr SIMDKernel) codegen.simdrrQr\rrHrErSrrFrarrrrTrUrGrVr is_compatible) rxr^ryrhrrar=rxexpected_numelexpected_groupss rsr|z/NestedReduction._pointwise_domain_is_compatibles= -'99DDF Q88=Ha S((00 0+99N49+4FO s**@@ @WW--66,,~/L/LLN-CCOS00<<< <<WW--66,,~/L/LLN-??Oww77 n I&& H Irucddlm}t|ttfsyt j jr|jnd}t|j}|j||||} t| tdk7rytjjj!| d|r-tjjj!| d|sy||j#||kDS)NrSIMDSchedulingT)xr0_rr)rrrrrr-rcoalesce_tiling_analysisget_coalesce_analysisrr select_tilingrrarrr_max_min_block_group_size) rxr6 outer_numel outer_rnumel grouped_axisrcrcoalesce_analysis node_scheduletilings rs"_min_block_unprofitable_for_kernelz2NestedReduction._min_block_unprofitable_for_kernels 1*}6H&IJ}}55  , , .  Z1134 --       f L!9 9 GG   4 4VC[+ N  88 UC99-VVVrucPddlm}|Dcgc]M}t|tr;|j r+t|j t r|j|O}}||jjur |rtd|Dr |jS|jScc}w)NrSIMDKernelFeaturesc3@K|]}|tjuywr~)rIr)rhints rsrz.?sLDDM///L) codegen.simd_kernel_featuresrrrrrrrrMrKrMAX_INNER_R_GROUP_SIZEMAX_NON_INNER_GROUP_SIZE)rxrrrr^reduction_hintss rsrz)NestedReduction._max_min_block_group_size-s E$ "m,!277N3  - -b 1   COO-- -LOLL-- -+++ sAB#c|j||sy|j||sJ|}|}|j\}}|j\}}|\}} |\} } tjj j | | ry|j|| } | ytjj j|| z} tjj j| | z}tjj j | |sy| \}}|j||| ||}|y||jjur| n|}|j\}}t|dk(rW||jjur|dn|d}tjj j t|||s@ytjj j tj ||dsyt#|}d|kr t%|sy|j'||| ||ry|j\}}|j)|| | g|||| f}|j+|||syy) zACheck whether a dependent cross-axis reduction pair can be fused.Fr6rrr)rrcrQrSrTrUrVT)r;r8rrarrrrdrget_grouped_axisrMrKr\rr'rModrrKrrWrj)rxrrr6r7r= outer_group grouped_grouprrrSrTgrouped_reduction_info outer_total grouped_totalrQrcrparent_grouped_axisragrouped_axis_groupsgroup_size_intrbrhs rsrzNestedReduction.can_fuseDs ""5%0//u===  $));'--=$/! \(5% ~ 77   3 3L. Q "%!D!D ."  " )gg&&// l0JK ((11-.2PQ ww77 ]S(>%:++    ! ,   (COO,=,==L; +557 Q { q ".#//2C2C"C AUV 77##;;,j9;N!!99 II): 6 Z^# n(E  1 1   %% 2 %6%A%A%C" ]33/')#A[#A=#A +\: 4 88    ruNrctjj}|j\}}t |dk7st |dk7rt |dk(rt |dk(r|j |d|sy|j t |||dr(|j |dr|jjS|j t ||dr+|j |d|r|jjSy|j |d|sy|j |d|r5|j t |||dr|jjS|j |d|r5|j t |||dr|jjS||j||Sy)zAReturn which parent axis is split by the grouped local reduction.rrrN) rarrr\rrr'rMrKrL _get_grouped_axis_from_loop_body) rxrQrrrcr6rrarbs rsrz NestedReduction.get_grouped_axiss77##%6%A%A%C" ] { q C $6!$;;1$]);q)@77 a8H*U33\:6 A66{AF??,,,33[*5q66{1~|T??,,,// a0@*M  + + NK .. \: . A ??$$ $  + + NL .. [* -{1~ ??$$ $  !77 DUV Vrucxddlm|jDcgc]}|js|}}t |dk7ryt j t|d}t|dd}t|dd}||y|j\}} |j\} } t | dk7st | dk7ryd fd } t |jdk7st |jdk7ryt |jt |k7s!t |jt | k7ry| |} d}|jd}| |jD]\}}| j|}|s|D]}|j|dk(r|D]|jd}j|}|k(}tfd|jD}||k(rM|r|j j"n|j j$}| ||k7ry|}|Scc}w) z>Use LoopBody iter/reduce vars to disambiguate equal-size axes.rrrNrctt}|jjjdD]D}|j ||j j |j|jF|SNr) rrrgetrrrr r)bodyresultentryrs rsload_exprs_by_namezLNestedReduction._get_grouped_axis_from_loop_body..load_exprs_by_namesu2=d2CF**../C/CRH $$05,,-44++E,<,<=  Mrurc3FK|]}j|k(ywr~)coeff)rvar grouped_coeffouter_read_exprs rsrzCNestedReduction._get_grouped_axis_from_loop_body..s)'&)>)>s)CC's!)rrErzdict[str, list[sympy.Expr]])rrrrrtypingcastrgetattrr\ reduce_vars iter_varsitemsrrrrMrKrL)rxr6rQr^outer_reductionsouter_reduction outer_body grouped_bodyouter_iter_rangesouter_reduce_rangesgrouped_iter_rangesgrouped_reduce_rangesrouter_reads_by_namergrouped_reduce_varrgrouped_read_exprsouter_read_exprsgrouped_read_exprouter_reduce_varouter_reduce_coeffmatches_reduction matches_iter candidaterrrs @@@rsrz0NestedReduction._get_grouped_axis_from_loop_bodysj >)3)=)=)?U22??CTBUU  A % ++m5Ea5HI_gt< 0'4@  !51@1K1K1M..5F5Q5Q5S22 " #q (C0E,F!,K  z%% &! +s<3K3K/LPQ/Q z## $,=(> >#  " "C $ %C&0<59)55b9(:<(H(N(N(P ' $D$266t< #%7 '! 1 7 78J K  A%'7'O'1'='=b'A$)8)>)>?O)P&(59K(K%#&'#-#7#7'$L)L8 ->))COODUDU)f .A#&F!'  '  '2 uVs H7H7)r6rfr7rfrr}r+)r7rfrTrRrz*tuple[SchedulerNode, sympy.Integer] | None)r6rfr7rfrhrWrr})r6rfr7rfrhrWr2list[tuple[SchedulerNode, PointwiseDomain]] | None)rhrWrqSequence[BaseSchedulerNode]rr)rhrWriz/Sequence[tuple[SchedulerNode, PointwiseDomain]]rr})r^rryrHrhrWrr}) r6rfrrRrrRrrMrcrrr})rrrrMrr) rQrrrRrrRrcrRr6BaseSchedulerNode | NonerGroupedAxis | None)r6rfrQrrr)rrrrrrr0r8r;rr@rCEnumrHrMr dataclassrWrdrjrfrlrgr|rrrrrrrursr2r2s:"!" % 5F    % 5F   NN"$))" dii[$'33(3&4,&4>H&4 3&4&4PX%X(X/ X  XX."F%"F("F/ "F < "F"FH5!.5!+5! < 5!5!n . K     I I I/ I  II8%W%%W %W! %W " %W%W %W%WN,2,", ,,,__B041(1 1! 1  1-1 11f@*@?L@ @@rur2ceZdZUded<ded<ded<ej eZded <ej eZ d ed <dd Z dd Z ddZ ddZ ddZddZddZddZddZddZy)SchedulerBuffer Scheduler schedulerz ir.Bufferrr defining_oprlist[NodeUser]usersrF mpi_buffercB|j}|J|jSr~)rget_name)rrops rsdefining_op_namez SchedulerBuffer.defining_op_name(s#   ~~{{}ruc@t|jjSr~)hashrrrqs rs__hash__zSchedulerBuffer.__hash__-sDIINN##rucvt}|j}|j|dt|jj |j|d|jj |jr-|j|dt|j|jr-|j|dt|jt|jdkr0|j|d|j|jS|j|d|jd5|jD]}|j|d ddd|jd |jS#1swY*xYw) N: z .layout = z .aliases = z .mutations = rz .users = z .users = [,])rXr writelinerrrlayout get_aliasespformat get_mutationsrrindent getrawvalue)rrrrusers rs debug_strzSchedulerBuffer.debug_str0s!}}D6DO$<$<#=>?D6DII,<,<+=>?       v[9I9I9K1L0MN O       v]74;M;M;O3P2QR S tzz?a    vYtzzl; <!!##   vZ0 1q! 1 JJ1D$$vQZ01 1   S !!!##  1 1s &F//F8c6|jjSr~rrrqs rsrzSchedulerBuffer.get_nameDyy!!##ruc|jJ|jjsy|jjsL|jjs2t |jj t jr4tjjj|jyttjdr|jtjjvrtjj|j}||j j"vr$|j j"|j}n#|j j$|j}tjjj'||jytjjj|jy)Nargs)rshould_allocateget_inputs_that_alias_outputget_mutation_namesrget_output_specr0CommBufferLayoutrar wrapper_codecodegen_allocationhasattrkernelrinplace_update_buffersrname_to_donated_buffer name_to_bufcodegen_inplace_reuse)rrinput_buffer_name input_buffers rsallocatezSchedulerBuffer.allocateGsVyy$$$yy((*  II 2 2 4yy++-$))335r7J7JK GG 3 3DII >  AHHf % 188#B#BB!" ? ?  P  DNN$I$II#~~DD% $ $~~99:KLQQ GG 6 6   GG 3 3DII >ruc|jJt|jjtjst |jry|j D]}t|jtsyyNFT)rrrr0rDr\r OutputNode)rruses rscan_freezSchedulerBuffer.can_freehsgyy$$$ dii&& 6:S II; :: C#((J/ ruc,i}|D]o}t|j|vr>|j|t|j|t|j<X||t|j<qt|j |_yr~)idrmergerrr)rrrrr s rs set_userszSchedulerBuffer.set_userstst&( +C#((|v%'*yy388 1E'Fr#((|$'*r#((|$  + &--/* rucR|jJ|jjSr~)rrrqs rsrzSchedulerBuffer.get_aliases~s%yy$$$yy5577rucR|jJ|jjSr~)rrrqs rsrzSchedulerBuffer.get_mutations%yy$$$yy++--rucR|jjjSr~)rrrrqs rsrzSchedulerBuffer.get_devicesyy((*5577ruNrr,rrrNonerr})rrrrrz Sequence[str]rtorch.device | None)rrrrrrrrrFrrrrrr rrrrrrrursrrsz O))-K--dCE>C.?k.?.?3/J+ $$($?B +8.8rurceZdZUdZded<y)SchedulerDonatedBufferNrr)rrrrrrrursr r s ,0K)0rur c.eZdZUded<ded<ded<ded<ded<ded <ded <d ed <d ed<dZded<ded<ded<dZded<ded<ded<dZded<dVdZdWd ZdXd!Z dXd"Z dXd#Z dYd$Z dXd%Z dZd&Z d[d'Zd\d(Zd]d)Zd^d*Zd_d+ZdZd,Zed`d-Z dad.ZdZd/Zdbd0Zdbd1ZdZd2ZdZd3Z dcd4ZdXd5ZdXd6Zedbd7Z edbd8Z!ed^d9Z"ed^d:Z#ddd;Z$ded<Z%dfd=Z&dgd>Z'd^d?Z(d^d@Z)d^dAZ*d^dBZ+d^dCZ,d^dDZ-d^dEZ.d^dFZ/dhdGZ0d^dHZ1dZdIZ2 di djdJZ3edkdKZ4edkdLZ5edkdMZ6 dldNZ7 dmdOZ8edndPZ9dodQZ:edodRZ;dpdSZ drdUZ?y)srfrrz7tuple[torch.device, tuple[tuple[sympy.Expr, ...], ...]]r last_usagermin_input_distancemax_input_distance min_order max_orderrGmpi_nodedict[str, str]mutation_renamesNir.Operation | Nonerlist[SchedulerBuffer]outputsdict[str, SchedulerBuffer]outputs_by_name float | Noneoverride_estimated_runtimedependencies.ReadWritesrOrderedSet[Dep]unmet_dependenciesFr}writtenc"||_d|_y)NcgSr~r)rkwargss rsz,BaseSchedulerNode.__init__..sBru)rdebug_device_str)rrrs rs__init__zBaseSchedulerNode.__init__s$- & rucv||_t|_d|_d|_tt |_d|_|jDcgc]}t|j||c}|_ |jDcic]}|j|c}|_ i|_ycc}wcc}w)NrF)rrr)rrrr#r$r,r"r4 get_outputsrrr,rr.r))rrroutputrs rs_init_from_nodez!BaseSchedulerNode._init_from_nodes #"#"#$    **,   ..    @D||L  3L!#  Ms B1 B6cTt|jd|jdS)Nz(name=)rrrrqs rs__repr__zBaseSchedulerNode.__repr__s't*%%&fT]]_,?qAAruc|j}t}|j|dt|jdtt |ddjd|dt |jjd|dt |jd|d t |jj|jz d|d |jd|d |jd|d |j5|jD]!}|j|j# ddd|j!d  |j|j#|j+j-S#1swYXxYw#t$$rt&j)ddYLwxYw)#Longer form printout for trace logsr(rN) .writes =  .unmet_dependencies = .met_dependencies = .min_input_distance = .max_input_distance = z.outputs = [ rIgnoring error in debug_str()Texc_info)rrXsplicerrrrrwritesr3rr#r$rr<rrdebug_str_extra Exceptionlogwarningrrstrip)rrrrrs rsrzBaseSchedulerNode.debug_strs}} bd  QtGD&$$?@IIJKj))00123WT%<%<=>?74#3#3#9#9D"> ?j33$$&'%4H EOO$6$6$@$@ Aj33..0*2O2O2QR4H z""rucptjd||j|jjy)Nz(%s: unmet_dependencies = %s, writes = %s)rTinfor3rrQrqs rs log_detailszBaseSchedulerNode.log_detailss,  6   # #    # #  rucyNFr)rrself_dep other_deps rsreorder_loops_by_dep_pairz+BaseSchedulerNode.reorder_loops_by_dep_pairrucd|jjDDcic] }||vr|||c}|_|j|jj |jycc}w)Nc34K|]}|jywr~rrrs rsrz9BaseSchedulerNode.update_mutated_names.. sQcQ)rreads_and_writesr)set_read_writesrename)rrrenamesrs rsupdate_mutated_namesz&BaseSchedulerNode.update_mutated_namesspRT-=-=-N-N-PQ! w '$- !  T--44T5J5JKL ! sA2cX|j|jj|yr~)rxr with_readrrrs rs add_fake_depzBaseSchedulerNode.add_fake_deps! T--77<=rucBtd|jDS)Nc3`K|]&}|jxs|j(ywr~)rrrrs rsrz=BaseSchedulerNode.has_aliasing_or_mutation..s- 9.+s!U1"4"8"8A">!U)used_or_aliased_buffer_namesrr")rrfuture_used_buffersr used_bufferss ` rsset_last_usagez BaseSchedulerNode.set_last_usage's088: !!U !UU &)<<rucF|jD]}|jyr~)r,r )rrrs rsmark_runzBaseSchedulerNode.mark_run.s<< C LLN ructdtj|jj|jj DS)Nc34K|]}|jywr~rtrus rsrz6BaseSchedulerNode.used_buffer_names..3s  HH rv)r itertoolschainrrrQrqs rsrz#BaseSchedulerNode.used_buffer_names2s?  t'7'7'='=t?O?O?V?VW   ruc\ttj|jj|jj Dcgc]*}t |tr |js |j,}}t|dkDr|j}j|tjjj!|rC|j#fdtjj|j%Dt|dkDrScc}w)z Returns buffer names used by this node, including aliases. Note: is_fake WeakDeps are excluded since they are purely for ordering and should not affect buffer lifetime. rc3*K|] }|vr| ywr~r)ralias used_namess rsrzABaseSchedulerNode.used_or_aliased_buffer_names..Js#J. s)rrrrrrQrr;is_fakerrpopaddrarname_to_bufferrextendr)rrrdepsrs @rsrz.BaseSchedulerNode.used_or_aliased_buffer_names8s'1l !t'7'7'='=t?O?O?V?VW sG, HH  $i!m((*C NN3 ww%%))#. !"!7!7"224  $i!m! s /D)cLtfdjD_y)Nc3fK|](}|jjjvr|*ywr~)rravailable_buffer_namesrrrrs rsrz/BaseSchedulerNode.prune_deps..Ts/- xxt~~DDD - s.1rr3rqs`rsrzBaseSchedulerNode.prune_depsSs#",- ..- # rucdfd tfdjjD}jjj |y)Nct|tsy|jjjvryjj|jj }|t jjvSrm) rr;rrrrrarremoved_operations)rop_namerrs rs should_prunez7BaseSchedulerNode.prune_weak_deps..should_prune\s_c7+xxt~~999nn00:KKMGagg888 8ruc34K|]}|s |ywr~rrrrs rsrz4BaseSchedulerNode.prune_weak_deps..es \#5FC  rr8rr})rrrrx remove_reads)rr to_removers` @rsprune_weak_depsz!BaseSchedulerNode.prune_weak_depsZsN 9 ++11   T--::9EFrucFt|||jjyr~)_prune_redundant_depsrr)rrname_to_fused_nodes rsprune_redundant_depsz&BaseSchedulerNode.prune_redundant_depsjs d$68R8RSrucR|jJ|jjSr~)rget_operation_namerqs rsrzBaseSchedulerNode.get_nameorruc"|jSr~rrqs rsget_first_namez BaseSchedulerNode.get_first_namess}}rucBtd|jDS)Nc3<K|]}|jywr~rrrs rsrz8BaseSchedulerNode.get_operation_names..xsGd$--/G)rrrqs rsrz%BaseSchedulerNode.get_operation_namesvsGdnn6FGGGruc:td|jDS)Nc3<K|]}|jywr~rrrs rsrz5BaseSchedulerNode.get_buffer_names..|sAS#,,.Ar)rr,rqs rsget_buffer_namesz"BaseSchedulerNode.get_buffer_nameszsADLLAAArucBtd|jDS)Nc3ZK|]#}t|txr t|d%yw)T)disallow_fp32_opsNrrr2rns rsrzABaseSchedulerNode.can_codegen_in_low_precision..s7  q- ( G+AF G )+rrrqs rscan_codegen_in_low_precisionz.BaseSchedulerNode.can_codegen_in_low_precision~s% ^^%   rucBtd|jDS)Nc3VK|]!}t|txr t|#ywr~rrs rsrz@BaseSchedulerNode.can_codegen_without_upcasts..s-  q- ( K-H-K K s')rrqs rsr2z-BaseSchedulerNode.can_codegen_without_upcastss# ^^%   ruc|gSr~rrqs rsrzBaseSchedulerNode.get_nodess v ruc|jSr~)r,rqs rsr<zBaseSchedulerNode.get_outputss ||ruc |j|Sr~)r.)rrbuf_names rs get_outputzBaseSchedulerNode.get_outputs##H--rucR|jJ|jjSr~)rrrqs rsrzBaseSchedulerNode.get_devices%yy$$$yy##%%rucL|j}|duxr|jdk(SNcpu)rrrrdevices rsis_cpuzBaseSchedulerNode.is_cpus'"T!:fkkU&::rucX|j}|duxrt|jSr~)rr[rrs rsr[zBaseSchedulerNode.is_gpus'"T!9fV[[&99rucyrmrrqs rsrzBaseSchedulerNode.is_reductionrucyrmrrqs rsis_native_matmulz"BaseSchedulerNode.is_native_matmulrrucyrmrrqs rs is_split_scanzBaseSchedulerNode.is_split_scanrrucyrmrrqs rs is_templatezBaseSchedulerNode.is_templaterrucyrmrrqs rs is_externzBaseSchedulerNode.is_externrrucyrmrrqs rs is_foreachzBaseSchedulerNode.is_foreachrrucyrmrrrread_deps rs can_inplacezBaseSchedulerNode.can_inplacerrucyrmrrqs rshas_side_effectsz"BaseSchedulerNode.has_side_effectsrruc  ddlm}ttrtj rt jjjtjr{tt jtjjj j"rt%t jddt't jdsyj(t jj*zj,j.z}dfd }j1D]Q}|j2}|J|j5rr|j7sb|j9sR|j;t jj<vs(t|j?t@jBrjDjFD]}|jHj,jJvr$j,jJ|jH}n/j,jLjO|jH}|s|t jjPjS|st|jTtVr|jXJ|jXDcgc] }|j2j;|vr|"} }j,j[|jH} | r/t]| dk(s?| dj^sP| dj2usc|j2qt|j2j?t@j`t@jbt@jdt@jBfr|jTrft|jTj2t@jft@jhfr(t]|j2j7dkDrE||j2|j2sd||snt jjjjm|j;|j;tt jtjjj j"rnt jjnjq|j;t jjnjq|j;|j;t jjr|j;<QTycc}w) z~ Decide if there should be inplace updates for the node and record the decision in the active kernel. r)can_match_buffer_size mutationsNrc|jj}|jt}|jD]}|j }t |ts |j|jjvs|jj||urd|fd|jjDz}t|dkDsyy)Nc3@K|]}|jk(r|ywr~rt)rors rsrz^BaseSchedulerNode.decide_inplace_update..single_index_in_fused_node..s%vv)rFT) rget_fused_noderrrrrrfrrrrwr)buf_to_be_inplaced fused_noderr user_noderrrs @rssingle_index_in_fused_nodezKBaseSchedulerNode.decide_inplace_update..single_index_in_fused_nodes ,55DDTJJ)224H%/LD*00 ! II !)->?,,.-77JJK)33BB9M%&&22CCE t9q= ' !*rur)rrrr}):codegen.wrapperrrrr-inplace_buffersrar has_featurerr3INPLACE_BUFFERSrr`racodegensimdrrrrrrcompleted_operationsr<rrrrrremoved_buffersrr0rrrrrrrr can_reuserNopKernelSchedulerNoderhas_cross_stream_hazardrrrDrCMutationLayoutSHOULDREMOVEFallbackKernelrBr make_inplacerrr) rrrinconsequential_nodesrrbuf_noderead input_bufrremaining_usesrs ` rsdecide_inplace_updatez'BaseSchedulerNode.decide_inplace_updates ; t] +&&##DOO$5~7U7UVqxx)@)@)E)E)P)PQ188[$7C&)  NNgg(( )nn11 2  D##%L CxxH' '',,.88:..0<<>QWW%<%<<h668":M:MN((..> 99 E EE $ E Edii PI $ : : > >tyy II,,66y$G&y'<'<>TU$??666"+&66??,4II&N& /3nn.T.T 4/+ 4/14*1-99*1-22d:%NN6 *%NN::< " " 4 4 " = = " 3 3 !&11 * ) 5 5 : :!#!2!2BNN C!!$INN$O$O$Q RUV V1)..#((K6yA  2293E3E3GX%HHeoo&=&=&B&B&M&MHH..2293E3E3GHHH..223<<>B&..077 G}> L 6&s?%V cRtjsy|r |jry|jJ|jj }g}|D]0}|j dk(r|j d|j dd|j d|j}d|jvr|d|jdz}|j |d|jvs|jd}|jd d d }|j d|jd djddjddjddz|j d|j d3t|dk(ry|j|d|_y)Nr=rXz#pragma CMT ORIGIN:z #pragma CMT  seq_nrz seq_nr: stack_trace|r)maxsplitr{z{{}z}}rH\z\\z#pragma CMT END ORIGINrT) r-comment_originr4r get_originsrrtargetmetarsplitreplacer writelines) rrbuffer only_onceorigins out_linesr op_info_strrstack_trace_last_lines rscodegen_originating_infoz*BaseSchedulerNode.codegen_originating_infoEs$$   yy$$$))'')  %Attx   R   2 3(az:K166!)hqvvh7G6H,II   [ )&!" !6 7 (3(:(:3(:(KB(O%  "+33C>WS$'WT4(Wf   !9:  $3 %6 y>Q   )$ ruc(|jddS)NT include_readsinclude_writes!get_read_write_buffers_sizes_implrqs rsget_read_write_buffers_sizesz.BaseSchedulerNode.get_read_write_buffers_sizests55t6  ruc(|jddS)NTFr*r-rqs rsget_read_buffer_sizesz'BaseSchedulerNode.get_read_buffer_sizeszs55u6  ruc(|jddS)NFTr*r-rqs rsget_write_buffer_sizesz(BaseSchedulerNode.get_write_buffer_sizess556  rucZt|j||jdS)Nr*r)start)rget_read_write_buffer_accessesr)rrr+r,s rsr.z3BaseSchedulerNode.get_read_write_buffers_sizes_impls3  / /+N 0 fh   ruc ttriSttrtjtriSttr`tjt j r<jjtjjjuriSd dttr@tjdtjdzn tdt!j"t$}|r9j&j(D] }||j*j-|"|r9j&j.D] }||j*j-|"|r&t1dj&j(Dn t1}|r&t1dj&j.Dn t1}d fd tt2rt1fd|D}||z }||z }i}||zD]} t5fd || D | t6j8j:vrt6j8j:| } n;| t6j8j<vrt6j8j<| } n d fd  | } | |vr| || <|| xx| z cc<|S)az Counting the number of bytes accessed for a kernel is surprisingly tricky. In particular, there is a differentiation between 'theoretical' memory accesses and practical memory accesses. For example, a layernorm kernel may actually access an input 3 times, but in theory, it only needs to access its input once (and may be optimized to do so through say, persistent reductions) Another example is that even though a buffer is passed in, we may not access the entire buffer. This may occur if we are accessing a slice of the buffer. Another tricky case is for indirect indexing, where the amount of bytes accessed depends on the values of the input. What this function aims to compute is the memory accesses for worst-case inputs, best-case optimization. What this means is that for each buffer we compute the amount of potential accesses in two ways and take the minimum. 1. Numel in ranges multiplied by number of deps the buffer has 2. The buffer size Returns memory accesses per buffer. cXtjjj|dS)Nrr)rarrr)ss rs try_size_hintzGBaseSchedulerNode.get_read_write_buffer_accesses..try_size_hints"77##55a!5D DrurreAc34K|]}|jywr~rtrus rsrzCBaseSchedulerNode.get_read_write_buffer_accesses..sBCsxxBrvc34K|]}|jywr~rtrus rsrzCBaseSchedulerNode.get_read_write_buffer_accesses..CCsxxCrvcjj|j}td|D}t |t|z dkDS)Nc34K|]}|jywr~r)rrs rsrz\BaseSchedulerNode.get_read_write_buffer_accesses..is_materialized..s!>$))!>rvr)rrrrr)rrrbuf_usesrrs rsis_materializedzIBaseSchedulerNode.get_read_write_buffer_accesses..is_materializedsGNN..s399E!!>!>>Hx*V"4459 9ruc3JK|]}|jr|ywr~r)rrrBrrs rsrzCBaseSchedulerNode.get_read_write_buffer_accesses..s#)_S$++-N)s##c3"K|]}ywr~r)rr node_numels rsrzCBaseSchedulerNode.get_read_write_buffer_accesses..s$RCZ$Rs cB|syt|tjr|jSt|jt r͉j j|jj}d}|D]}t|jtrt|jtsJt|jjtr5|jjD]}||jz }y|St|jtjr"t!fd|j#DS t%|j'}t)|j+t-|zS)Nrc3hK|])}tjj|+ywr~)rar get_buffer)rmut_name get_buf_bytess rsrzZBaseSchedulerNode.get_read_write_buffer_accesses..get_buf_bytes.. s-$&agg&8&8&BCs/2)rr0TorchBindObjectrKrrCrrrrrr rfrBr<rDrrr_rcrT get_dtyperZ) rrtotr sched_buf buf_elemsbuf_accessed_elemsrKrrr:s rsrKzGBaseSchedulerNode.get_read_write_buffer_accesses..get_buf_bytessTc2#5#56,,.. ,=>!NN66s||~FLLEC % %%dii<$)$))5FGGG%diinnkB-1YY-B-B-DE #}Y^^'D DE $% %J BMM:(+(>(>(@ !.mCLLN.K LI)#--/:S*I>ru)r9rRrr)rr,rrrr})rz4ir.Buffer | ir.TensorBox | ir.TorchBindObject | Nonerr)rrExternKernelSchedulerNoderrBr0r  op_overloadr`_prims rng_primsgraphsafe_run_with_rng_staterr_r\r collectionsrrrrrrrQrrrrarr graph_inputs)rrr+r, buf_accessesrrrQrbuf_byte_accessesrr buf_bytesrQrKrBrFr:s` @@@@@rsr6z0BaseSchedulerNode.get_read_write_buffer_accessess6 d2 3I d5 6: II{< I t6 7499b&7&78 %%||%%BBCI E dM *&doo/23 1! 456J SJ"..t4 ''-- 3SXX&--c2 3 ''.. 3SXX&--c2 3  B4+;+;+A+AB B  C4+;+;+B+BC C   : d. /()%)Oo-FO+E,.3 9H!$$R<;Q$R!R 177111gg,,X6QWW111gg**84# I# # # J&c*I00.7!(+!(+y8+g3 9j! rucT|jy|jj}|yt|}|yt|tj r|jj }tjjj|d}tddxx|z cc<|S)Nrrinductor flop_count) rget_origin_noder>rr`SymIntexprrarrrr)rrfx_nodeflopsresolved_flopss rsestimate_flopsz BaseSchedulerNode.estimate_flopss 99 ))++- ?w' = eU\\ *JJOOE));;EA;N\*n<*rucR|j |jS|jSr~)r0_get_estimated_runtimerqs rsget_estimated_runtimez'BaseSchedulerNode.get_estimated_runtime1s)  * * 622 2**,,ruc|jdjd}|jj}t t |syt |jrt|jtjsJ tjrst|}t}|j|}|t|tsJ|St!|}|t#|j}|j%|||St#|jSt/|jryt1|}||S|jj3} t5} t7| dz} | dkrt9d| | dkrt9d|  |j=} | dk(s| |j?| z } | dz }|Sd}|j?}|dn|}|| z| z d z}|| z }tA||} | dz }|S#t&$r}t(j+|Yd}~yd}~wt,$r}t(j+|Yd}~yd}~wwxYw#t:$rYywxYw) zC Returns estimated op runtime in milliseconds (ms) rNvaluelJ)z-gpu_memory_bandwidth cannot be <= 0, but got z"gpu_flops cannot be <= 0, but got g.Ag?r;)!rr<rrr[r@rYrr0IRNoder.,runtime_estimations_use_nccl_lib_estimations)get_estimate_runtime_cache_key_from_snodeget_estimate_runtime_cachelookupfloatr7r6 set_value ValueErrorrTrj TypeErrorr^ maybe_estimate_runtime_benchmarkmaybe_get_dtyperUrSAssertionErrorrSrer/rY)rrrr cache_keycache cache_valmsr&retdtypegpu_memory_bandwidth gpu_flops flops_estnsfactor counted_bytes compute_time transfer_times rsrgz(BaseSchedulerNode._get_estimated_runtime7sw nnq!--/2))+of-.  #dii3 33 LL I$ OI68E % Y 7I ,))U;;;((HNBz=diiHOOIRO8I7 BBTYY  .t4 ?J((* #4#6 )%069I$q($CDXCYZA~$'I)%UVV '') >Y.2247KKBcBI99; *2 *Y6#= %(<< } - #X o     :  sCA H6HH(>I$ I!H66 I!II!$ I0/I0cyr~rrqs rsget_template_nodez#BaseSchedulerNode.get_template_noderuc.|j}|J|Sr~r)rrtemplates rsget_template_node_or_throwz,BaseSchedulerNode.get_template_node_or_throws!))+###rucftdt|D}|d|}||}||dzd}|||fS)zQ For the list of nodes, get the prologue, template, and epilogue c3HK|]\}}|js|ywr~r)rirs rsrzCBaseSchedulerNode.get_prologue_template_epilogue..sPDAq aPs""Nr)next enumerate)rqtemplate_indexprologue template_nodeepilogues rsget_prologue_template_epiloguez0BaseSchedulerNode.get_prologue_template_epiloguesNPIe,<PP.)n- !+-.00ru)rrrr)r ir.Operationrrr)rr-rrnr9ror9rr}rzr(rr)rr8rrr)rr1rr)rzCoalesceVarAnalysis | Nonerrrr(rrrrrdict[str, BaseSchedulerNode]rrrr)rzSequence[SchedulerBuffer])rr,rrrrzdependencies.Deprr}T)r"rXr#r}rrr)r+r}r,r}rr)r+r}r,r}rzdict[str, int]r int | Nonerrqrzir.TemplateBuffer | None)rzir.TemplateBuffer)rqlist[BaseSchedulerNode]rzJtuple[list[BaseSchedulerNode], BaseSchedulerNode, list[BaseSchedulerNode]])@rrrrrr0r4r:r>rBrrRr[rhrkrpr{rrrxrrOrrrrrrrrrrrrrr2rr<rrrr[rrrrrrrrrr(r/r1r3r.r6rerhrgrrr0rrrursrfrfs BB NN''$$ $D $ ""///3 3((''GT #4B*6+ # !.7  M>  500=#2=HV= = 6 G T">T T .HHBB    .&;:IX9=-$-15- -^       ! 37  L!!L!37L! L!\$- UUn  1& 1 S 1 1rucRtjjjSr~)r`ra codecache LocalCacherrursroros ?? $ $ / / 11ruct|jdd}|jj}|jjg||jj|jj }|jj }t j||f\}}ddt|ftfd|Dz}|S)Npython_kernel_namerXct|tjxr+t|tjtjf Sr~)rr0rlGeneratorStateOpaqueObjectStaters rs _is_tensor_irz@get_estimate_runtime_cache_key_from_snode.._is_tensor_irs<!RYY' !!2#7#781 -  ruc3dK|]'}|rt|jnd)ywr~)rrc)rars rsrz.s(Ua}Q'7ajjl#TAUs-0r) rrinputsfill_non_provided_args constant_argsr7pytree tree_flattenr,r)snoderrr7 flat_argsflat_args_pytree_specrxrs @rsrnrns -A2F ::  D :: , ,*$*))*  DZZ  F'-':':D&>'J$I$   U9U U VI ruct|tsyt|jtjsy|jj }t|t jjsy|j}ddl m }||vry|S)Nr) flop_registry) rrRrr0 ExternKernelrSr`_ops OpOverloadoverloadpackettorch.utils.flop_counterr)rrSrrs rs_get_benchmarkable_extern_fnrsk e6 7 ejj"// 2**((K k5::#8#8 9  # #B6  IrucZ d}d}tjrt}|y|} fd}nyt}t }|j |}|t |tsJ|Sddlm |\}}ddl m } | j|||ddd} |j|| | S) NcSr~r)rsnode_args_kwargssrsr8z2maybe_estimate_runtime_benchmark..s !25!9rur)rr) benchmarker )memory_warmup_itersbenchmark_itersmax_benchmark_durationrj)r-!runtime_estimations_mms_benchmarkrrnrorprrqutilsr$torch._inductor.runtime.benchmarkingr benchmarkrr) rbench_fnargs_kwargs_fnmm_fnrxryrzrr7rr{rs ` @rsrurusHN //,U3 =99%@I & (E Y'I)U+++(!#LD&@    !  B OOIRO( IrucNeZdZUded<ded<ded<ded<d dZd dZd d Zy ) WhyNoFuser,name1name2reasontuple[Any, ...]rcX|j|_|j|_yr~)rrrrrrrs rsr:zWhyNoFuse.__init__s^^% ^^% rucJ||_||_tj|yr~)rr fusion_logdebug)rrrrs rs__call__zWhyNoFuse.__call__s  rucpd|jd|jd|j|jzzS)Nz cannot fuse z with r)rrrrrqs rs__str__zWhyNoFuse.__str__ s6djj\ |2> KK$)) #  ruNrrfrrfrr)rr,rrrrr)rrrrr:rrrrursrrs& J J K &  rurct|ttfrt|t}t j |d}d|vrdtj|dS|S)Nkey)rrH ) rrsetsortedr,pprintrtextwrapr)objrs rsrrsR# C()Sc" ^^C *F v~HOOFG4566 Mruc0eZdZddZddZddZd dZeZy) r c&t|g|_yr~rr~s rsr:zOutputNode.__init__s",cU"3rucyrmrrqs rsrzOutputNode.is_reduction!rrucyrrrqs rsrz'OutputNode.get_inputs_that_alias_output$rYrucy)NOUTPUTrrqs rsrzOutputNode.get_name'sruN)rr:rrrrr)rrrr:rrrrBrrursr r s4Hrur ctjjD]N}t|tr|j j }|jxxdz cc<Pdfd tfdjD}|r?j|z _jjj|yy)am Prunes weakdeps intended for mutation ordering on an upstream fused node if after fusion there is another dependency on the fused upstream node, making the weakdep redundant In essence this enforces an ordering on fusions. As fusions occur, weakdeps will be incrementally removed, enabling other fusions, ensuring they are fused in order. rct|trf|jj}|j dkDxr j j ||}|k(}|xs|Sy)NrF)rr;rrrrfusable_weak_dep)rr is_redundant is_self_deprname_to_dep_countrrs rsrz+_prune_redundant_deps..should_pruneAs c7 #!#((+<<>G,"7+446nn55'0$ -W5=K.; .ruc34K|]}|s |ywr~rrs rsrz(_prune_redundant_deps..Qs,s2CrNr) rWrr3rr;rrrrrxrr)rrrrr deps_to_prunerrs``` @@rsrr-s'2&9&9&;&&K#w'!#((+<<>G 09BBD E J EK  ..M"&"9"9M"I T--::=IJrucHeZdZdfd ZddZd dZd dZd dZd dZxZ S) rRcnt|||j||j|j t |t jr[|jrJtj|jdj}d}|j}|||ff|_yyyNrr)superr:r>rxget_read_writesrr0UserDefinedTritonKernelcan_fuse_epiloguemathr mutable_argsshapeget_device_or_errorr)rrrrnumelrr __class__s rsr:z"ExternKernelSchedulerNode.__init__[s # T" T1134 dB66 7DEGR= Bxruct|jtjsJ|jj |Sr~)rrr0rr)rrwrappers rsrz!ExternKernelSchedulerNode.codegenzs/$))R__555yy  ))rurrrrrrrrrSequence[Sequence[sympy.Expr]]r rrr) rrrr:rRrrr\r __classcell__rs@rsrRrRZs# 3cW*rurRc eZdZdfd ZxZS)rct|||j||j|j yr~)rr:r>rxrrrrrrs rsr:zNopKernelSchedulerNode.__init__s5 # T" T1134rur)rrrr:rrs@rsrrs 55rurceZdZUdZded<ded< d&fd Z d' d(dZ d' d)dZ d*d Zd+d Z d,d Z d-d Z d.d Z d/dZ d0dZd.dZd1dZd.dZ d2dZd.dZ d3dZd4dZd5dZd6dZd6dZd6dZd6dZd7dZd8dZ d9dZd:dZ d; dd#Z#e d?d$Z$e d6fd% Z%xZ&S)@rzu A SchedulerNode is a node for scheduling that encapsulates either a ComputedBuffer or a TemplateBuffer. z tuple[Sequence[sympy.Expr], ...]_sizesrErctt||d|_|j||j yr~)rr:_loop_mutation_listenerr>_compute_attrsrs rsr:zSchedulerNode.__init__s4 #OS$ T" ruct|jtjtjfsJ|jj ||\|_}||_|jj}|jj|j}|||j f|_ tj xst|j  }t|jtjr,|j#|jj%|y|j#t'j$|jg|j d|iy)Nextra_indexing_constraintsrecompute_sizes_body_func) normalizer )rrr0rTemplateBuffersimplify_and_reorderrrrr get_backendgroup_fnrr-loop_ordering_after_fusionr[rrxextract_read_writesr/)rrrrrrr$should_normalizes rsrzSchedulerNode._compute_attrss7 $))b&7&79J9J%KLLL II::'A&?;  T ..0>>--f5>>ht{{34  &@@@  KKI E  dii!2!2 3   --8H-I   00JJ!%8H ructd|jjD}|j|||rD|j |jj |j |jyy)Nc3NK|]}t|ttfs|ywr~rr;r:rus rsrz8SchedulerNode.recompute_size_and_body..#0 ZgwEW5XC0 %%r)rrrrrxr}ryr))rrrr fake_depss rsrecompute_size_and_bodyz%SchedulerNode.recompute_size_and_bodys &00 ++110 &  'A&?       **95<.r+r,r ) rrrrxr/r&rrr}ryr) clear_loop_body_dependent_caches)rrr need_clear_tiling_cacher-s rsrefresh_dependenciesz"SchedulerNode.refresh_dependenciess &00 ++110 &   , ,  ![[ 4= Yy ! VD)) *   --.EFruc|j|jj||r!ddlm}|j j yy)Nrr)rpointwise_read_writesrrrcandidate_tilings cache_clear)rrr2rs rsr1z.SchedulerNode.clear_loop_body_dependent_cachessA //1 ""..t4 " 4  , , 8 8 : #ruct|j|j|j|j|jfS)zSnapshot mutable state modified by loop transformations (apply_new_loop_order, apply_loop_reindexing). Must be kept in sync with those methods and restore_loop_state.)rrrrr3rqs rssnapshot_loop_statez!SchedulerNode.snapshot_loop_states5 JJ KK JJ     # #   rucj|\|_|_|_|_|_|j dy)z'Restore state from snapshot_loop_state.T)r2N)rrrrr3r1)rrstates rsrestore_loop_statez SchedulerNode.restore_loop_states8   J K J    # --d-Kruc@|j|j|yyr~)rrqs rs_before_loop_state_mutationz)SchedulerNode._before_loop_state_mutation s!  ' ' 3  ( ( . 4ruc|j|jj||_|jj|_|j ddyNFTr r2)r>rreorder_iter_loopssizesrr3)rr new_orders rsapply_new_loop_orderz"SchedulerNode.apply_new_loop_order sM ((*ZZ22   jj&&  !!E4!Pruct|jtjtjfsJ|j |j j||_|j j|_ |jj}|jj|j}|||jf|_|jddyr@)rrr0rr!r>rreindex_iter_loopsrCrrrr#r$rr3)rrnew_iter_sizesrr$s rsapply_loop_reindexingz#SchedulerNode.apply_loop_reindexing s$))b&7&79J9J%KLLL ((*ZZ22>B jj&& ..0>>--f5>>ht{{34  !!E4!Pruc|jj}t|jj|z }t t |}t t |||z}|j ||zt|jddk(sJ|jd|jdd|jddff|_y)Nrrr)rget_original_num_rdimsrrrrangerEr)rr num_rdims num_pwdimspwdimsrdimss rsswap_pw_red_dimensionz#SchedulerNode.swap_pw_red_dimension sJJ557 --.: uZ()eJ Y(>?@ !!%&.14::a=!Q&&&ZZ]TZZ]1%5tzz!}Q7G$HH rucD|jj|_|Sr~)rextract_pw_from_reductionrqs rsrSz'SchedulerNode.extract_pw_from_reduction( sZZ99;  ructj|syt|jtj sJ|jj 5|jdddy#1swYyxYwr~)rrrrr0rwith_original_inner_fnrrqs rscancel_reduction_splitz$SchedulerNode.cancel_reduction_split, s^ 33D9 $))R%6%6777 YY - - / "    ! " " "s A11A:ct|jtjtjfsJ|j j |||_|j j|_|jj}|jj|j}|||jf|_ |jddy)NTrA)rrr0rr!r#expand_dimension_for_pointwise_noderCrrrr#r$rr3)rr dimension new_rangerr$s rsrXz1SchedulerNode.expand_dimension_for_pointwise_node3 s$))b&7&79J9J%KLLLZZCC y  jj&& ..0>>--f5>>ht{{34  !!D$!Oruc|jj|_|jj|_|j ddy)NTFrA)r merge_loopsrCrr3rqs rsr\zSchedulerNode.merge_loopsD s<ZZ++- jj&&  !!D%!Prucd}|jd}t||jcxk(r|jk(rnn|j|}|rPtxj dz c_t jd|j||j|yt jd|jy)Nrrz"Reorder loops for %s with order %sTzEDon't reordering %s because we can not decide the suitable loop orderF) rrnum_varsdecide_loop_order_to_matchr1num_loop_reorderingloop_ordering_logrrrE)rrrnrorD self_sizess rsrpz'SchedulerNode.reorder_loops_by_dep_pairP s [[^ z?h// E93E3E E ;;IFI   ' '1 , '  # #4dmmoy   % %i 0  # #W  ruc$|j}|d|jd|d|jd|d|jg}|jj D]}t |t r|j}tjj|}t |tjrZ|j|dt|jt |j t"rR|jd|d|jt%j&|j j)d |j*J|j-|j/d j1|S) Nz.group.device = rz.group.iteration = rz .sizes = z _layout = zclass z _loop_body:rrH)rrrrrwrr;rrarrIr0rLrrrrrErrrrrr[join)rrrlinesrrrs rsrRzSchedulerNode.debug_str_extrag sK}}f$TZZ]O 4f' 1  7fIdkk] +  ##446 OCc7+88gg((2!#r'9'9:LLH:Z 8K7L!MN  O djj( + LL6${3 4 LL)=)=)?H Iyy$$$ T//12yyruc|jSr~)rrqs rsr\zSchedulerNode.get_ranges} {{ruc<t|jtjtjfsJdt |jt |jjxr'|jduxs|jj SNtype(self.node)=) rrr0rr!rr}rrhas_partial_accumulaterqs rsrzSchedulerNode.is_reduction s$))b&7&79J9J%KL tDII ! LDII0023 JJ$  Gdjj&G&G"G ruct|jtjsJdt |j|jj dk(S)Nrjdot)rrr0rrrrqs rsrzSchedulerNode.is_native_matmul sJ$))R%6%67N. s?Ss ?rrztype(write_dep)=)rrr<rrrQrr/r9riterrrr)rrr write_deps rsrzSchedulerNode.can_inplace s     ?D,<,<,>? ? t&& '1 , l,,2 T$"2"2"9"9:;Ii)?)?@ WEUT)_DVBW W@>>Y__4X)..9X Xruct}t|jtr|jj D]}|j dk(s|j dk(s#d|jvr|jddk(s,t|jdk(s\|jddk(so|jd|jvr|jdn(t|jdk\r|jd nd |S) N call_methodstoremode atomic_addrrrrrrX) rrrrErrrr7rrr)rrbuffers_store_as_atomic_addrs rs_get_atomic_add_buffersz%SchedulerNode._get_atomic_add_buffers s7A|# djj( + ,,. GG}, w.4;;.4;;v3F,3V Na/DIIaLL4P033!T[[0 F+.1$))n.Adiilr +*rucp|j|jjdryt| S)Ndevice_assert_asyncT)rhas_oprrrrrs rsrzSchedulerNode.has_side_effects s2 :: !djj&7&78M&Nw'))ru)rrrz%ir.ComputedBuffer | ir.TemplateBufferrrNN)r'tuple[dict[Any, Any], list[Any]] | NonerzCallable[_P, _T] | Nonerr)rrrzCallable[..., Any] | Nonerr)r r}r2r}rr)r2r}rr)rr)r;rrrr)rD Sequence[int]rr)rHSequence[sympy.Expr]rrrrf)rYrrZrrrrrrrr)rtrrr)rtrrzdict[sympy.Expr, sympy.Expr])rtrrrr)rr}rr1)rr1rr)'rrrrrr:rr.r3r1r9r<r>rErIrQrSrVrXr\rprRr\rrrrrrur{rrrOr5rrrrrrs@rsrrs -, O4  OS=A$K$;  FOS?C$K$=  "GG8<G G* ;   L/Q QI"PP),P P" Q!.7 . ,  7 8O! 8  % 0!%      GG HH ++&**rurc nj}jtjj |Dcgc]}|j c}t fdt j|Dcgc]}|jc}Dj jz _ycc}wcc}w)Nc3ZK|]"}|jjvr|$ywr~rr)rr group_snodes rsrz2refresh_group_node_dependencies.. s. xx{;;==  (+) rrxr/ ReadWrites merge_listrrunionr3rQ)rrrs` rsrefresh_group_node_dependenciesr s  F**6+JaAMM+JK  !'')O1!*>*>)OP   ! ! ( (  )",K *Ps B- 0B2rclt|ttfsJ||_||_d|_t j|Dcgc]}|j|jc}|_t|td|jD|_ td|jD|_ td|jD|_td|jD|_|j!Dcic]}|j#|c}|_ycc}wcc}w)Nc34K|]}|jywr~r%rrs rsrz"init_group_node..+ H Hrvc34K|]}|jywr~)r&rs rsrz"init_group_node.., rrvc34K|]}|jywr~)r#rs rsrz"init_group_node..- )!")rvc34K|]}|jywr~)r$rs rsrz"init_group_node..0 rrv)rrGroupedSchedulerNoderrrrrrrrZr%rYr&r#r$r<rr.)rrrrrs rsinit_group_noder s k$68L#M NN NK%KK&,,% A!)@!++ AK$K0H[5G5GHHKH[5G5GHHK%()&1&8&8)&K"&))&1&8&8)&K"(3'>'>'@# # #K B#sD,D, D1ceZdZUdZded<e d!dZd"dZd#dZe d$dZ d%dZ d&fd Z e d'd Z d'd Ze d(d Zd)d Zd'dZd'dZ d*fd Ze d(dZe d(dZd+dZd'dZe d,dZe d,dZe d,dZe d,dZe d-dZd.dZe d,dZd/dZd0dZ d1dZ!d'dZ"e d,fd Z#xZ$S)2rz This is a "fake" scheduler node that represents a group of scheduler nodes that are meant to be fused together. The way it does this is by maintaining its unmet dependencies as the union of its constituent nodes. rrc|j|jusJt|ttfsJ|j r6t|t r&t|j tjsJt|ttfsJttj|j|j}||j|Sr~) rrrrrrRrr0rBrrrr)rxrrrqs rsryzFusedSchedulerNode.fuseA s%//111%-1C!DEEE    :e5N#Oejj"..9 99em5G%HI IIY__U__%68IJK5??E**ruc|jD]6}t|tsJ|jsJ|j 8|Sr~)rrrrrSrrrs rsrSz,FusedSchedulerNode.extract_pw_from_reductionN sJ{{ 0Gg}5 55'') ))  - - / 0 rucj|jD]$}t|tsJ|j&yr~)rrrrQrs rsrQz(FusedSchedulerNode.swap_pw_red_dimensionU s1{{ ,Gg}5 55  ) ) + ,ructtdd|jD}t|dk(ryt |}|S)Nc3|K|]4}|js|jr|j6ywr~rrrers rsrz4FusedSchedulerNode.estimate_flops..` 6'')T^^-=''):<rrfilterrrrrrfpsr|s rsrez!FusedSchedulerNode.estimate_flopsZ K  $ 0    s8q=#h ruc|jryd}|jD]`}t|tsy|;t |t |j dk7rt jdy|j d}bd}|Jt||jcxk(r|jk(rnn|j|}|s%t jd|jytxjdz c_ t jd|j||jD]%}t|tsJ|j|'t|y) z@ Return true if a loop reordering is performed. FNrz1Can not reorder fused node due to different sizeszODont reordering fused node %s because we can not decide the suitable loop orderrz-Reorder loops for fused node %s with order %sT)rrrrrrrarrr^r_rr1r`rEr)rrrnrorbrrDs rsrpz,FusedSchedulerNode.reorder_loops_by_dep_pairl sI     [[ )Ee]3%% *;uU\\RS_?U*U!''GaJ ) %%% z?h// E93E3E E ;;IFI  # #a  ##q(# ;T]]_i [[ 2Ee]3 33  & &y 1 2 (-ruct||t|||g|_t |dj |_y)Nc4t|jSr~)rrrs rsr8z-FusedSchedulerNode.__init__.. ss1>>3C/Drur)rr:rrrYr)rrrrrs rsr:zFusedSchedulerNode.__init__ s8 #i0%' %DEKK ruczdj|jDcgc]}|jc}Scc}wNr=rdrrrrrs rsrzFusedSchedulerNode.get_name )xxt{{;!;<<;8c<|jdjSrrrrqs rsrz!FusedSchedulerNode.get_first_name {{1~&&((ruc|tj|jDcgc]}|jc}Scc}wr~rrrrrs rsrz#FusedSchedulerNode.get_buffer_names . !L1!"4"4"6!LMM!L9cjg}|jD]!}|j|j#|Sr~rrr<rrrrs rsr<zFusedSchedulerNode.get_outputs 4(*KK .D MM$**, - . ruc ~t|jDcgc]+\}}|jd|d|j-}}}|jdj}||j |j tjdj|jdScc}}w)Nz.snodes[z] = rrHr) rrrrrrr[rrrdrV)rrrrres rsrRz"FusedSchedulerNode.debug_str_extra s%T[[1 4}}xs%0@/A B  {{1~""   LL335 6tyy/668&AA s0B9ch|jDcgc]}|j}}|d|Scc}w)Nz , snodes: )rrh)rrr snodes_strs rsrhz"FusedSchedulerNode.debug_str_short s99=Ed**,E Ez*..Fs/ct|||t}t|jD]/}|j|||j |j 1yr~)rrrrrupdater")rrrrrrs rsrz!FusedSchedulerNode.set_last_usage s\ 24FG0:|T[[) 8D    35G H  & &t 7 8ruc|tj|jDcgc]}|jc}Scc}wr~)rrrrrs rsrz$FusedSchedulerNode.used_buffer_names s.!MA!"5"5"7!MNN!Mrc|tj|jDcgc]}|jc}Scc}wr~)rrrrrs rsrz/FusedSchedulerNode.used_or_aliased_buffer_names s38< D1a,,. D  Drc|jSr~rDrqs rsrzFusedSchedulerNode.get_nodes rgrucTt|jd|jdS)Nz(nodes=r@rArqs rsrBzFusedSchedulerNode.__repr__ s't*%%&gdmmo->a@@ruc:td|jDS)Nc3<K|]}|jywr~)rrs rsrz2FusedSchedulerNode.is_reduction.. s91>>#9rrrrqs rsrzFusedSchedulerNode.is_reduction s9T[[999ruc:td|jDS)Nc3<K|]}|jywr~)rrs rsrz6FusedSchedulerNode.is_native_matmul.. s=A1%%'=rrrqs rsrz#FusedSchedulerNode.is_native_matmul s====ruc:td|jDS)Nc3<K|]}|jywr~)rrs rsrz3FusedSchedulerNode.is_split_scan.. s:1??$:rrrqs rsrz FusedSchedulerNode.is_split_scan s:dkk:::ruc:td|jDS)Nc3<K|]}|jywr~rrs rsrz1FusedSchedulerNode.is_template.. s8q1==?8rrrqs rsrzFusedSchedulerNode.is_template s8DKK888rucj|jD]$}|js|jcSyr~)rrrrrrs rsrz$FusedSchedulerNode.get_template_node s5KK 0D!--// 0ruc |jdSr)rrqs rsrzFusedSchedulerNode.get_device szz!}ruc:td|jDS)Nc3<K|]}|jywr~)rrs rsrz>FusedSchedulerNode.has_aliasing_or_mutation.. sEA1--/Errrqs rsrz+FusedSchedulerNode.has_aliasing_or_mutation sEEEEructr~NotImplementedError)rrrzs rsr{z'FusedSchedulerNode.update_mutated_names !!ructr~r)rrrs rsrzFusedSchedulerNode.add_fake_dep rructr~rrs rsrzFusedSchedulerNode.can_inplace rruc|j}djd|jD}t}|j |dt |j d|d|dt|jjd|dt|jd|d t|jj|jz d|d |jd|d |jd|d |j5|jD]!}|j |j!# d d d |j#d |j |j%|j-j/S#1swYXxYw#t&$rt(j+ddYLwxYw)rDrc3FK|]}t|jywr~)rrrs rsrz/FusedSchedulerNode.debug_str.. sFQQ 0 0Fs!rrErFrGrHrIrJrKrLz.outputs = [ NrrMTrN)rrdrrXrPrrrrrQr3rr#r$rr<rrrRrSrTrUrrV)rrr node_typestrrrs rsrzFusedSchedulerNode.debug_str s}}xxF$++FF  bd  Q|n-j))00123WT%<%<=>?74#3#3#9#9D." sG4t,,.Gr)rrrrrs rsrz#FusedSchedulerNode.has_side_effects s0 ;; "G4;;GG Gw'))rurrfrrfrrrrrr)rrrrrrrrrr+rrrr)r torch.devicer)rr8rrr)%rrrrrrryrSrQrOrerpr:rrrr<rRrhrrrrrBrrrrrrrr{rrrrrrs@rsrr8 s $# +% +.? +  + +, ")!).7) )VL ==)NN B/ 8#2 8HV 8  8OO  A::>>;;99 FF """*8**rurcHeZdZdZdfd Z ddZddZddZxZS) FusedMixOrderReductionszNFused node for two reductions with different iteration orders (inner + outer).c`tj|stj|sJ||}}||_||_t||j t|jt|jztj|j|_ yr~) rrrrrr:rrrrr)rrrrrs rsr:z FusedMixOrderReductions.__init__) s 33E:$77> >> %5E    OOT%//"34tEOO._get_ancestors..S s:qq{{:rvrrrqrs rs_get_ancestorszAFusedMixOrderReductions.sub_node_can_fuse.._get_ancestorsQ s ,C399:E:; ;rucBt}|jd|DS)Nc3<K|]}|jywr~)rrs rsrzZFusedMixOrderReductions.sub_node_can_fuse.._get_operation_names..Y sF1q446Frrrs rs_get_operation_nameszGFusedMixOrderReductions.sub_node_can_fuse.._get_operation_namesU s",C399FFG Gru) count_bytes)rqtuple[BaseSchedulerNode, ...]rr) rrrrrrrscore_fusion_memoryr)rrrr other_nodesrr s rssub_node_can_fusez)FusedMixOrderReductions.sub_node_can_fuse5 se%<===e%<=== ~~&&ueu&U  / /  #66u= < H0 H  H u~.1Ek1RR{+.BE5>.RR""$ $ ~~11%E1Rzz ruc@tjj}|dkDrt}t j |j |j D]I}|jjD].}t|ts|j|j0Kt||kDrtxjdz c_yt|t sR|j#|j$||j&fxs(|j#|j&||j$fS|j#|j$|j$|j&|j&fxr/|j#|j&|j&t)S)NrrF)r-rmix_order_reduction_max_readsrrrrrrrr9rrrr1#rejected_mix_order_reduction_fusionrrrrr)rrother max_reads all_readsr^rs rs can_fuse_withz%FusedMixOrderReductions.can_fuse_withg sDMM?? q=)3Ioodnn&68IJ 0>>//0C!#y1! chh/0 09~ );;q@;%!89)) EDJJ=J'' EDJJ=I J )) EKK$**ekk)BK((U[[%'J KrucT|jj}|jj|}t |t rX|j |j|j}|j |j|j}t ||S|j|j||jfr2|j |j|}t ||jS|j |j|}t |j|Sr~) rrrr#rrryrr)rrrrbackend fused_node1 fused_node2rs rs fuse_withz!FusedMixOrderReductions.fuse_with s&&(..,,V4 e4 5!,,tzz5;;?K!,,tzz5;;?K*; D D%%djj%$**G$\\$**e< .z4::FF$\\$**e< .tzz:FFrur)rrfrrfrr )rrf) rrrrr:rrrrrs@rsrr& s9X =0  0 !0 3 0 dK2Grurc4eZdZdZdfd ZddZddZxZS)FusedNestedReductionsz Fused node for two dependent reductions over the same logical elements. The outer reduction owns the codegen grid; the grouped reduction is staged inside that grid. c||_||_t| |jt |j t |j z|xj|jzc_|}|j\}\}}tj||}|J|\}} ||_ | |_ |j\}\} } tj|| | | |} | J| |_|jtj j"u|_|j'\} }tj)|||g| || | f|_y)Nrr)rrrr:rrrrrrr2rdrQrcrrrMrKgroup_size_in_rr\rWrh)rrrrr7r=rSrTrrQexact_group_sizerrrrarbrs rsr:zFusedNestedReductions.__init__ sn    OOT%//"34tEOO!0!P!P ." &111.D++0A)9).& &K&77     8 '''9E%)%6%6/:U:U:W:W%W%6%A%A%C" ]  2 2"3+-'E'E}'E$/#> 3  ruc|jry|jj|jzsytj |j |j}|ytj|j |sytd|Dry|jj|j|||S)zAllow downstream pointwise of the grouped reduction to fuse in. Consumers fused directly into the grouped reduction must run at either reduced-output resolution or full parent-tile resolution. Fc3ZK|]#\}}|tjju%ywr~)r2rHrF)rr=rys rsrz6FusedNestedReductions.can_fuse_with.. s- 6 o55KK K r can_reorder) rrrrr2rlrhrrgrr!_can_fuse_nested_reduction_append)rrrr%ris rsrz#FusedNestedReductions.can_fuse_with s     ..05??B  = =    OO     $@@   !2   .  ~~?? JJ  # @  ruc|jj}|jj|}|j |j|}t |j |Sr~)rrrr#ryrr)rrrrr new_node2s rsrzFusedNestedReductions.fuse_with sM&&(..,,V4LLU3 $TZZ;;rur)rrfr%r}rr})rrfrr)rrrrr:rrrrs@rsrr s ( T% N s013 &&,, Q Y( (ruct|}jr|jrtjttjt|}t j t |j k(}|s|d|xr2tfdtj |j DS|jrkjr |dytjt|}|j}||jj|S|dyjrk|jr |dytjtj|}|jj||S|dytd) Nzforeach do not have same lengthc3\K|]#\}}jj||%ywr~)rr)rlrrEs rsrz6ForeachKernelSchedulerNode.can_fuse..Z s0)Aq""++Aq1)s),zXcandidate producer is a reduction, foreach ops cannot be fused with reductions currentlyFz5candidate producer is not dep of any foreach consumerzXcandidate consumer is a reduction, foreach ops cannot be fused with reductions currentlyz5candidate consumer has no dep in any foreach producerzXAt least one node passed to ForeachKernelSchedulerNode.can_fuse should be a foreach node)rrrrrBrrrryrrFrrrNrw)rxrErJwhy foreach_matchconsumer_subnodeproducer_subnodes ` rsrz#ForeachKernelSchedulerNode.can_fuseQ s(+    X%8%8%:{{#=xHH{{#=xHH0C4HHM 56 S)A)&  "$$&n{{#=xHH'@@J +))228=MNN G H  "$$&n{{#=xHH'@@J +))223CXNN G H f  ruc |js|jsJ|jr3tjt|}|j}|j }n2tjt|}|j}|j }d}d}|jr|jr|tjt|}tjt|}t |j|jDcgc]\}}tj||} }}n/|jrtjt|}|j|} g} |}d}|jD]A} | | ur*tj| |} | }| j| 1| j| Cn|jrtjt|}|j|} g} |}d}|jD]A} | | ur*tj|| } | }| j| 1| j| Cn td||j| ||||Scc}}w)NzTAt least one node passed to ForeachKernelSchedulerNode.fuse should be a foreach node)use_custom_partition_algo prev_node_1 prev_node_2enable_autotune)rrrrBrXr[ryrrryrNrrFrwr)rxrErJrXr[rYrZrQrR fused_nodesrVrnew_noderUs rsryzForeachKernelSchedulerNode.fuse sZ""$(;(;(===    {{#=xHH(0(J(J %&66O{{#=xHH(0(J(J %&66O     X%8%8%:{{#=xHH{{#=xHH AAq#''1-K "{{#=xHH'@@J K"KK  -++166tXFH"*K&&x0&&t,  - "{{#=xHH'@@J K"KK  -++166xFH"*K&&x0&&t,  -!f     &?##+   Ks I?c|i_i_||qt |||D]Z}|jj D]} |j| j <|jD]} |j| <\n<|_|_ d_ g_ jtjj|j|jgt!fdt!j"|j$|j$Djj&z _t)|j*|j*g_t-|j.|j.g_t)|j0|j0_t-|j2|j2_|j5rt7|t8sJ||} } nt7|t8sJ||} } | j:_j:j=| j:| j_| jD]} | j| <jD cic]'} | j>jAD]\}}|| )c}}} _|_!|djE}|sJ|tGjHdfff_%t!tLjNjP_)|_*|_+ycc}}} w)Nc3ZK|]"}|jjvr|$ywr~rrs rsrz6ForeachKernelSchedulerNode.__init__.. s0xxt'<'<'>> rr combo_kernel),rDrIrr:rrrrrrrrrxr/rrrrr3rQrZr%rYr&r#r$rrrBrrr.rrXrrExprrr`fxNoder$r[per_subkernel_blocks)rrrrrXrYrZr[rdrrr foreach_noder rrvrrs` rsr:z#ForeachKernelSchedulerNode.__init__ s  +"5 G Y / 3 ,,228D37D%%dii08!4463D.2D%%d+3  3'DN DKDI)+DJ  ''22 ,,k.E.EF )//#66 8V8V ""))*  #!+"7"79N9N!OPDN +"7"79N9N!OPDN&).. 0N0N'D #'*.. 0N0N'D #%%'!+/IJJJ+6 j !+/IJJJ+6 j )33DN NN ! !*"6"6 7 , 9 9D "668 5*4!!$' 5#'++@@%:O:O:U:U:W@26!Q1@@D *C&%%' v > :<>? !%((--02 .$8!@s ,L7c |Dcgc]}t|ts|}}|rStjdt ||Dcgc])}|j |j j +c}|Dcgc]}t|ts|}}|rtjdt ||Dcgc]}t|ts|}}|rtjdt ||Dcgc]}t|ts|}}|rtjdt ||Dcgc])}t|tttttfs|+}}|Dcgc]}t|ts|} }| rtjdt | |Dcgc]}t|tr|}}|Dcgc]}|js|} }| r tjdt | | |Dcgc] }|| vs| }}tjra|Dcgc]}|js|} }| rtjdt | |Dcgc]}|jr|}}|Scc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}w)Nz/ComboKernels: %d external nodes are filtered %sz+ComboKernels: %d grouped nodes are filteredz;ComboKernels: %d FusedMixOrderReductions nodes are filteredz9ComboKernels: %d FusedNestedReductions nodes are filteredz+ComboKernels: %d foreach nodes are filteredz0ComboKernels: %d template nodes are filtered: %szCComboKernels: %d reduction nodes are filtered (pointwise_only mode))rrRrTrrrrrrrrrBrr-combo_kernels_pointwise_onlyr) rxrqrexternrgrouped mix_ordernested_reductionsfiltered_nodes foreach_nodestemplate_nodesreduction_nodess rscombinable_nodesz+ForeachKernelSchedulerNode.combinable_nodes s#Oj4M&N!OO  IIAF 5;UTtyy?T&&(U  $Kz!5I'J1KK  II=G  !&P1A7N)OQP P  IIMI ).V1A?T1UQVV  IIK%&   *-(+)       & A7Q)RA    IICSEW X% Z;U-VA  &4Gq}}!GG  IIBN#  &4Oq7N!OO  . .*8MQANN">tQ"GHOOPTUV$1$8$8$:L!((&+1c,.?%O !)Q->? % :K @s 5F .F4Callable[[Scheduler], list[list[BaseSchedulerNode]]]!group_algorithm_for_combo_kernelsc|t_yr~rBr)custom_group_algorithms rs%set_group_algorithm_for_combo_kernelsz@ForeachKernelSchedulerNode.set_group_algorithm_for_combo_kernels s # #Druc,tj|Sr~rrs rsgroup_nodes_for_combo_kernelsz8ForeachKernelSchedulerNode.group_nodes_for_combo_kernels s*KKIVVructr~rrqs rsrz#ForeachKernelSchedulerNode.mark_run rructr~rrqs rsrz"ForeachKernelSchedulerNode.codegen rrucyr rrqs rsrz%ForeachKernelSchedulerNode.is_foreach rruc,t|jS)zeReturns a list of nodes which comprise the combo kernel. These nodes may be vertically fused.)rrrqs rsget_subkernel_nodesz.ForeachKernelSchedulerNode.get_subkernel_nodes sDKK  ructttjjd|jDS)zqReturns all nodes contained in this kernel, unpacking fused nodes into their constituent scheduler nodes.c3<K|]}|jywr~)rrs rsrz7ForeachKernelSchedulerNode.get_nodes.. s1UA!++-1Ur)rrrrzrrqs rsrz$ForeachKernelSchedulerNode.get_nodes s(IOO111U1UUVVruc<|jdjSr)rrrqs rsrz)ForeachKernelSchedulerNode.get_first_name s{{1~,,..ruct|||jj|jD]}|j |yr~)rrrrr)rrrrs rsrz/ForeachKernelSchedulerNode.prune_redundant_deps s= d$68R8RSKK :D  % %&8 9 :ru)rErfrr)rJrfrrrErfrJrfrr})rErfrJrfrrB)NNFF)rrrrrXr}rYrrZrr[r}rdr}rrrqrrr)rrrlist[list[BaseSchedulerNode]])rrrrrrrrrrr)rrrrrFrNrrryr:rqr0rrrrrrrrrrrrrrs@rsrBrB/ s ) !) !&, , \> (> 4E> #> > J1504 %%*N9N9(N9$( N9 . N9 . N9N9#N9 N9`F+F FFP00 &00h /&(/ T   WW &WW ""! W /:">: :rurBceZdZUdZded<eddZ d dfd ZddZddZ e ddZ dd Z e dd Z dd Ze dd Zdd ZddZeddZxZS)raC This is a "fake" scheduler node that represents a group of scheduler nodes that are meant to be *grouped* together (it does not allow another node to be scheduled in between its constituent nodes, nor does it allow another node to fuse into any of its constituent nodes). The way it does this is by maintaining its unmet dependencies as the union of its constituent nodes. Fusion will still happen among the nodes within each GroupedSchedulerNode. At codegen time, this scheduler node will be unpacked and codegen is called on each constituent node. rrc|djtfd|DsJ||}|D]}|j|j<!|j|j<|S)Nrc3:K|]}|juywr~r)rrrs rsrz.GroupedSchedulerNode.create.. sB44>>Y.B)rrrr)rxr grouped_snoderrs @rscreatezGroupedSchedulerNode.create sy1I'' B6BBBBIv.  KE=JI ( ()9 : KAN $$]%;%;%=>rucLt||t|||||_yr~)rr:r temp_grouping)rrrrrrs rsr:zGroupedSchedulerNode.__init__ s( #i0 +ruc6|jr |jS|jD])}||jj|j <+|jj|j =|jj |jS)z Do fusion among nodes within this GroupedSchedulerNode, and then unpack this GroupedSchedulerNode into regular nodes. )rrrrr fuse_nodes)rrrs rsunpackzGroupedSchedulerNode.unpack sx   ;; [[ HEBGDNN - -enn.> ? H NN - -dmmo >~~((55ruc|j|jj||jj |yr~)rxrr}r3r)rrfake_deps rsrz!GroupedSchedulerNode.add_fake_dep s5 T--77AB ##H-ruczdj|jDcgc]}|jc}Scc}wrrrs rsrzGroupedSchedulerNode.get_name rrc<|jdjSrrrqs rsrz#GroupedSchedulerNode.get_first_name rruc|tj|jDcgc]}|jc}Scc}wr~rrs rsrz%GroupedSchedulerNode.get_buffer_names rrcjg}|jD]!}|j|j#|Sr~rrs rsr<z GroupedSchedulerNode.get_outputs rructtdd|jD}t|dk(ryt |}|S)Nc3|K|]4}|js|jr|j6ywr~rrs rsrz6GroupedSchedulerNode.estimate_flops.. rrrrrs rsrez#GroupedSchedulerNode.estimate_flopsrruc|jSr~rDrqs rsrzGroupedSchedulerNode.get_nodesrgrucX|jr|jdjSdSr)rrrqs rsrzGroupedSchedulerNode.get_devices$.2kkt{{1~((*CtCrucyrmr)rxrErJs rsrzGroupedSchedulerNode.can_fuserqru)rrrr)F)rrrrrr}rrr)rr8rrrrrrrrr)rrrrrrrr:rrrOrrrr<rerrrrrs@rsrr s $#$ + +( + +  + 6.==)NN "Drurc  tjdfd }ttt t d}t |dkDr|Dcgc]}| c}t jr|j||Scc}w)z A heuristic to decide loop iteration orders. This has not been well tuned and may be something we should autotune. ct|dk(s|dk(rt|dk(|dk(SDcgc]}t||}}Dcgc]}t||}}tdt||D}tdt||D}||kDry||kDryt||Scc}wcc}w)Nrc3:K|]\}}|dk(xs||kywrNrrsl_asl_bs rsrz5pick_loop_order..index_cmp..:) )3tDAI $ $ c3:K|]\}}|dk(xs||kywrrrs rsrz5pick_loop_order..index_cmp..=rrr)rPabsrry) rbsl stride_len_a stride_len_ba_firstb_firstrCstride_lengthss rs index_cmpz"pick_loop_order..index_cmp-s 8q=E!HMuQx1}eAh!m4 4.<>#33 .  E$5$55 .  -  ruc6|jjSr~rrqs rsrzNodeUser.get_namerruc|j|jusJt|j|jxr |j|jxr |jSr~)rr1rrrs rsrzNodeUser.mergesPyyEJJ&&& II    2!2!2 LL *U]]  ruNr)robjectrr}r)rr1rr1) rrrrrrrrrrrrursr1r1s3 ((KGTL $ rur1ct}|j}t|tjr|j t |jt |jzt |jzt|tjr$|j t|j|S| Jd||S)z=Get free symbols from a node's layout (size, stride, offset).z*Expect layout to be None but found layout=) rmaybe_get_layoutrr0Layoutrr&rstrideoffsetr get_layout_symintsr)rfree_symbol_usesrs rsrrs1;  " " $F&"))$  %6==) *6==) * fb;; <  # #$6v}}$E F ~T!KF8TT~ ruc"t|tr(tjd|jDS|j J|j j }|jd|j jD|S)z Gets symbols used in a scheduler node, including free symbols from the node's operations and layout symints from outputs. c32K|]}t|ywr~get_scheduler_node_symbol_uses)rrs rsrz1get_scheduler_node_symbol_uses..s M,U3 Mc32K|]}t|ywr~)r)rir_nodes rsrz1get_scheduler_node_symbol_uses..s M' W % Mr) rrrrrrget_free_symbol_usesrr<)rrs rsrrs$*+!z|!! M M   99  yy557 MTYY5J5J5L M rucv|j}||j |jStjSz4Check per-template flag, fall back to global config.)rallow_epilogue_fusionr-epilogue_fusionrtbs rs_is_epilogue_fusion_enabledr8  ( ( *B ~"22>'''  ! !!ru check_configcH|rtjsyt|tsyt|jt j syt|jjt jsy|jjjdk7ry|j}t|dk7ry|djst|djdk7ry|jj}t!|xrt#d|DS)NFrrrc3^K|]%}t|txr|jdk('yw)rN)rr9r)rwrites rsrz3_is_atomic_add_mutation_epilogue..s- HM 5)$C|)CC s+-)r-epilogue_fusion_with_atomic_addrrrr0rrScatter scatter_moder<rrrrrQr}r)rrr,rQs rs _is_atomic_add_mutation_epiloguersFBB dM * dii!2!2 3 diinnbjj 1 yy~~""l2 G 7|qqz3wqz'?'?'A#Ba#G    $ $F < C QW rucp|j}t|tjxr t |Sr~)rrr0TritonTemplateBufferr)r epilogue_node template_bufs rs&_can_fuse_atomic_add_template_epiloguer s7!224L b--  : *= 9:rucv|j}||j |jStjSr )rallow_prologue_fusionr-prologue_fusionrs rs_is_prologue_fusion_enabledr$(rrucb|jxr|j xr t|Sr~)rrrs rsis_epilogue_fusionr&04  /!!# # / ' .rucb|jxr|j xr t|Sr~)rr$rs rsis_prologue_fusionr)8r'ruc6t||xs t||Sr~)r&r)rs rsis_template_fusionr+@s eU + O/A%/OOruc"t||r|S|Sr~)r&rs rstemplate_fusion_pw_noder-Ds&ue45?%?ruceZdZUdZej eZded<ej eZ ded<e d dZ ddZ dd Z dd Zdd Zy )_LoopStateSnapshotaCaptured loop state for a set of scheduler nodes, restorable on rollback. Stores both SchedulerNode loop state (body, sizes, deps) and FusedSchedulerNode group assignments, since the latter is reassigned directly after child reindexing and has no mutation listener. rz$dict[SchedulerNode, tuple[Any, ...]]scheduler_node_stateszdict[FusedSchedulerNode, Any]fused_node_groupscD|}|D]}|j||S)zDCapture scheduler-node boundaries and their mutable leaf loop state.) snapshot_node)rxrqsnapshotrs rsrz_LoopStateSnapshot.createXs-5 )D  " "4 ( )ruc^||jvsJ|j|j|<y)z?Capture one leaf scheduler node before its first loop mutation.N)r0r9rrr^s rs_snapshot_scheduler_nodez+_LoopStateSnapshot._snapshot_scheduler_node`s/33333)+)?)?)A""2&rucV||jvsJ|j|j|<y)zACapture fused-node group metadata changed outside leaf listeners.N)r1rrs rs_snapshot_fused_nodez'_LoopStateSnapshot._snapshot_fused_nodees*411111'+zzt$ruct|tr|j||jD]$}t|ts|j |&y)zBCapture a scheduler node boundary and all mutable leaf loop state.N)rrr9rrr7)rrrr^s rsr3z _LoopStateSnapshot.snapshot_nodejsI d. /  % %d +.." 2B"m,--b1 2ruc|jjD]\}}|j||jjD]\}}||_t |y)z>Restore all captured loop state and fused-node group metadata.N)r0rr<r1rr)rrr^r;rrs rsrestorez_LoopStateSnapshot.restorersc3399; )IB  ! !% ( )11779 2KD%DJ +D 1 2ruN)rqr rr/r^rrr)rrrrrrfrrr)rrrrrrrxr0rr1rrr7r9r3r<rrursr/r/HstCT+BSBSC?8I{7H7H84B 2 22rur/ceZdZUdZded<ej eZded<dZ ded <e dd Z dd Z dd Z dd Zy)_LoopMutationTrackera+Rollback scope for speculative loop mutations during can_fuse(). can_fuse() may speculatively reorder or reindex loops while evaluating whether a fusion is legal. If the final decision rejects the fusion, this tracker restores the original loop structure so later fusion candidates do not inherit a speculative layout chosen for a fusion that did not happen. The first active tracker for a SchedulerNode leaf owns that leaf's listener. Recursive can_fuse() calls reuse the outer listener instead of installing nested listeners, so the captured state is the original state at the outermost decision boundary. Usage: call finish(commit=True) to keep mutations, or finish(commit=False) to restore the original state. If no mutation occurred, finish() is a no-op. r rqrzOrderedSet[SchedulerNode] watched_nodesNz_LoopStateSnapshot | Noner;ct|}|t|}|D]9}|jD]$}t|ts|j |&;|S)z?Create a rollback scope and watch mutable leaf scheduler nodes.)rq)rrrrrwatch)rxrqseentrackerrr^s rsrz_LoopMutationTracker.creates^% E$K( &Dnn& &b-0MM"% & &rucv|jy|jj||j|_y)z$$ .B)-B & .4::-  ru)rqr rr@r=)rJr}rr)rrrrrrrrrAr;rrrCrGrKrrursr@r@{sZ" )(/@{/@/@"0M,(,E $+0 ;rur@c.eZdZdZddZdfd ZddZddZddZddZ ddZ e dd Z e jdd Z dd Zdd Zdd ZddZddZddZddZddZ ddZddZddZddZddZddZddZddZddZ ddZ d ddZ ddZ! dd Z"dd!Z# dd"Z$dd#Z% d dd$Z& dd%Z'dd&Z( dd'Z) dd(Z* dd)Z+ dd*Z, dd+Z- dd,Z. dd-Z/e0 dd.Z1ddd/Z2dd0Z3 dd1Z4 dd2Z5dd3Z6 dd4Z7 dd5Z8 dd6Z9 dd7Z: dd8Z; dd9Z< dd:Z= dd;Z> dd<Z? dd=Z@dd>ZA dd?ZB dd@ZC ddAZD ddBZE ddCZF d ddEZG d ddFZH d ddGZIddH ddIZJ ddJZKe0ddKZLdDdL ddMZM ddNZNe0ddOZO ddPZPe0ddQZQdddRZR d ddSZS d ddTZTeU d ddUZVeU d ddVZV d ddWZV ddXZW ddYZX ddZZY dd[ZZdd\Z[dd]Z\dd^Z] dd_Z^dd`Z_ddaZ`ddbZa ddcZbdddZcedddeZe ddfZf ddgZg ddhZh ddiZi ddjZj ddkZk ddlZl ddmZm ddnZn ddoZo ddpZpddqZqddrZr ddsZs ddtZt dduZuddvZvddwZw ddxZxddyZyddzZzdd{Z{e dd|Z|e dd}Z}dd~Z~ddZddZxZS)rz A Scheduler is a graph of BaseSchedulerNodes. It is responsible for optimizations such as fusion, reorder, and graph partition. cftd5|j|dddy#1swYyxYw)NzScheduler.__init__)r_initrrrqs rsr:zScheduler.__init__s, . /  JJu    s'0c t tj_i_t t_tj_ t_ tgtjjjtjj jtjj"j_|Dcgc]}j'|c}_d_d_j/j$j1tjj jj(D]}|j3d_j7_j(Dcic]}|j;|c}_j(Dcic](}|j?D]}|j;|*c}}_ j<jC_"i_#i_$t_%tMjNj(j@jD_jQjSj(_jUj(Dcic]}|j;|c}_"jWjYtZxj\t_j(z c_.ddl0m1}m2}|j(t_j(_3jijSj(_ttjtltlf_7tpjr$tqjrj(_tpjtr'ddl;m<}|j{jWi_>i_?d_@i_Ajjj(_tpj$tqjj(_tdj(DrjUjjtpjstpjr./s   tA B rz#Scheduler.create_combo_kernel_nodesTlog_pt2_compile_eventlog_waitcounter) num_ck_nodes)reorder_for_peak_memory)1assign_memory_planning_info_for_scheduler_buffers)6align_runtime_estimations_across_all_distributed_ranks)trace_structuredartifactc dddS)N#scheduler_nodes_before_comm_overlapstring)rencodingrrrursr8z!Scheduler._init..us E$,)ruc djtjDcgc]0\}}d|d|jzd|j z2c}}Scc}}w)Nz zsnode[rz buffer_names:)rdrrqrr)rrrrs rsr8z!Scheduler._init..yslv{{ )2$**(= !%1%QCqMkkm, .q/A/A/C.DEF(s5A" ) metadata_fn payload_fn graph_statsc^jjtjdS)N)graph_idnum_nodes_before_fusionnum_nodes_after_fusion)post_grad_graph_idnum_orig_nodesrrqrqsrsr8z!Scheduler._init..s' 33+/+>+>*-djj/ru)wrr:rarrbackendsr_post_grad_graph_counterrjrcount_graph_partition_counterrrrXr! constantstorchbind_constantsrcreate_scheduler_noderq previous_node current_nodeupdate_zero_dim_cpu_tensorrrdefault_device_contextget_donated_buffersrrrIr<rcopyrrr)seen_template_fusionsr,decide_global_ordering_of_commsrdtopological_sort_scheduledead_node_eliminationcompute_ancestorscompute_input_distancesr1ir_nodes_pre_fusionrtorch._inductor.debugrQrRrkcreate_foreach_nodesrr,logged_slow_fusionr-_pre_fusion_custom_passdistributed_max_autotune_gemmrXrSschedulervbuff_to_stream_multi_stream_nodesstream_idx_to_user_obj_idx_populate_stream_assignmentsr_post_fusion_custom_passrr\finalize_multi_template_buffersmax_autotune_gemm max_autotunerr`raselect_algorithmPrecompileThreadPoolshutdown_instance combo_kernelsrcreate_combo_kernel_nodes_enforce_conditional_orderingrYmemoryget_output_names deterministic reorder_for_compute_comm_overlaprZr.6runtime_estimations_align_across_all_distributed_ranksrYrr[reorder_sink_verbose_loggingtorch._loggingr\$reorder_compute_and_comm_for_overlapprocess_grouped_nodesgraph_partitionrre%reorder_for_reducing_graph_partitions&maybe_reorder_for_minimizing_partition,reorder_for_partition_with_simple_dependencycompute_last_usage test_configstrack_memory_lifecycleinsert_memory_check_nodesr graph_diagramdebug_draw_graphbuffer_names_to_freeorigin_to_index_current_stream_ctxr#add_row removed_ops)rrrqrrrrQrRrSrYrZhas_collectivesr[r\rs` rsrNzScheduler._inits  <> "&'?"@(1(9%5?\!&0 %%**, ""'') ,,113 ' #>CCd003C 7;6: '') ##**177+<+<+A+A+CDJJ D OO  <@#  $ $ & #&*ZZ; !AJJL!O;  -1JJ8 $($BRBRBT8 ;>CLLNC 8 8 AE@Q@Q@V@V@X3513 L " :: JJ     # #  !!#33DJJ?  ""$<@JJ"Gq1::<?"G   $$& ##s4::6#O$**%!$**o !!#33DJJ? ",U38_"="?  ) ) 577 CDJ  / / . ) )$ /  " " $=?.0). :<' ))+__TZZ0  * * 688DDJ       & & (  ,,.  $ $(;(;&( OO , , A A S S U   5&* $ B ..D.A  B **,  ) ) 70   ''177//44671773356 DJ##(O(O11UAJJ 0 0RR"' JJD$TYY/*.#K4::V88; !  CCDJJODJ ""$  " " (( CCDDTZZPDJJJ4::VDJ ! ?? ! ! . . E E  * * ,4::& djj) 6@\!:<GK '//  -7LsD; 8 H#Hh B Bs$5b9?b>.-c(c  ccc i}tjjD]d}ttjj|tj s9t |tjj|d||<f|S)N)r)rargraph_inputs_originalrr0 DonatedBufferr )rrname_to_donated_bufrs rsrwzScheduler.get_donated_bufferssp GG11 D!''77=r?O?OP,BGG11$7 $-#D) #"rucv ddlm i}tjd}|jD]} }|j D|j j }|(||vrt|}|||<||j|<||}||j|<|jD]}||j|<t fd|jjDrtd|jDd}||jD]z}|j } |j t| t j"s;t| j$t j&s`t!j&|| _|t fd|jjD|_y)a=Populate node_to_stream and buff_to_stream from IR node stream_idx. Reads the stream_idx field set on IR nodes during lowering to determine which stream each scheduler node should run on. This field is propagated from 'custom.stream' FX node metadata via IRNode.current_stream_idx(). r)DEFAULT_STREAM_IDXNc3(K|] }|k7 ywr~r)rr9rs rsrz9Scheduler._populate_stream_assignments..sM1q&&Mc3^K|]%}|j|j'ywr~rrs rsrz9Scheduler._populate_stream_assignments..s!RAq||~7QRs--rc3(K|] }|k7 ywr~r)r stream_idxrs rsrz9Scheduler._populate_stream_assignments..s'  , ,' r)stream_constantsrrrnrqrget_stream_idxrrrvrrrrrrr0BufferrrDr) rruser_obj_to_stream_idxstream_idx_counterrr user_obj_idxnew_stream_idxrrr rs @rsrz&Scheduler._populate_stream_assignmentss 924&__Q/JJ 6D+Jyy$#yy779 +#+AA)-.@)A?M.|>:D AS H +  6 Iructjtjr8tj d||j D]}|j yy)Nz%s:)rT isEnabledForloggingINFOrjrqrk)rrlabelrs rsdebug_print_nodeszScheduler.debug_print_nodessF   GLL ) HHUE "  #  " # *ruc6|jJd|jr t||St|tj tj fr t||St|tjr t||St|)Nz2All nodes passed to scheduling must have an origin) ris_no_oprrr0rr!rrrRrrs rsrrzScheduler.create_scheduler_nodes!- @ - ==?)$5 5 r00"2C2CD E t, , boo .,T48 8%d+ +ruct}g}|jj}tjj j D]}|Dcgc]%}||vrt|j|ts|'}}|s6|j||Dcgc]}|j|}}tjdkD}t||d|}|j||D]}||j|<|jD cgc]} | j!|vs| c} t#|z|_ycc}wcc}wcc} w)NrF)rXr[)rrr!rarlistsrrrIrrr-combo_kernels_autotunerBrrqrr) rrremoved_node_namesfe_nodeskept_node_namesnamesrrr[fe_noders rsrzScheduler.create_foreach_nodes(sN.8l11668WW]]))+ 8E"?*"4#4#4T#:eZdZdZ d ddZddZd fd Zy) 1Scheduler.compute_dependencies..DedupListan This data structure behaves like a list except it makes sure the elements remain unique. Normally one could use a OrderedSet/dict for this purpose however the list in question gets elements appended as it is being iterated over which means that we need to keep the list semantics. Nc@|xsg|_|xs t|_yr~)rr membership)rrrrs rsr:z:Scheduler.compute_dependencies..DedupList.__init__\s #[b ","< ruc||jvry|jj||jj|yr~)rrrr)rr node_users rsrz8Scheduler.compute_dependencies..DedupList.appendds5/ !!),##I.ructj|j|j}|j|jDcgc]}||jvs|c}z}||Scc}wr~)rrrr)rrrnew_membershipr new_items DedupLists rs__add__z9Scheduler.compute_dependencies..DedupList.__add__jsc!+!1!1$//5CSCS!T JJ${{*at.FA* !N;;*s A+A+r)rzlist[_T] | NonerzOrderedSet[_T] | Nonerr)rrhrr)r DedupList[_T]rr)rrrrr:rr)rsrsrrRs; *.48 =& =2 =  = /  .renames,D)))d33A677HruFcP|jt|||yr~)rr1) used_by_namerrr name_to_usersrys rsadd_userz0Scheduler.compute_dependencies..add_users) &. / 6 6K9 ruNc|jSr~rtrs rsr8z0Scheduler.compute_dependencies..s AFFrurTz scheduling %s) unbacked_onlyc|jSr~rtrs rsr8z0Scheduler.compute_dependencies..s !&&ruz not in )r mutating_bufr)r)rzscheduling output %sz+scheduling output %s for unbacked symint %sr'z': rrzBUFFER USER LIST z===== AFTER SCHEDULING ===== %s)rr,rr,)FF) rr,rrrr}rr}rr)Er rhrWrrqr<rrrrr0rDrrrarrXrrrar& TensorBoxrcrget_unbacked_symbol_defsSymbolrTrr rIrr:rrQrrr9rrrrfr;additional_buffer_depsadditional_star_depsrrrr{r)rrrr rr!rmutated_inputsrrprmutated_input_idxsrrrXrPrrrVcompute_dependencies_log)*rrrbuf1 buf1_name buf2_namelist1list2combinedrrunbacked_symbol_to_origin_nodevalfsr9sym_sizehas_non_input_unbacked_defsunbacked_symbol_defsunbacked_symbol_usesrRrr node_modealt_namerout_buf other_nameis_aliasadd_deprrrrr inp_nameslogbufrkrfrr,rrrys*` @@@rsrdzScheduler.compute_dependenciesLsB  <  <@@K?V?V @ JJ LD((* L MMO tyy//?D,,./!3!%!1!1!3LI M1i=6P -i 8 -i 8#(5=#0>C -c 2e ;#0#5#>5= c 2 > #m33@3K i03@3K i0L L L< !&!  6      JL& 77''..0 BC#uzz***>B9=226>C.(+||~S!Auzz9RASS!BAnnB=A6r:BB B',#JJ HD99( (($* 224:J$ * H!!U\\222/3+::8< 215 H H JJV D IIotyy 1*yy,,,'-II222F(($ .GA >>#X&D%EF>>K#'#4#4Q#7#C#C#EGC --gclln.EFG GD$$++,1 d&6&6&=&=!>??S?sI.HH  '') E3,,./1444 # 1 1 3EH%h/HXt,%%ghY&GH -h 7 = =E==?dmmo=$)$))5FGGG'+yy'<'<'>EG)0)9)9);J)/ );J(073F3F3H'HH -- '$.1408L!"%ZtD%E E E EB7799$--/J S$5!!''4==?D"QR  S 7777 H 4$6!!''"23 4 ((.. F!$0TYYd.>.>t.DE F  % %d&;&; <'')  # 1 1 3H>AllnD))&*:;69llnD))(3//33HhG++CLLN; aV r002 >H II,h 7 Xz'(*;< = > 'ww,, N111E NA >>#X&D&I&I&K%LM>;1==q=(,(9(9!(<(M(M(ONHII M ( ! %Xz'(:K/LM N N N)) :Dqww+++z'$-89&&**40***z'$-89  :,5QWW5I5I5N5N5P+Q 'E4D%K  )*(>(>& $IdO& " JJ CD'') C mCLLN;AAB C C// SD  ' ' - 7 7 d8K8Q8Q R S ! c'--/ 4JC 4/4{{;!;; #c%23 4 4 4  c  "))+ &&';< &&'I3OKTX & "< 4 4s65i4i42 i9 i?j j5j j  j c ddlm}m}m}m}t t jjj}|j|}tjjjs|jjt t jj!}|j||\}} } t#t%jD cgc]} ggfc} |D]}} | j&dk(r| j(dk(r"| j*j-} | j.dj1| | j2dj1| ddlm} |  dfd } g}t9jD]H\}}|j1||j1| ||t%jdz k(J|_ ycc} w)Nr)rZcompute_memory_timelineFreeableInputBufferget_freeable_input_bufr)register_check_mem_opcX|d}|d}|||g}tjttjdtj j jjg|d}dj|j|_ t|S)Nrrrrc$||d|d|ddfS)Nrrr)alivedead is_final_stepr) tensor_argsrs rsr8zWScheduler.insert_memory_check_nodes..construct_mem_check_node..s*!.q!1 -a 0)6q)9Cru)rrr!nontensor_argsunflatten_args mem_check_) r0MemoryCheckKernelrDr`rops_inductor_debugcheck_memory_stepdefaultrqrrrR)step_idxr expected_newly_aliveexpected_newly_deadr"rrrstep_allocs_deallocss rsconstruct_mem_check_nodezEScheduler.insert_memory_check_nodes..construct_mem_check_nodes$8#A!#D "6x"@"C 24GWN''!e)<=yy00BBJJ-  D%/tzz(/C/L/L/N.O"PD ,T48 8ru)r )r*rr r}rrR)rrZrrrrrarrXr!rqr`rar-rYrrrLr size_alloc size_freer"r start_steprend_step#torch._inductor.runtime.debug_utilsrr)rrrZrrrrXname_to_freeable_input_bufr buf_info_listr=buf_inforrr. new_nodesrrr-s` @rsrz#Scheduler.insert_memory_check_nodes[s  )31773G3G3L3L3N(O "4::| < #%%== = D,, *4AGG4L4L4N)O 5 JJ &   q!$C O4C RHC & HH""a'H,>,>!,C//1H !4!4 5a 8 ? ? I !2!2 3A 6 = =h G H N 9 9*. 9 & 92  , GAt   T "   (1DJJRS@S;SU    eC s3 Hc| tjsyg}t|jD]}dd d}|j D]}t fd|j D}|r\tjd|jtjjj|jd}|j xr| }|s|j|tjd|jtjj j|j|j"j$D]}|j&|j(vs|j(|j&j }|Dcgc]0}|j*j|jk7s/|2c}|j(|j&_t-t||_|jD]}|j/ycc}w) z0 Remove any nodes without users Ncr|jxs*|jtjjvSr~)rrrarr)rs rscan_eliminate_userz;Scheduler.dead_node_elimination..can_eliminate_users&||Tt}}!'':T:T'TTruFc3.K|] }|ywr~r)rur:s rsrz2Scheduler.dead_node_elimination..s#Ma$6q$9#Mzremoved dead buffer: %sTzremoved dead operation: %s)rr1rr})r-use_dcerrqr<rrrTrrrarrrrrrrrrrrrr) rr updated_nodesractive_buffersr can_eliminaterrr<r:s @rsr|zScheduler.dead_node_eliminations~~   TZZ( D U#N'') * ##M399#M M  II7HGG++// ?%)N  *!% 5 5 77N.visit..s affrur)rrr3rr)rrrIrrDvisits rsrHz2Scheduler.topological_sort_schedule..visitse} !!"6"6.s%Xc))#.??AXs'*c3<K|]}j|ywr~r)rrrrs rsrz1Scheduler._get_unmet_dep_nodes..sQat66q9Q) rrrrRrrrr3rr RuntimeErrorrr)rrr unmet_depsr unmet_dep_opss` rs_get_unmet_dep_nodeszScheduler._get_unmet_dep_nodess&0l  )&"$   // )sxx( )=d5k]!L YZX JQ=QQRRruczg}tj|jd}i}|jD]P}|j|}t |||<|D]*}|j |g}|j ||||<,R|jD cgc] \}} | dk(s |} }} | rx|j | | D]7}|j |gD]} || xxdzcc<|j|9|jD cgc] \}} | dk(s |} }} | rx|rJd|Scc} }wcc} }w)zU Sort nodes by their topological order, return a list of node lists. rrzTopological sort failed!) rxfromkeysrqrVrrrrr) rrrrqchildrenrrrcrrfzero_deg_nodesrs rsrtz!Scheduler._topological_sort_nodessF djj!,#%JJ "D,,T2Dd)E$K "LLb) !  " "). @1a!@@ LL (# $LLB/%D$K1$K% !  -2KKMDDAqQ!VaDND 444y AEs D1%D1 D7D7cji}|jD]w}t}|jD]B}|j|jj }|j ||||z}D|||j<||_yt|jD]\}}||_ ||_ y)z. Populate each node.ancestors N) rqrr3rrrrrrrr%r&)rrname_to_ancestorsrrr dep_node_namers rsr}zScheduler.compute_ancestors.s 9;JJ 'D)3I.. > $ 0 0 : K K M  m,.}==  >2; dmmo .&DN '%TZZ0 #KE4"DN"DN #ruci}i}|jD]}|jsd}d}n|jDcgc]/}||j|jj dz1}}|jDcgc]/}||j|jj dz1}}t |}t |}|||j<|||j<||_||_ ycc}wcc}w)z Populate each node's min/max_input_distance with the depth from graph inputs, measured as dependency hops before fusion. Nodes whose dependencies are all satisfied by graph inputs/constants have depth 0. rrN) rqr3rrrrZrYrr#r$) rrname_to_min_distancename_to_max_distancermin_distmax_distr dep_min_dists dep_max_distss rsr~z!Scheduler.compute_input_distancesAs  02/1JJ /D** $66!))9)9#(()C)T)T)VW! ! $66!))9)9#(()C)T)T)VW! ! }-}-4<  14<  1&.D #&.D #) / ! !s 4C:74C?cHtjsy|jD]}t|tt fr#|j stjdk7r=|jD]3}t|tr|jr$|j5y)Nhalide) r-r%rqrrrr[ cpu_backendrrr\)rrrrs rsr\zScheduler.merge_loops_s00 JJ $Dd]4F$GHKKMf&8&8H&D) $!%75;L;L;N!!#  $ $ructddd5tdD]}t|}tj d|dz||j |d}t|}tj d |dz||||k(s|dk(sltj d |dznt jst jr|j |d}|cd d d S#1swYy xYw) zB Combine eligible nodes into FusedSchedulerNodes. zScheduler.fused_nodesTrUrz/===== attempting fusion (%d/10): %d nodes =====rF)is_reorder_roundz=completed fusion round (%d/10): fused %d nodes into %d nodes z+===== fusion complete (%d iterations) =====N) rrLrrrfuse_nodes_oncer-r%loop_index_inversion_in_fusion)rrrqrold_lennew_lens rsrzScheduler.fuse_nodeszs #4QU  2Y e*  EE ,,UU,Ke*  TE  g%A$$Eq1u' ,1188,,UT,J;   sA7C!AC!!C*cg}|jD]4}|jt|tr|j n|g6||_y)zA Unpack GroupedSchedulerNode into regular nodes. N)rqrrrr)rrr7rs rsrzScheduler.process_grouped_nodessJ.0 JJ D   !+D2F!G dV   ruct|dkDsJ|dj}||_|j|}t ddd5|j |cdddS#1swYyxYw) Benchmark fused list of nodes and return the execution time in milliseconds on randomly generated inputs. rbenchmark_fused_nodesTcompile_time_autotune_time_us)rVdynamo_compile_column_usN)rrrr#rrr)rrrqrrs rsrrzScheduler.benchmark_fused_nodesst5zA~~q$$&$""6*  #"&%D  8 007  8 8 8s A%%A.Nct|dkDsJ|dj}||_|j|}t d5|j |||cdddS#1swYyxYw)D Generate a kernel given a list of pre-fused nodes. rgenerate_kernel_code_from_nodes hint_overrideN)rrrr#rrw)rrrqbenchmark_kernelryrrs rsrwz)Scheduler.generate_kernel_code_from_nodessw5zA~~q$$&$""6* ; < ::'};   s A%%A.c||_|j|}td5|j|cdddS#1swYyxYw) Benchmark a compiled module and return the execution time in milliseconds on randomly generated inputs. benchmark_codegened_moduleN)rr#rr})rrmodulerrs rsr}z$Scheduler.benchmark_codegened_modulesH%""6* 6 7 >55f= > > >s ?Actjj}|sytj d|||j D]}|j }tjjj|s=|j}||}t|tjr'|j|j |j}t|tj"s||k7stj%d|||yy)z Check if selecting a Triton template would cause layout conflicts. Returns True if there's a conflict and we should fall back to ATen. FzNode %s has constraints %szOLayout conflict detected for %s: template expects %s but layout is frozen to %sT)rarbuffer_layout_constraintsrTrrrr`rarshould_use_layout_constraintsrrr0FlexibleLayout freeze_layout_with_exact_stridesr FixedLayoutrU)rr multi_node constraintsinpinp_namerexpected_layouts rs!_has_layout_conflict_for_templatez+Scheduler._has_layout_conflict_for_templatesgg77  . KH$$ C||~H??33QQRUVZZF)(3O&""3"3444_5K5KL&"..1o6O e#  3 6ruc &t|jD]\}}t|tst|jt j s=|j}tjjs%|j|s|j\}}n|jDcgc]2}t|tjjj r|4}}|sJdt#|dkDr!|j%t'|fd}n|d}t|tjj j(rtj*ri}||d<tj*D]k} |j%| j-D cic]\} } t| t(r| | } } } t'| j-dd} | || <m|jj/|n|jj1|t j2j5|j65|j9}dddj:}t|t j<sJ|j:}t|t j>sJ|j@rtC||j@|jD|_"|jG||||ycc}wcc} } w#1swYxYw) a Finalize a backing choice for MultiTemplateBuffers which did not already have a choice finalized through fusion. In the case of an extern choice, this will result in replacing the SchedulerNode. If a MultiTemplateBuffer did not have any fusion opportunities, finalizing a choice will force completion of compilation and benchmarking. z&No extern kernel detected for fallbackrc|Sr~r)rZtimingss rsr8z;Scheduler.finalize_multi_template_buffers.."s WUVZrurrNrxc |dSrHrrs rsr8z;Scheduler.finalize_multi_template_buffers..7s qQRtru)$rrqrrrr0MultiTemplateBufferr-r%force_extern_kernel_in_multi_templaterget_min_choicechoicesr`rarExternKernelCallerrchoice_timingsrZr"multi_kernel_hintsrfinalize_as_triton_callersfinalize_as_triton_callerrlcurrent_originsr$ output_noder StorageBoxOperationBuffer origin_noder?r _replace_node)rrrrrmin_node_unfusedr=rZextern_choicescallersrrrftriton_timingschoice out_tensorbox out_storage out_bufferrs @rsrz)Scheduler.finalize_multi_template_bufferss!,? DGAt$ .: 2114"YY ++QQ BB:N+5*C*C*E'$a",!3!3&%!OO<<OO&N&*S+SS>>*Q.",";";"=+.~CW+X(+9!+<($OO&&?? 00NP(8 %+$=$=3D&0&?&?d&?&SG-4MMO.$(Aq#-a1I#J!"1.N. &))=)=)?^%TUV%WF,2GDM3 <.rename_deps..asKscjj)9:Krr)rr)s rs rename_depsz,Scheduler._replace_node..rename_deps`sKdKK Kru)rr2rr2)rrrrqrIrrrrrrr3rrrryr<rrr%r&rr") rrrrrrnew_scheduler_noderr3rnew_outold_outr)s @rsrzScheduler._replace_nodeLs "*j9!77 C* 1 -?$--/*3E 0??4#3#3#9#94;R;RS 7C 3377$GGyG.1hh + 7 L1<  1 11 -0;  * * 0 00 &&,!$  * * ,d.>.>.@!  * GW4;D  W--/ 0#MMGM  * (,~~$'+~~$'+~~$(,%ruc&td|DS)Nc3K|]q}t|jdxrU|jduxrEt|jjdxr#|jjjdk(syw)rNrr)rrrrrs rsrz,Scheduler._any_atomic_add..vso  AFFF # 9d" 9 ^4 9 ((L8 9 sA7A9r)rr node_lists rs_any_atomic_addzScheduler._any_atomic_addus     ruc"|j|d|}tj|}tjj j }|jsd}||fS|jd|}t|tsJ||fS)NT)rzrytriton_) kernel_name source_code) rwr!loadr`ra async_compile AsyncCompileuse_process_poolrrr )rrrqrysrc_codemodrfuts rscompile_kernelzScheduler.compile_kernel~s77 D 8 x(55BBD --/C Sz &&9(&SCc<0 00Szruc !"#$%&'()*+,-./0tdfD}tjtjxr t d}|r%t jstjdSt js|stjdSjr(tjtjr jsjrtjdSj}|dj!!sJ!jdk(r(t j dk7rtjdSj}t#t%j&||/j)/}|r|stjdSddlmt/0/dj!!Jdfd )|rStd fDr>jd u""rjnj.t.tj0sJj3.rtjdSi(|sWg&t j4D]A}.j7| t9 j;d D]~\} } t| t<j>j@jBs5.jE| 5&jG| gjI/| jJd d d tMd} d } i} &D]V\} }} ||jO.jE| 5j]|!\}}|| | <|| kr|} | } d d d X| .j^|<t| t`sJ| (|<Dt jbted.jfD}tixr xr|t jjk'tMdtMdc+,d *'sR.j7 .jm\*+t9 j;tojpd }n.jfDcgc]}|df}}ddl9m:d".fd } d"./fd }|r/"stjdSgt j4d D]}.j7| d } t9 j;tojpd D]R\} } || stwjxt<j>j@jB| }||sP|} n| tjdcS| (|<t j4r.j{(n.j}(d tjdSr("rj|nj|\,}n4"stjdSj,t,-g&d}|D]\} }|| stwjxt`| } r |+,zk\rn\|dz }|t jjkDrnB.jE| 5 &jG| gjI/ d d d t&dk(rtjdSd !"&'()*+,-.fd }tj|&ddSjI|#jI|%jI/$d!#$%)0fd }tj|$dS#1swYxYw#tP$rI}tRjUtVjXrtRj[d"sdnd|Yd }~d }~wwxYw#1swYxYwcc}w#$r Yd d d wxYw#1swYxYw) If config.benchmark_fusion is False, always return True. Otherwise, return True if fusion can brings speedup. c3K|]>}|jxr(t|jtj@ywr~)rrrr0rrs rsrz.Scheduler.speedup_by_fusion..sE  MMO J1..0"2H2HI J sAAFrTrrrCompilationErrorNc ttjtjr|||zkrFtj dj j t ||z|z dytj dj j t|||zz dyy)Nz9can fuse (benchmark): fusing %s with %s cause %sx speedup.3fz=cannot fuse (benchmark): fusing %s with %s cause %sx slowdown)rrrDEBUGrrrJrL)ms_fusedrrrrs rs log_fusionz/Scheduler.speedup_by_fusion..log_fusions&&w}}5cCi'$$S..0..0"sSyH&.s#% 23A   ! -% rc |dSrHrrs rsr8z-Scheduler.speedup_by_fusion..s RSTURVrurrxinfException in compiling %s: %srrc3<K|]}t|tywr~)rr")rrZs rsrz.Scheduler.speedup_by_fusion..s%<= 167%rr) CantSplitct|tjjjsy xr't |dxr|j j k7 S)NFallowed_prologue_inps)rr`rarTritonTemplateCallerrr)rrrs rschoice_supports_fusionz;Scheduler.speedup_by_fusion..choice_supports_fusion5sb!EOO<<QQ!('Y(?@Y44 8X8XXrucj|5 j|j\}}||jn|jj dddy#$r Ydddyt $rP}tjtjrtjdsdnd|Yd}~dddyd}~wwxYw#1swYyxYw)NrxFrrrT) swap_as_triton_callerrryrr precompilerSrrrrr) rro mod_fusedr&rrrnode_list_fusedrrs rscompile_without_benchmarkingzAScheduler.speedup_by_fusion..compile_without_benchmarkingEs 55f=%%,0,?,?+6;O;O-@-) "-"MMO%--88:%&%%$%%%%%227==A&,, ?2A z ! %%%%%%&s;CA A++C0C:C>C C CCCc td}d}i}rhrttjsJj j \Dcgc] }|dvr| c}t fdD]0\}}} ||j}n!s|j}|jnd}r>j!|5j#|\} } | ||<| |kr| }|}ddd}|k(xsz|zkD} |s| s|j|j$r |j&sJ|j$d} | j&} | j(}t+|j&| ||j,j.t1j2}|s/|}nr |r|zkrL|Jt4j6r|d<j9nj;|r|j<d<yy cc}w#t$rI}tjtjrtjdsdnd|Yd}~d}~wwxYw#1swYxYw) Nrrc|dSrr)rrs rsr8zKScheduler.speedup_by_fusion..benchmark_when_ready..snQqT&:rurrrrTF)rqrr0rrrrrrrrSrrrrrrr} launchersn_regsn_spillsrbmreqrrHrr-rrr_choice_timings) min_ms_fusedms_fused_choice new_timings fut_choicerrorresr&rpathfusible_choicecompiled_kernelrrshould_fuse_epiloguebench_epiloguerrrfuture_choicesget_choice_timings_async hint_override_best_fusion_choicer min_choicerr ms2_fusedrrrs rsbenchmark_when_readyz9Scheduler.speedup_by_fusion..benchmark_when_readys$U| "& +%*ZAWAW*XXX%/%>%>%@N&0&?&?&AOJ +9&&%a=N:#&N &,&:&N 2@<&-FFI!!-"(--/C!/"+"3"3CNN,"&C&'==fE 9-1-L-L ) &.NHd 3;K/',6/7 28 9 9'&0N"Sy>&+AI+MM' >"--/#&==V]]B#B.1mmA.>O+:+A+AL-<-E-EN3A # # & , . & 6 6 0 7 7 ?40 428 %y<&|"|S#6',#)*D%100AP8>"==< #<<_M%;F 2248 }&.%!%227==A&,, ?2A z ! !! 9 9s*H4H $I. I+">I&&I+.I8 cddlm} d d dfD]}||jj d \t j r dyj d \t j r dyj d \t j r dyt drWzk\rOfjvr?jjftdjfd zkS#|$rYy $r}d t|vrYd}~y d}~wwxYw) Nr)NoTritonConfigsErrorrz%register spilling of the first kernelFz®ister spilling of the second kernelz%register spilling of the fused kernel slow_fusionc $zz dS)N) kernel1_pathkernel1_latency kernel2_pathkernel2_latencyfused_kernel_pathfused_kernel_latencyslow_down_ratior)rrrpath1path2 path_fusedsrsr8zKScheduler.speedup_by_fusion..benchmark_when_ready..Qs(053605365?8@3;sSy3I%ruLoop-carried variableT) )torch._inductor.runtime.triton_heuristicsrrr}risinfr$rrr#rr,)rrr&rrrrrrrrfuture_and_mod_l1future_and_mod_l1_fusedfuture_and_mod_l2rrrrSs @@@@@@rsrz9Scheduler.speedup_by_fusion..benchmark_when_readysA*!,)!,/2 ) ?JJL )"&!@!@)!,"JC zz#CD$!%!@!@)!,"JC zz#DE$+/+J+J/2,(Hj zz(+CD$xc20 >$c 1"EN$2I2II//33UENC(7?? $cCi//+! '.#a&8#s<E AE +5E !5E A3E E.E. E)(E))E.r{)rrqrrqrrqrr)rzir.ChoiceCallerrr})rz5torch._inductor.select_algorithm.TritonTemplateCallerrr}r)Drrrr0rrr-rrkrybenchmark_fusionrrrrrrhrrrrtriton.compiler.errorsrrrrrrrrr`rarrrrrryrqrrSrrrrrr}rr"benchmark_epilogue_fusionrrr max_epilogue_benchmarked_choicesroperator itemgetterr7rrrrrrrrgrrr|)1rrrris_multi_templateatomic_add_template_epilogue node_list_1 node_list_2has_atomic_addryrr=rrrrorr&rrnum_triton_callerschoice_timings_iterrZrr triton_choicertriton_choices unfused_timerrrrrrrrrrrrrrrrrrrrrSs1``` @@@@@@@@@@@@@@@@@@@@rsspeedup_by_fusionzScheduler.speedup_by_fusionse  U^   (2  # # %r'>'>( (J.u5I % (0V0V$$U+ +&&/@$$T* *    u668":Q:QR!! $$T* *oo' Q**, v ;;% F$6$6($B$$T* *oo' y{KHI--o> "3$$T* *;u% #..0!!! " % 8=u~% " $557tCO#''),,.  j"*@*@A AA55jA#((// -"TV%+%>%>,VM%/%>%>}%MN%+N,@,@,B%W )"!OO<<QQ %'==fE *11$*!"%)%8%8(76<6J6J&9&&!"   "$)   M   0&',,U33%Gv'@'@%G$%GVM%/%>%>}%MNGKO%+&,,.H4G4G4J& "  6f=$(. !OO<<QQ") 8 F.;O! "'.+0077FU4]C#V&,,998888>$((..'..{;33K@U'',,U33224"a'#((//j !j !j !X --$nQ&7&:  !% 3 3K @  $ 3 3K @ &*&9&9/&J #F F P --09PQR9S. e  $ )%)66w}}E * 0 0$C6EJ:$%!" %%99F'K^%!!!! ! !!s`!1^55_$`: `$`=$`)5^?  ` >```! )`:.`=9`::`==a c<|j|jS)z0Look up the node in Scheduler name_to_fused_node)rrrs rsrzScheduler.get_fused_nodejs&&t':':'<==rucLtjd|j|j|j}|j|k(sJ|j |j ||}|j ||j ||j||jj|jDcic]}|j|c}|jj|}|||j|<|Scc}w)Nzfusing %s with %s) rrrrr#ryr0rrrrrvr)rrrrr\rnode3rstream1s rsfuse_two_nodeszScheduler.fuse_two_nodesns ,enn.>@PQ!!#!V+++  (--eU;5!5! &&U__EV'W e(;'WX%%))%0  )0D   & (XsD!c|j||r-|j||s|r|j|||yyNTF)rwill_fusion_create_cycler)rrrr speedup_fnr\s rsfuse_if_speedupzScheduler.fuse_if_speedups? MM% '11%?   uk :ruc|rg}i}t}|D]}||vrt||dk\sJ||jd}t||dk(r|j||j \}} | |k(rt || sJ|} n||k(sJt || sJ| } |j| | ur|jr3|jj} | J|j| ||f|| <|j|| |j|s|j|t|D]l} || \}} |j|j|j|j|j|j|s\|j| n|D]} |j| |ryy)z Evaluate pending template fusions for a set of fusion candidate nodes. The fusion candidate nodes are pointwise nodes as potential epilogue or prologue fusions rrN)rrrrrr&r)rrorrrmrrr) rrtemplate_fusion_candidatesr\template_futuresfuture_to_pending_fusionfusions_to_removerpending_fusionrrrfcands rs"_evaluate_pending_template_fusionsz,Scheduler._evaluate_pending_template_fusionss )-/  %@J| 7% 9 !;;6yABaGH">@ uI%-eU;;;$)M I----eU;;;$)M &&}5]J!((&--44A=(=$++A.3A92M,Q/++un&@&@+*--i8K% 9P""23 0'?'B$''''(<(<=''(<(<="..  &))$/ 0' 2*..q1 2q)ruc dfd }|D]H\}}|||j|}j|}t||r||fjvrOj|||scj ||rvj ||} | j t| j ||| j} t||rY||fjvsJjj||ft||} | |vrg|| <|| j| n | |<| |<(| js6j||Ky)Ncbj|vsj|vrjj|jj|}|J|j\}}|j}j |dj |dj||usJj||usJ|rj ||rj ||j|vrj|vryyr~)rrrrmrrr) rrr# node_key1 node_key2 is_speedupr\pending_fusionsrrs rsresolve_pending_fusionsz.resolve_pending_fusionss1 ##E*o=&&u-@!0!4!4''.#''(;(;E(BC"&111'5'F'F'H$ 9+77 ##It4##It4**95BBB**95BBB!|t'D'DUE'R##Iy+F+##E*o=&&u-@ru)rmrrror)rr+ryrrrrmrrorr-rrlr) rrpossible_fusion_pairsr,template_fusion_nodesr\rjr-rr fusion_resr#template_pw_nodes ` ` ` rs_try_fusion_pairszScheduler._try_fusion_pairss G$ G$ G G82+ ?LE5 $E5 1''.E''.E#5%0ENd&@&@@}}u.33E5A!33E5A ))5%2$.$:$:##)00 &N*%7 %u~T5O5OOOO2266u~F+B5%+P(+3HHFH12BC-.>?FF~V1?.1?.!--##E5+>W+ ?ruc@t}|jD]}|j\}}|j}||vs t ||r3|j ||j ||usJ|j ||usJ|j||||yr~)rrrrmr+rrr)rrr\r,seen_pair_speedup_fnr#r)r* is_speedup_fns rs_finish_pending_fusionsz!Scheduler._finish_pending_fusions.s @J|.446 SN#1#B#B#D Iy*66M 448J99 $ $] 3&&y1Y> >>&&y1Y> >>  I}k R Sruc t|Dcgc]\}}t||s|c}}}g}|D]<\}}t||r||vr|j||f*|j||f>|Scc}}wr~)rr&r)r)rrpossible_fusionsdeferred_prologue_fusionsn1n2epilogue_template_nodesnew_possible_fusionss rs_handle_template_overlapz"Scheduler._handle_template_overlapFs#-. MFB2DR2LR M#  "& 6FB!"b)b4K.K)00"b:$++RH5  6 $# Ns A1 A1 c|j|t|}tjtj r@tj d|D]&}tj d|j(i}i}g}|j||}tjstjr|j||}|j||||||j|||j|||j!|r'|j||||||j||t#|d}|j%|}|S)a Combine eligible nodes into FusedSchedulerNodes. This relies on two key functions to control the logic: - self.can_fuse(): checks if a fusion is legal - self.score_fusion(): assigns priority to a given fusion zfuse_nodes_once, candidates:z %sc|jSr~rrs rsr8z+Scheduler.fuse_nodes_once..s !++rur)rrrrrrrrhget_possible_fusionsr-rrr>r2r6r&clearrr{) rrrqrjr\rr,r/r9r8s rsrkzScheduler.fuse_nodes_onceZsk !!%( '  " "7== 1   ; <# A  )=)=)?@ A   OQ  " 44     # #v':':#<< ";      !     $$[/B //0E{S##% $  " ")%    3 34I; W{(=>..u5 ruc#Kt|fd}|dkr|r|yg}d}|D],}|}|r||z |kDr|g}|s|}|j|.|r|yyw)uSort `nodes` by baseline index, then yield groups whose span is at most `max_distance`. Start a new window whenever the next node would push the span past the limit. Negative `max_distance` means "no limit" — yield everything as one window. c|Sr~r)rrs rsr8z-Scheduler._distance_windows..s k!nrurrNr)rr)rqr max_distanceorderedwindow window_minridxs ` rs_distance_windowszScheduler._distance_windowss$<= !  *,  Aa.C# *\9  MM!   L sAAc <tjdtj}tj d|t j dkD}t j}t j}|duxs|du}t j}d}d} |rj} | j} n(tjD cic]\} } | |  } } } dfd } ttjD]\}}||kDrntj|}t|dkr3t j#|| |D]}||kDrVt|dksj%|s,|rI| Jt'j(}j+||| || |t'j(|z z }wt|dj,|d|t j. }| |||t1d _j3j_tj5d |tj|rtj5d |j7jycc} } w)a"Group parallel nodes into combo kernels. Each parallel group is split into windows whose baseline-index span is at most `combo_kernel_max_distance` (set the config to a negative value to disable splitting). If a peak-memory threshold is set, each window goes through the gate, which simulates the post-fusion peak and accepts the combo only if the peak stays under the threshold; rejected windows are halved and retried. Without a threshold, every window becomes a combo directly. rz2ComboKernels: Generating with num_ck_nodes = %s...Ncdz tjdt|||D]}j|j |j j |jDcic]}|j|c}jj|d}||j|<yycc}w)Nrz0ComboKernels: Combining %d nodes for %d-th groupr) rTrjrr0rrrrrrvr) combo_nodeacceptednumrrstreamrnr\rrs rs_register_acceptz=Scheduler.create_combo_kernel_nodes.._register_accepts QJE HHBH   ! )""4( ) OOJ '  # # * *3=3G3G3IJaz)J ((,,Xa[9F!28##J/"Ks7Cr)r[ on_acceptTrXr[rdc|jSr~rrs rsr8z5Scheduler.create_combo_kernel_nodes..s q{{rurzDGenerated ComboKernel nodes: %d ComboKernels, totally %d -> %d nodesz9ComboKernels memory-aware: %.3fs spent in peak simulation)rNrBrOrrPrrr)rrqrrTrr-r$combo_kernel_peak_memory_increase_gb&combo_kernel_peak_memory_pct_thresholdcombo_kernel_max_distance_init_peak_memory_contextrrrBrrqrrJspeedup_by_combo_kerneltime perf_counter_try_combo_with_halvingr!combo_kernel_per_subkernel_blocksrr{rjr)rrrXnum_nodes_origr[ abs_thr_gbpct_thr memory_checkrEmemory_sim_timemem_ctxrrrrRrPrmembersrG sim_startrNrnr\s` @@rsrz#Scheduler.create_combo_kernel_nodess!, TZZ F U 77!;@@ ??!-D1D 77 37 446G!--K,5djj,ABDAq1a4BKB 92 9- 9 9  9,( & D DT J # >NC'EL,@0AA)LG7|a#55l > + 0Dv;?$*F*Fv*N"... $ 1 1 3I00(7"2 1$t'8'8':Y'FFO!;q ++26(7-3-U-U "J%Z=5 ># >JK-BC 33DJJ?   R    O   HHK  !!$**-UCs Jc ddlm}m}m}m}m}m}ttjjj}ttjj}||j|} ||j|j||j|j|j| ||j| |\} } } || t!|j\} } || t!|j} t#|t%|jDcic]\}}|| c}}| | | Scc}}w)zBuild the immutable baseline state the gate compares against: original buffer lifetimes, original peak, and step indices. r)rZ/assign_memory_planning_info_for_scheduler_nodesrr+live_memory_before_steps_from_buf_info_listpeak_memory_from_buf_info_list)rrrrr)rrZrhrrrirjrrarrXr!rrqrrrrr)rrrZrhrrrirjrXrname_to_freeabler5r=rrrIrs rsrYz#Scheduler._init_peak_memory_context#s   "!''"6"6";";"=> "177#;#;#=> 1$**lK9$**dFVFVW7 JJ  # #      6 JJ(-  q!: 3tzz?  q K 3tzz? ('4=djj4IJysDsJ'&!5   Js3 E c: ddlm}t|dj|d|tj }t }|D]'}|j|jj)t||_|j t }tj} d} |D]'} |j|  | } | | kr| } | | kDs&| } )gd} dfd }t| | dzD].}|j |} | |vr| s ||| |d} %|| ||0t#d Dcgc]}|j$}}|j'|}t)|D cic] \}} | | |z c} }|}|D]} || < d fd }|j*| }||| | ||j,| }|j.}t1|j2|}||z }tj4}tj6}|t9|dzgng}||j;||z|sJ|t=|k}|dkDrd|z|z nd}|s"t>jAdtC|||yt>jEdtC|||||_||fScc}wcc} }w)a4The gate: does fusing `group_nodes` into one combo keep peak memory under the threshold? Returns `(combo_node, combo_step)` if accepted, or `(None, 0)` if rejected. The running peak lives on `mem_ctx.running_peak`. The pretend rewrite only changes node order inside the window `[region_start, region_end]` (the smallest range containing all members). Inside that window: - the members collapse into one combo node, all at `combo_step`, - everything else is evaluated from the original schedule. Outside the window, nothing moves. Earlier accepts are accounted for only by `mem_ctx.running_peak`. The threshold is checked against `mem_ctx.baseline_peak` (the original graph peak) so total peak drift is capped. r)estimate_region_peak_memoryrTrT) pred_buffersrFc>jt|||yr~)rr)rrr local_entriess rs add_localz9Scheduler._try_combo_with_memory_check..add_locals  S(D!A Bruc2|j|jfSr~)rr)r&s rsr8z8Scheduler._try_combo_with_memory_check..s @Srurc |vr|S|Sr~r)rnew_steprs rsstep_ofz7Scheduler._try_combo_with_memory_check..step_ofs x~%t$ $ru) region_start region_endrur cur_memoryi@gY@rLzLComboKernels memory-aware: rejected %d nodes (peak delta %+d bytes = %.3f%%)rzLComboKernels memory-aware: accepted %d nodes (peak delta %+d bytes = %.3f%%))rrfrrrrrrr.)#rrmrBrr-r^rrr'rnrGrsysmaxsizerrLrqrrr{rrrrrYrrVrWrqrrZrTrrrj)!rr group_nodesrdr[rmrNcombo_pred_buffersm group_setrvrwrrIinserted_comborqrr& local_nodes combo_steprurx region_peak original_peaknew_peakdeltar`ralimitsacceptpctrprtrs! @@@rs_try_combo_with_memory_checkz&Scheduler._try_combo_with_memory_checkMs, 8/ N $ $ &*+!'!I!I  (\ ?A  % %ajj&=&= > ?7+ )) 4>< {{   !A MM! a.C\!" Z  !,.  C|Z!^4 A 1 AI~%j,:%)N aA  #=6ST AFF  44[A 4=k4JKDAqA|a''Kj)  %A$HQK % %11,? 1 %!!//!   -- w++[9=(@@ ??4>4J% #w/0PR   MM'M1 2 v#f+%1>1Buu}}, II2K     .       (:%%y Ls *JJc|j}|g}|r%|j}t|dks|j|s3tj ||||\} } | || ||[|D cgc]} ||  } } t | t| }} | |k(r| |zdz}|D cgc] } || |ks | }} |D cgc] } || |kDs | }} dt|cxkrt|krnn|j|dt|cxkrt|krnn|j||r$yycc} wcc} wcc} w)zkTry the full candidate; on reject, halve at the baseline-index midpoint and try each half. rN) rrrrZrrrZrYr)rrrrPrdr[rSn2istacksubsetrNrridxslohimidearlylates rsr]z!Scheduler._try_combo_with_halvingsA!!09{YY[F6{Qd&B&B6&J%.%K%Kfg& "J %*fc2$*+qCF+D+YD BRx7q.C &81#a&C-Q8E8%6!Q#A6D6CI+F + T"CJ,V, U#-, 96s/ D9% D>3D>= E EcH|D]}|j|jyr~)rr)rrrqrs rsrzScheduler.prune_redundant_depss% ?D  % %d&=&= > ?ruc g ttttf d fd }tjt }|D]=}j |r|jD]}||j|?|jD] }|| tjratjt }|D]&}t|dd} | s|| j|(|jD] }|| j  jjdt j#dt%  S)z^ Helper to find all legal fusion opportunities, sorted by self.score_fusion() c|t|D]\}}||dz|dztjzD]}||f}|vr j|j ||rj |B|j s|jscj ||swj ||fyrH)rr-)max_fusion_buffer_group_pairwise_attemptsrrrrr) rq node1_indexrrrrjr8rDrrs rscheck_all_pairsz7Scheduler.get_possible_fusions..check_all_pairs s&/&6 @" U"!Ok'FF'G@E !%.Cd{ HHSM}}UE3CD(//4++-1A1A1Cu&6J)//?!@ @rurNT)rreversezfound %d possible fusionsrqrrr)rrrfrWrrunfusable_noderrrr-aggressive_fusionr*get_possible_fusions_with_highest_priorityrscore_fusion_keyrrr) rrrqrjrbuffer_names_groupingrr node_groupinggroup_groupingrr8rDs ` ` @@rsrAzScheduler.get_possible_fusionssn% 13D DEFH @ @(!, 7 7 = 8D""4(--/ 8%c*11$7 8 8 399; +M M * +  # #(44T:N 7gt4"5)006 7"0!6!6!8 /  . / JJ   $"7"7F4c:J6KLructtdfd |jjj |jjj z|j jj |j jj zz t fdD}|rt||d|S)z~ Finds whether there's a path from node1 to node2 (or vice-versa) caused indirectly by other fusions. ct|trq|vrmj||jj ryt |j zxs"tfd|j z DSy)NFc3HK|]}j|ywr~rQrr found_pathrrs rsrzIScheduler.will_fusion_create_cycle..found_path..Rs+H#4#:#:1#=>H")rrrrissubsetr}rr)rcombined_ancestorscombined_namesrrrvisiteds rsrz6Scheduler.will_fusion_create_cycle..found_pathAs$ 23G8K D!++-667IJ!  ?@CH!%2D!DHEruc3HK|]}j|ywr~rQrs rsrz5Scheduler.will_fusion_create_cycle..`s!WqJt66q9:Wrzwill create cycler*)rrr_dictr!rrr)rrrrcyclerrrrs` @@@@rsrz"Scheduler.will_fusion_create_cycle7s/02  2  % % ' - - 2 2 4'')//446 7  OO ! ! & & (5??+@+@+E+E+G G WDVWW  #IeU #$7 8 ruc ddlm  d fd }||}||}t fd|D}t fd|D}|j|}d} |D]} | t | dz } j ||} tjjj| d | zry y#t $rYywxYw) a Return true if fusing the two nodes can potentially increasing peak memory. The implementation is more like a heuristic since we don't really know if we are at peak or not when trying to fuse these two nodes. The order of nodes may change later which makes the peak memory estimation hard. Here is how we decide the LOWER BOUND of extra memory allocation if we fuse these 2 nodes: 1. find all buffers read by each node with a single user. These buffers are supposed to be reused if we don't fuses these 2 nodes 2. find the intersection of these buffers for the two node and sum the total buffer size. If we don't fuse these two nodes, we can at lease avoid this much memory allocation. Note that the extra memory allocation is not necessarily causing peak memory increase. This is just a heuristic. We return true only if the saving for fusion can not trade off the extra memory allocation. r)buffer_reuse_keyc0g}|jjD]y}jj|j}|s+t |j dk(sD|jjs_|j|j{|SrH) rrrrrrrrhas_tensor_outputr)rr=rLrrrs rs_find_single_user_inputszKScheduler.can_fusion_increase_peak_memory.._find_single_user_inputs|swF&&,, ,&&**27733syy>Q.3883M3M3OMM#((+ ,Mruc3.K|] }|ywr~rrrrs rsrz.#Sc$4S$9#Sr=c3.K|] }|ywr~rrs rsrz.rr=rrF T)rrfrzlist[ir.Buffer]) rrr intersectionrrsrrarrstatically_known_gt) rrrrr lhs_dep_nodes rhs_dep_nodeslhs_reuse_keysrhs_reuse_keyscommon_reuse_keysmemory_overheadr bw_savingrs ` @rscan_fusion_increase_peak_memoryz)Scheduler.can_fusion_increase_peak_memoryes* 6 #   17 07 ##S]#SS##S]#SS*77G$ C 3s1v;. ,,UE:  77   / /i P  s$B88 CCct|jDcgc]}|jc}|jDcgc]}|jc}z}td|jjD}td|jj D}||z}t} |jjD]:} |j | j|s | j| j<td|jj Dtd|jj Dz} td|jjDtd|jjDz} | |z } | | z }| |z}t||kDScc}wcc}w)Nc34K|]}|jywr~rtrus rsrzEScheduler.fusion_prevent_too_many_reads_and_writes..s&TCsxx&Trvc34K|]}|jywr~rtrus rsrzEScheduler.fusion_prevent_too_many_reads_and_writes..%R3chh%Rrvc34K|]}|jywr~rtrus rsrzEScheduler.fusion_prevent_too_many_reads_and_writes..s$ CHH$ rvc34K|]}|jywr~rtrus rsrzEScheduler.fusion_prevent_too_many_reads_and_writes..r>rvc34K|]}|jywr~rtrus rsrzEScheduler.fusion_prevent_too_many_reads_and_writes..s% CHH% rvc34K|]}|jywr~rtrus rsrzEScheduler.fusion_prevent_too_many_reads_and_writes..sDCsxxDrv) rrrrrQr$can_buffer_be_removed_through_fusionrrr)rrrr thresholdrfused_node_namesnode1_write_namesnode2_read_namesreads_removed_through_fusionwrites_removed_through_fusionrall_read_namesall_write_names unique_reads unique_writesunique_io_bufferss rs(fusion_prevent_too_many_reads_and_writesz2Scheduler.fusion_prevent_too_many_reads_and_writess&).): ;T]]_ ;+0??+<=4t}}= > '&T5;L;L;S;S&TT%%R%:K:K:Q:Q%RR'7:K'K$:D%**11 BI88 0.11)..A  B$$ % 1 1 7 7$  C5+<+<+B+BC CD %% % 1 1 8 8%  D5+<+<+C+CD DE &(DD (*GG )=8$% 11M <=s GG ctt|j|jz t|j|jz }|dkDS)aA This function prevents fusion for nodes that can increase memory footprint. This problem is more common in horizontal fusion, where nodes that are far apart in the original order get fused, lengthening the live intervals of tensors. This is very evident in models with activation checkpointing, where the recomputed nodes from different checkpointed regions get fused and significantly increase the memory footprint. The current attempt is a quick, possibly hacky, heuristic to prevent the fusion of nodes that are far away in the original order. A better but difficult to implement heuristic would be to use live intervals of the buffers, find region of peak pressure in the original program and prevent fusion that crosses that peak region. We might need special care or good approximation in this implementation, as fusion of node changes live intervals, and re-computing live intervals and peak memory after each fusion can introduce large compilation overhead. @)rYrr%r&)rrrrproximity_scores rsare_long_distant_nodesz Scheduler.are_long_distant_nodessE* %//1 2 %//1 2 ##ruci}|jjDcic]}|j|}}|jjDcic]}|j|}}|D]}}tjj |} ||} ||} t | trt | tsdt| dt| ||<k| j| jk7r(d| jd| j||<t| jt| jk7rd||<| j} | j} | | k7r d| d| ||<| j| jk(r d| d| ||<Ed}t | tjsd| j }d | d| d |||<t#|Scc}wcc}w) z} Try to decide reasons why fusion fail due to no shared memory even though there are common buffers. znot MemoryDep: rzdifferent numel: broadcastzdifferent offset: zMismatch loop orders: rXzLayout: zUnknown reason: z. )rrwrrarrIrr9rrr_r get_offsetnormalize_with_stride_orderr0rLrr,)rrrrcommon_buf_namesreasonsrnode1_name2depnode2_name2deprrlhs_deprhs_deplhs_offrhs_off layout_strs rsdecide_fusion_fail_reasonz#Scheduler.decide_fusion_fail_reasons383D3D3U3U3WXC#((C-XX383D3D3U3U3WXC#((C-XX(, H''$$X.C$X.G$X.Ggy1GY9W%d7m_F4=/J!  "g&7&7&99'(9(9(;'j@jC|<tE|}|||d }|y|j&j(||j&j(|<||j&j(|<|jGd d |jI||}t|tJsJtLjOd ||Scc} w)aW Attempts to enable fusion between two nodes by inverting indexing patterns. This optimization targets cases where node1 has a contiguous write and node2 has a contiguous write but discontiguous read. By inverting the indexing in node2's read and write operations, we can make them compatible with node1 for potential fusion. Args: node1: First scheduler node (source) node2: Second scheduler node (target for inversion) Returns: int: Fusion score if successful, 0 if optimization not applicable rc3<K|]}|jywr~rrs rsrzAScheduler.shared_data_after_inverting_indexing..=2aqxxz2rc34K|]}|jywr~rtrus rsrzAScheduler.shared_data_after_inverting_indexing..Is. CHH. rvrrindex0index1c3 K|]}|ywr~r)rras rsrzAScheduler.shared_data_after_inverting_indexing..s%Ttd%Ts r)generate_inverse_formulaTFz!Shared memory after inversion: %d)(r-rlrr buffer_namesrr3rrrQrrrr9rr rr var_namesrr  subblocksget_read_exprs$torch._inductor.invert_expr_analysisrvarsrAdd make_argsrrarrcombine_modular_indexing_pairsrr3rrrrj)rrrrnode1_buffer_namesnode2_buffer_namescommon_buffer_namesnode2_unmet_dependencies node2_read node2_writer node1_writes node1_writenode2_read_exprs read_exprread_expr_indexwrite_expr_indexrrtsimplified_termstermsimplified_read_exprinverse_formulascores rs$shared_data_after_inverting_indexingz.Scheduler.shared_data_after_inverting_indexing's&44 2E5>2 2#..;;="..;;=03EE"$.. % 8 8. $   $&8 8 ' (1 , u  && '! +s53D3D3K3K/Lq/P$u006678 4 1 1 8 89: *i0 9 161B1B1I1IJ## J J ??, .":??3 +y1 "++-    !2!2 2  K$4$44 ??k.. .#j6J6J2Kq2P u{{)) *a / ;;   22 2EKK666  7 &%Tu{{7Q7Q7S%TT  A %./0   228< <&O'  : :8 DD DD&O' Q[[%%a( z?a II'' 2 D  # #  ??E   ##3423GTUW  " 7Scheduler.shared_data_after_reordering_loop..rrrr) rrrrr-r%!_try_reorder_loops_for_candidatesloop_reindexing_after_fusion$_try_reindex_pointwise_for_reductionr)rrrrrr s rs!shared_data_after_reordering_loopz+Scheduler.shared_data_after_reordering_loops 2E5>2 2    %"3"3"5    * * ,u/@/@/M/M/O O #  , ,::5%HEz 33<..%sQ2B66r2>QrNr)rr r_sizesr s @rsrz>Scheduler._try_reorder_loops_for_candidates..$s$QQQs#rrrFz?Don't reorder loops since both nodes are reductions: %s v.s. %s)rrrrrQrrrrarrrrrr9r rrrrYrrr^ dep_size_hintrrprarrr)rrrrrr node1_readsr node2_reads node2_writes candidatesrrr is_write_readwrRw_sizes_is_wr_numel reorderedr r s @@rsr z+Scheduler._try_reorder_loops_for_candidatess{    * * ,u/@/@/M/M/O O 160A0A0G0GHsxx}H H161B1B1I1IJ## J J050A0A0G0GHsxx}H H161B1B1I1IJ## J J .' "K"&&{3O{;7OG"&&{3O{;7OG|+J {0JN-L+2L  3356689!!%((::#--/!;   !$$[1R\5E5Ek5ROOK0PKOOK4Pa+ 1i0H))Bkkm00Gkkm00G") "O' "R z?a ,/ H//15, ('9-Z5S   w// /   "g&7&7&99))'22 !!#77II##%77II  # #Q    :Ct''u5JJcIJHJsN6>N;,OOc ddlm |js|jry|jr|js||}}n&|jr|js||}}ny|j\}}t j tj|dt j tj|dztd|jDsyt j tt|j}tfd|Dsyt fd|Dsyftfd|Drytj|f}|D]} | jgt!|t"r|dj|_t%||j&j)|j&j)z} |j&j+D cic]} | j,| c} |j&j+D cic]} | j,| c} t/fd | D} | s|j1yt2j4s5|D]} | j7d d t!|t"r t%|y cc} wcc} w) z Reindex a pointwise's iteration loops to match a reduction's groups. After reindexing, the shared reads have identical index expressions, enabling the codegen to CSE loads. Returns True if reindexing was applied. rrFrc3<K|]}t|tywr~rr)rr^s rsrzAScheduler._try_reindex_pointwise_for_reduction..msOR:b-0Orc3K|]D}tjjjt |j dFywr)rarrrr_r)rr^ target_numels rsrzAScheduler._try_reindex_pointwise_for_reduction..ssB  GG   4 4biil+\  sA A c3bK|]&}jf|j(ywr~)rr\)rr^r red_numel red_rnumels rsrzAScheduler._try_reindex_pointwise_for_reduction..{s1   $ $i%.s$IBuRYYq\"&77Is%(c3NK|]}j||ywr~)deps_match_normalized)rrn1_depsn2_depsrrs rsrzAScheduler._try_reindex_pointwise_for_reduction..s.   & &wt}gdm D s"%TrA)rrrrrrrrrarrrrr/rrIrrrrrrwrrr<r-r%r3)rrrrreduction_nodepw_noder=groupsrrollback_snapshotr^ common_namesr has_benefitrr3 r4 r- r. r0 r+ s` @@@@@@@rsr z.Scheduler._try_reindex_pointwise_for_reductionNs - <<>U\\^    (:(:(<&+UGN    !%*<*<*>&+UGN"(( 6KK F1I6 [[VAY7  :- O7;L;L;NOOT-0'2C2C2EF       ' 3 I&I I /55wjA >B  $ $i%< = > g1 2"1IOOGM +G 4    * * ,u/@/@/M/M/O O -2,=,=,N,N,PQS388S=Q,1,=,=,N,N,PQS388S=Q $    % % '00 W''$PU'V W'#56/8+RQs "KK cft|tr)|j xrt|j St|t rht|jt jr|jj S|j xrt|j Sy)z> Is this node unfusable under any conditions. F) rrrr]rrRr0rrrs rsrzScheduler.unfusable_nodes d2 3''))2U 3/  d5 6$))R%?%?@9966888''))2U 3/ ruc|jtjjkry|j }|j }d}|||zkDr |dyt d|jD}|tjjjjfk(r |dyd d}|j} | js+|| jr|j!s |dyy) zT Heuristics to avoid benchmarking predictably slow prologue fusions Tg?z@prologue fusion will not increase amount of bytes read in kernelFc3K|]J}|j<|jjD]}|jdk(r|j!Lyw)N call_function)rrrr)rrr&s rsrzEScheduler.check_prologue_fusion_heuristics_fusable..sT vv!VV'')  tt& HH  sAAz\prologue fusion will not increase attempt to fuse in padding bc it increases unaligned readsc<|jdkxr |jS)Nr)itemsizeis_floating_point)r}s rs low_prec_fpzGScheduler.check_prologue_fusion_heuristics_fusable..low_prec_fps>>Q&B5+B+B BruzVprologue fusion that must be upcast to fp32 not profitable for low precision templates)r}z torch.dtyperr})rrarinvoke_quant_opsr1r3rrr`r&atenconstant_pad_ndr)rr\r}r) rr prologue_noderrS read_bytes write_bytesBYTES_THRESHOLD_MULTIPLIERr$rB rs rs(check_prologue_fusion_heuristics_fusablez2Scheduler.check_prologue_fusion_heuristics_fusables  , , .!''2J2J J"88: #::< &)" 'AA B R S ",,.   uyy~~55==? ? n  C%??A 668L../!>>@ h ruc<t|trt|tsyt|jtjr$t|jtjsy|j s|j ryt jdk(ry|j|j}}|\}}|\}}|js,|js||k7st|t|k7ryt|jjdkDs"t|jjdkDryjtt|jj} jtt|jj} t!| | t j"kDrydfd } | |s| |ryg} t%t'||D]\} \}}||k7s| j)| !t| dk7ry| d}||||}}t*j,j.j1||r|||fSt*j,j.j1||r|||fSy)ao Fusing two small pointwise nodes significantly reduces kernel overhead and launch overhead. However, slightly different sizes would prevent fusion. Here, we decide if expanding sizes of one node is profitible by allowing fusion, and returns the dimension to expand, node with smaller sizes, and new size after expand. Nrgrc~|jjD]}|jjvrj|j}n%jj |j}|s]t jjj||st|jtryyr) rrrrrrrarrrrrr)rrrrrs rshas_reusable_bufferzIScheduler.get_expand_dim_for_pointwise_nodes..has_reusable_buffer5s((.. 99 ; ;; $ ; ;DII FI $ 0 0 4 4TYY ?I,,66y$G&y'<'<>TU rurr*)rrrr0rrr-rhrrrrrQr rrrYsmall_memory_access_thresholdrryrrarrstatically_known_lt)rrrrn1_sizesn2_sizes n1_iter_sizesn1_reduce_sizes n2_iter_sizesn2_reduce_sizesnode1_write_memorynode2_write_memoryrM mismatch_dimensionsrIn1_sizen2_size mismatch_dimmismatch_size1mismatch_size2s` rs"get_expand_dim_for_pointwise_nodesz,Scheduler.get_expand_dim_for_pointwise_nodess]%/z%7W uzz2#4#4 55::r'8'89  ) ) +u/M/M/O    )#\\5<<()1& )1&     !!#/1=!S%77 u  '' (1 ,E4E4E4L4L0MPQ0Q"//T%:K:K:R:R5S0TU!//T%:K:K:R:R5S0TU "$6 722 3  u %).`s,(   ! ! % %dD 1( rc3K|]:}jj|j|jxvr<ywr~r)rr)rrrproducer_buf_namesrrs rsrzDScheduler._producer_output_names_read_by_consumer..dsD --11#((CHHEE!"  sAA)rrrr)rrrErJrre s` @@rs'_producer_output_names_read_by_consumerz1Scheduler._producer_output_names_read_by_consumer]sN((  113(   ++11   ructj||sytj||sy|j||Sr~)r2r@rrf rs rs"_nested_index_equivalent_dep_namesz,Scheduler._nested_index_equivalent_dep_namesks? ++E59''u5;;E5IIrucH t}tfd|jD |D][\}}|tjjur#fd|j j D}|j fd|D]j||||S)Nc3VK|] }jj||"ywr~ra rb s rsrz>Scheduler._can_fuse_nested_reduction_append..s,'   ! ! % %dD 1' rc3~K|]4}jj|j|j6ywr~rd rs rsrz>Scheduler._can_fuse_nested_reduction_append..s4%%))#((CHH=s:=c3,K|] }|vs| ywr~r)rdep_namegrouped_buf_namess rsrz>Scheduler._can_fuse_nested_reduction_append..s.%(>O2O.s )r%index_equivalent_dep_names) rrr2rHrGrrr _can_fuse) rrr7rrir%ro r^ryrrn s ` @rsr&z+Scheduler._can_fuse_nested_reduction_append|s7Al"&' $557'  , JB_<<HHH>>//E ' - -.)..  ~~  #'A   ruFctj||f}|j||||}|j| |S)zDetermine if node1 and node2 can be combined into a single fused node. Speculative loop mutations (reordering, reindexing) are automatically rolled back if the fusion decision ultimately fails. r%r)rJ)r@r_can_fuse_implrK)rrrrr%rrErs rsrzScheduler.can_fusesP'--uen=&&  #&? '  H -ruc,|j||||S)zj Determine if it is possible to combine node1 and node2 into a single fused node. rr )rp )rrrrr%rs rsrs zScheduler._can_fuse_impls&~~  #&?   ruc!"#ury|tfd|D}jr@jj}jj}||||k7ryt t rj |St t ryt trj St tryt}jr0jjjryt tst tr |dyt trjs |dyt trt j t"j$s |dyj j's |dyt t(s |d yt j t*s |d yt j j,t.s |d yt1j j2d k(sJj j2d j4#t7#fdj8j:Dr |dyj j,j=} | D];} j j,j?| } t7d| Ds;yt1j j@d k(sJj j@d jB} j jB} t | t"jDrt | t"jDsJ| jF| jFk7s2| jH| jHk7s| jJ| jJk7r |dy d'#fd !t7!fdjLDryt ttfrjs |dyjOjPzr |dyjrtSs |dyjUsjr |dyjW}|jY}|s |dytd|jZD|z }j]|zr |dyj_s|jar |dyjc""ddD]B}|je}|D]+}tg"fd|jhDr"|dyDt tjsgn*jlDcgc]}|js|c}}t1|d k(sJ|d }t1"djnd k(rSt1"djnd jhd k(r+"djnd jhd j |us |dyjq|syjrts}j_r|rjUs tus |d yjw}|J|jyr-t j t"j*s |d!yj]tzj|j~zs+j]tzj|j~zr |d"yj}j}||k7r |d#||y~|j}j||$}|rL|tjkr9tjstjrj}|d k\r|}tjr@jx}r,|\}}} |j|| j|%}tjr,|tjkrj}|d k\r|}tjtjr4tjd&jj|tzjj|syjOjPzr։j|%rDtzjj|r"j|jrytjrmjr[j|%xrEtzjj|xr!j|jSytzjj|xr!j|jScc}w)(NFc3VK|] }jj||"ywr~ra rb s rsrz&Scheduler._can_fuse..s,4%%))$54rr$Tz/grouped node must not be fused with other nodesz node1 is nopz'node1 is extern but not a triton kernelz5node1's triton kernel doesn't support epilogue fusionz.node1 is extern but node2 is not SchedulerNodez3node1 is extern but node2.node is not SchedulerNodez4node1 is extern but node2.node.data is not Pointwiserrc3<K|]}|jk7ywr~rt)rrwritten_buffer_names rsrz&Scheduler._can_fuse..sVs38822VrRz9epilogue reads from buffers other than the mutated outputc3&K|] }|dk7 yw)rNr)rusages rsrz&Scheduler._can_fuse..s;5u;sz*node1 and node2 uses different buf layoutsc@|uxr|uxr|jvSr~)r)r rrrx s rs._is_other_node_that_references_mutation_bufferzKScheduler._can_fuse.._is_other_node_that_references_mutation_buffer-s7 u,N#50N+z/K/K/MMruc3.K|] }|ywr~r)rrr| s rsrz&Scheduler._can_fuse..6s?tDr=znode2 is extern or nopznode1 must go before node2zprologue fusion turned offz2prologue fusion only supported for pointwise nodesz'template has no allowed prologue inputsc3<K|]}|jywr~r)rrs rsrz&Scheduler._can_fuse..WsEc3<<>Erz;prologue fusion not implemented for kernel for these inputsz:template prologue can only fuse functional pointwise nodesrc3:K|]}|jvywr~r)rrprologue_nodess rsrz&Scheduler._can_fuse..isQttyyN:Qrz7template prologue can only fuse nodes with a single usezEtemplate prologue can only fuse nodes with a single use into templateztemplate epilogue not satisfiedz6multi-output template epilogue requires ComputedBufferz#fusion for buffer explicit disabledzdevice mismatch (%s vs %s))rro ro z%s and %s has %s shared data)r rf)Wrrrvrrrrrrrr#rcan_fuse_multi_outputs_templaterrrRrr0rrrrrrrr/rrrrinner_fn_free_symbolscollect_inner_fn_symbol_usagerrrrrrrqrrr$rrget_allowed_prologue_inpsrrr,has_aliasing_or_mutation_for_prologue_fusionrr<rrrrr,rJ r rrr\rarno_fuse_buffer_namesrh !_score_fusion_memory_for_can_fuser-score_fusion_memory_thresholdr%r r $expand_dimension_for_pointwise_nodesr^ rXrlr rarrrrrrrcan_fuse_verticalr can_fuse_horizontal)$rrrrr%rro rstream2rSnode2_inner_fn_free_symbolssymbolusageslayout1layout2rrunsupported_prologue_argsr node_outsrrtemplate_snodestemplate_snodeatomic_add_mutation_epiloguerrdevice2shared_data_scorenew_shared_data_scoreexpand_analysis expand_dim smaller_node expand_sizer| r rx s$``` @@@rsrp zScheduler._can_fuses  E> % 1)3464* &  ' ' )))--e4G))--e4G"w':w'?Q e2 3&&u+&F F e2 3 e4 5&&u- - e4 5u%    4#3#3    $ ) )% 7$8 e1 2j '7  A B e3 4U=N=N=P   e6 7ejj"*D*DE=>:://1KLe]3DEejj.9IJejjooy9JKuzz223q8 88"'**"="=a"@"E"E Ve>O>O>U>UVVOP+0**//*O*O*Q '5 !FFvN;F;;  ! uzz../14 44jj--a077Gjj''Ggryy1j"))6T TT  ,>>W^^3>>W^^3@A -  JJ u8:PQ R%%' ( )  $ $ & 8 , -    .u501!!#u'8'8':HI779H$,$F$F$H !(=>EX__EE'( & %%'*CCQR--/EEeLPQ"__.N&s+ % ,,. $%CQsyyQQUV$% %"%);<!&AAaA  '1, ,,,Q/NN2&../14r*2215;;<A"2&..q177:??>Q[@@sS    +Qu, (//1:V%%'25956 224L+ ++557  B--ALM  " " $qww'C'C C  " " $qww'C'C C 5 6!!#""$ W  ,fg >  % -)-)P)Pu* &!BB  &?'A C  !F$H$HH11V5X5X%)$J$J5RW$X !$)$9!  6 6#FFueT TO T6E 3Z{  < >ueL33==eUK**3M+Q  33eU,= Q((0BB5%P 9900eU$5M""6*>>ueL MUBs i?0i?r c|j}t||}tt}|jD]j}|j j |j|j}t|tr|j|||rW||j|l|jjD]} t| tst| ts%|j j | j| j} |j | } | si| D]} t| trG|j!| j#|j | |duxr| |vr| j%| Zt| tsk|j'| | |j(s| j%|  t+dt,j.j1|j3D} | |zr |dy|j5}| D]E}|j6|j9}||j:|j<zs=|dyy)a Check if it is legal to fuse a consumer (node2) into a producer (node1). We can fuse them if all the reads of node2 either match corresponding writes in node1, or are written by nodes that can be scheduled before the fusion of node1 and node2. ``index_equivalent_dep_names`` relaxes write/read matching only for named producer outputs; the remaining intermediate-dependency checks still run normally. Nallow_index_equivalencec34K|]}|jywr~rtrus rsrz.Scheduler.can_fuse_vertical..1s$  HH$ rvzmemory deps did not matchFz(intermediate nodes between node1 & node2T)rrrrr3r)rrrr;rrrrQr9r:fusable_read_and_writeryr0.fusable_stardep_write_and_read_on_empty_tensorrrrrrzrrrrrr)rrrrro node1_buf_namesrSremaining_deps_by_namerrcd write_name remainingrLremaining_depsnode1_op_namesrs rsr zScheduler.can_fuse_verticals:$ 002u%7B47H++ 5C((,,SXXsxx@D#w'D,A,A#ue,T "4 ( / / 4  5 ##** -Bb),ZG5L..22277BGGDJ.22:>I#-B!"i0T5P5P $"7"786dBI *.H H 6Q6"((,#B0KKEJJ"((,-  -.$$  445K5R5R5TU$   O + + ,224" D&&t,==?G 7 7 @ J JJ>?   ruc|j|jvry|jjDcgc]}|j|jk(r|}}t |dk7ry|dt tryt tsJtjtjrytjjjksy|j |j}|g}t |t"r |j$}d}|D]R} | jj&D cgc]} | j|k(r| } } | s8|dz }t)fd| DrRy|dkScc}wcc} w)NFrrc3K|]q}t|txr[t|jtj  xr4|jjk(xr|j j k(sywr~)rr9r(rr*TMPr)rrrs rsrz-Scheduler.fusable_weak_dep..rsm  4+,+DJJAA,JJ%++-,II+,sA7A:)rrrrQrrrr:r9r(rr*r rrr&rrBrrr) rrweak_deprrrmutating_writesr3relevant_reading_nodesnum_concurrent_reads reading_noderrelevant_readss ` rsrzScheduler.fusable_weak_depGs == 6 6 8 8 **11 zzX222    1 $" eW %%+++ u{{DHH 5 %//*ekk.F.FF++H,A,AB "' e7 8%*\\ " 2 L)44::99 )N " A %  + ! "$q((W 6s "E:;E?c|j|jk(xr\t|jt|jk\xr/|jdt|j|jk(Sr~)rrr)rrs rs_same_index_with_prefix_sizez&Scheduler._same_index_with_prefix_size|s[ JJ%++ % ;DII#ejj/1 ; +C O, : rur ct|tr|j|jk7sHt|jt j s$t|jt j ry|}|}|j|jry|j||rytjr9|j|jk7r |j}|j}|j||ry|sy|j||St|tr?|j|jk(r&|j|j|jk(ryy)aReturn whether a producer write can satisfy a consumer read. The default path accepts exact matches, plus the existing loop-ordering-normalized exact match when that config is enabled. ``allow_index_equivalence`` only runs after those checks fail. It keeps the producer write dense and injective, then accepts conservative consumer-side equivalent reads such as broadcasts or normalized loop-order changes. FT)rr9rr(rr*r rCrr r-r%r^r %_fusable_read_after_index_equivalencer:)rrrrr original_readoriginal_writes rsr z Scheduler.fusable_read_and_writes dI & UZZ'&tzz488<&u{{DHH= M"N 11.2E2EF00O00T]]enn5T~~') 00u=*==~ g & UZZ'JJ*II+ruc"t|j|jjksy|j j s|j j sy|j||xs|j||Srm) rrrr&r  is_contiguousrr2 _fusable_read_after_broadcast)rrrrs rsr z/Scheduler._fusable_read_after_index_equivalences| %//*ekk.F.FF OO  + + -002@@B)) % =  / /e < =ruc tfdjD}t|jk7r|Dcic]}|j|}}t j jt|t|jjj|jk(ryj|jk7rytjj}tdtjD}i}i}tjj ||j D]\} } } } |j#| | r| || < |j#t%j&| | dsy|j)t+| | } |j-| dsyt%j.dt|dd}| | z|z|| <| ||<|sy|j1t3j|it5t||j |}t3|jt5t|j|}|j#||Scc}w) aQMatch conservative broadcasted read forms. This handles two nested-reduction dependency shapes: - Pure broadcast dims absent from the read index: read: d1, {d0: 1024, d1: 16} write: d0, {d0: 16} - Same-rank expanded dims used through a quotient: read: 32*d0 + FloorDiv(d1, 128), {d0: 128, d1: 4096} write: 32*d0 + d1, {d0: 128, d1: 32} Producer-side broadcast and non-dense writes are rejected before this helper, so these cases only relax consumer-side broadcasts. c3TK|]}|jjvs|!ywr~)rr&)rrrs rsrz:Scheduler._fusable_read_after_broadcast..s' SDJJ4K4K-KC s((TFc3RK|]}tjd|dd!yw)_fusable_broadcast_Tinteger nonnegativeN)rr)rrs rsrz:Scheduler._fusable_read_after_broadcast..s0  LL.qc2Dd S S s%'rr_fusable_broadcast_tail_r )rrrr^rangesr9rrrrr rarrrLryrrrrrr'rrsimplify_with_rangesr`rx)rr read_varsr read_rangesr write_vars replacements tail_rangesread_var read_size write_var write_sizertail_var read_index write_indexs` rsr z'Scheduler._fusable_read_after_broadcastsW* >>   y>T]] *>#333rucBtjj||Sr~)rarget_dep_size_hint)rrrr s rsr zScheduler.dep_size_hintMsww((k::ruc n|j|jzsyt|jj}t }d}|jj D]}t|ts|jj|j|j} t|D]\} } | |vr |j| j|j||duxr| |vsA|j| |t!|j#| ||j#||z }|S)a0Score vertical producer-output deps missed by exact dep scoring. Exact scoring is set-intersection based, but vertical legality can also accept normalized equivalent read/write deps. Give those pairs a memory score so heuristics do not discard them before legality runs. rNr )rrrrrrrQrr9r)rrrr ryrrYr ) rrrErJr ro r matched_readsr rr rrs rs*_score_fusion_memory_by_fusable_read_writez4Scheduler._score_fusion_memory_by_fusable_read_writePs7,,.1C1CCX))//0)3 ))00 EeY/..225::uzzJJ$U+ 4 %..KK 5 562$>E&*DD /"%%a(S**4=**5+>E!   * ruc|j|||}t|tsJ|dk(r|r|j|||}|S)Nrrr )rrrr )rrrrrro r s rsr z+Scheduler._score_fusion_memory_for_can_fusezsd((  &?)  %%%% A:4CC+EDE  rucyr~rrrrrr return_is_mix_order_reductionrs rsrzScheduler.score_fusion_memorysrucyr~rr s rsrzScheduler.score_fusion_memorys!$ruc dfd}|r6tj||r tj||}||ddSt|jt j r|jjs |js|jr|jj|jjz}|jj|jjz} dd} d}|D]@} | D]9} | | | s |tj| j| z };B||ddS|jj|jjz}|jj|jjz} t|t| kDr| |} }|D cgc] } | | vs|  }} tfd|D}|r ||ddSd}|dk(r$j!||rj#||}|||dScc} w)a2 The first term in our fusion score that estimates number of saved memory operations. This function scores fusion candidates based on shared memory access patterns. Higher scores indicate better fusion candidates. Scoring strategy: 1. If nodes share exact memory deps (same buffer + same indexing), return the sum of shared dep sizes (original behavior). 2. If no dependency score remains, check for same-buffer reads with different indexing (e.g., split operations reading different slices). - Give bonus if nodes read from exactly the same set of buffers - Score based on overlap ratio: common_buffer_size / total_read_size - High overlap (>50%) suggests good cache locality benefit from fusion cr|||fS||zSr~r)r buffer_overlap_scoreis_mix_order_reductionr s rs_construct_return_valuez>Scheduler.score_fusion_memory.._construct_return_values#-35KLL// /rurTc||k(ryt|ttfr/t|ttfr|j|jk(Syr)rr:r9rr s rs_matchz-Scheduler.score_fusion_memory.._matchsD4<dWi$89j7I.? 99 11ruFc3BK|]}j|ywr~r )rrr rrs rsrz0Scheduler.score_fusion_memory..sWSD&&sK8Wr)r r8r r8)rrrrrr0rrrrrrQrYr rr_can_use_buffer_overlap_scoring&_score_fusion_memory_by_buffer_overlap)rrrrr r rr r node1_deps node2_depsr node1_dep node2_deprcommon_memory_depsr s` `` rsrzScheduler.score_fusion_memorys2 0 %):)C)CE5)Q &66ueDE*5!T: :5::r'A'ABJJ002  "  "**0053D3D3K3KKJ**0053D3D3K3KKJ E'  !+Ii3 ..y94;M;Mi;X" +5!U; ;&&,,u/@/@/G/GG &&,,u/@/@/G/GG z?S_ ,%/ J-7Mc3*;LcMMWDVWW *5!U; ; ! A:$>>ueL#'#N#Nu$ 'u.BEJJNs  H-H-ct|js|jry|js|jrytjstjrU|j }|j }|r|syt d|D}t d|D}t }|D]}|jD]} t| jts| jjs9t| jsO| jj} | Lt| tjr2| j} || zs|j!| j|j!| j|r|D]}|jD]} t| jts| jjs9| j|vsH| jj} | 3t| tjr| j} || zsyy||fD]d} | j"j$D]I} |j&j)| j*}|+|js.* +TsCLLN+Trc3<K|]}|jywr~rrs rsrz.+ r r)rrr-rrr<rrrrrfr$rr0rr rrrrrrr)rrrr node1_outputs node2_outputsnode1_output_namesnode2_output_names&node1_prologue_eligible_template_usersrrr allowed_inpsrrrEs rsr z)Scheduler._can_use_buffer_overlap_scoring sf4    5#5#5#7    %"3"3"5   &":":!--/M!--/M! !++Tm+T!T !++Tm+T!T    3% RIIRD"499.?@ II1137 B )- (C(C(E (4)2+B+B:,9+R+R+TL1L@ F J J499 UCFFtyyQ%R R.6(-C # -&tyy2CD $ 5 5 7 $ -S S-1II,G,G,IM,8Z -r/F/F>0=/V/V/X #5 #D+0(-!--& %++11%C#66::388DH ,$0027A$% %ruc| d d fd td|jjD}td|jjD}||z sytfd|jjD}tfd|jjD}t ||}|dk(ryt fd|jjD}t fd |jjD} t || } | |z } | t j k\r| SdS) a8 Score fusion based on buffer name overlap when exact dep matching fails. This handles the split/cat fusion case where nodes read from the same buffer but at different indices (e.g., different slices from a split operation). Scoring logic: - If nodes read from exactly the same buffers: high bonus (encourages fusion) - For common buffers: score based on overlap ratio - overlap_ratio = common_buffer_size / max(node1_total_reads, node2_total_reads) - If overlap_ratio > threshold (e.g., 0.5): give proportional score - If overlap_ratio < threshold: minimal/no score (not worth fusing) Note on dynamic shapes: - When deps have unbacked symbols (dynamic shapes), dep_size_hint returns 0 - In this case, we use count * 10 as a proxy for size - This ensures fusion still works for models with dynamic batch sizes Note on multiple deps from same buffer: - A node may have multiple MemoryDep entries for the same buffer name (e.g., 4 split reads from arg0_1 at different indices) - We sum ALL dep sizes for each buffer, not just take max - This ensures overlap ratio is calculated correctly when nodes read multiple slices from the same underlying buffer rc8j|}|dkDr|SSrr )rrFALLBACK_DEP_SIZErrs rs get_dep_sizezFScheduler._score_fusion_memory_by_buffer_overlap..get_dep_size s%%%c*D!84 :): :ruc34K|]}|jywr~rtrus rsrzCScheduler._score_fusion_memory_by_buffer_overlap.. rrvc34K|]}|jywr~rtrus rsrzCScheduler._score_fusion_memory_by_buffer_overlap.. rrvrc3.K|] }|ywr~rrrr s rsrzCScheduler._score_fusion_memory_by_buffer_overlap.. $ "%L $ r=c3.K|] }|ywr~rr s rsrzCScheduler._score_fusion_memory_by_buffer_overlap.. r r=c3JK|]}|jvr |ywr~rtrrr9 r s rsrzCScheduler._score_fusion_memory_by_buffer_overlap.. *% xx<'  % r}c3JK|]}|jvr |ywr~rtr s rsrzCScheduler._score_fusion_memory_by_buffer_overlap.. r r})rr8rr)rrrrrYr-min_overlap_ratio)rrrrnode1_read_namesrnode1_total_read_sizenode2_total_read_sizemax_total_read_sizenode1_common_read_sizenode2_common_read_sizecommon_read_buffer_size overlap_ratior r9 r s` @@@rsr z0Scheduler._score_fusion_memory_by_buffer_overlapi sQ< ;&%R%:K:K:Q:Q%RR%%R%:K:K:Q:Q%RR(*:: !$$ ).):):)@)@$ ! !$$ ).):):)@)@$ ! ""79NO ! # "%% ((..% "  "%% ((..% " #&&<>T"U02EE (58P8P'P # VW ruct|dk(r|Si}|D]\}}|j|jk(sJ|j}t|j|j ||}||vr ||fg||<p||j ||ft |jtjdd}t|dkDsJ|S)Nrrr) rrrr#get_fusion_pair_priorityrrZrrr)rrr8"possible_fusions_group_by_priorityrrrfusion_pair_priority&possible_fusions_with_highest_prioritys rsrz4Scheduler.get_possible_fusions_with_highest_priority s  A %# #  +- LE5##%)9)9);; ;;%%'F#&  (AA%O$ $+MMENL23GH33GHOOEN 25 . 4 4 6H>>55rucBtjj|g|S)z- Shim for list.sort(key=...) )rar score_fusionrOs rsrzScheduler.score_fusion_key s yy%%d3U33ructtjj}t |j D]9}|j ||j|j|j;y)zg Populate node.last_usage recursively (also for the nodes within a FusedSchedulerNode) N) rrarrrrqrrrr")rrrrs rsrzScheduler.compute_last_usage s] ))A)A)CDTZZ( 8D    3T5L5L M  & &t 7 8ruct|jtjjz tjj j z D]z}||jvrT|j|}|js2tjj j|jf|tjjvstjj|}t|tjr*tjj j|t|tjtj fr|j"}t|tj$r|j'sJtjj j|j"}|jj)y)z*Free any buffers that are no longer neededN)rrrarrrfreedrr codegen_freerrXrr0rLrrrris_input_bufferrB)rrrrrstorages rs free_bufferszScheduler.free_buffers sV  % %gg%% &gg""(( )  DD t'''&&t,<<>GG((55chh?---gg**40c2#5#56GG((55c:b&7&79M9M%NO!hhG"7BMM:w?V?V?XXGG((55gllC) D, !!'')ruc|jjD]}|j|jyr~)rlrflushr# )rrrs rsr% zScheduler.flush !s3}}++- G MMO  rucvt|ttfsJtddxxdz cc<t j t d5|j|jddd|jtjj|jy#1swYCxYw)Nr] extern_callsrF)increase_kernel_count) rrRr*rraset_kernel_handlerr5rrrrrr# )rrscheduler_nodes rscodegen_extern_callzScheduler.codegen_extern_call!s  &(L M    ^,1,  ! !&u"E F &  0 0 2  # # % & qww334   & &s !B//B8cPt|jr|j J|dtjj |t |j}|td|jts|jdk(rLtjj|x}jdkrt|tjt|jr,|jdk(st!tj||S)Nz( should have been normalized in loweringzUnsupported device type: rrs)r[rrraradd_device_infor4rSr+r`rget_device_propertiesmajorr<inspect currentframer=)rrrdevice_schedulingrs rscreate_backendzScheduler.create_backend"!s&++&&,,*B h> ? B '5fkkB  $!:6;;-HI I| v%%*ZZ%E%Ef%MM\TTWXX(w7K7K7MNN $V[[E-A#G$8$8$:;; &&ruc|J||jvr|j||j|<|j|Sr~)rlr4 rs rsr#zScheduler.get_backend7!sB!!!  &$($7$7$?DMM& !}}V$$rucdfd }|jDcic]8}|j*|jjD] }|||fd:}}}t|j }|rMt |t jd\}}tjjj|yycc}}w)Nc|jvrLjjt|jjDcic]\}}|| c}}jScc}}wr~)rrrrrq)rrrrs rs get_orderz*Scheduler.enter_context..get_order>!s\,,,$$++i >V,WdaQT,WX''* *-Xs A+ rr)rz torch.fx.Noderr) rrrrr!rYrrrarr enter_context)rrrr8 rr&r$r=lasts` rsr9 zScheduler.enter_context=!s +^^% vv!VV'')  q\1 t #   w||~& 'x':':1'=>GAt GG . .t 4  s=Cc |j|j}tfd|Dxr||jvxr||j vS#t$rYywxYw)NFc3^K|]$}|jxs|jv&ywr~)rr)rrrs rsrzAScheduler.can_buffer_be_removed_through_fusion..W!s)V C 3C CCVs*-)rrKeyErrorrr)r)rrrrrs ` rsrz.Scheduler.can_buffer_be_removed_through_fusionO!sn $$T*00E VPUV V 4D111 4D333   sA AAc|j}t|tjjj rk|j x}r]t|\}}|tjvs|tjvr+t|tjjsJd|Stjjjjstjyt|t r)|j"D]}|j%|}|s|cSy|jJ|j's|j)dSt|jtj*ryt|jtj,ryt/|jddryt1|jry |j3|x}r|Stjj4r t7|ry y) z Return the reason why we should partition the inductor graph on this node, or None if the node is cudagraphable. zcustom partition op: Nz6partition includes all ops when cudagraphs is disabledz opszDeviceCopy opszConditional opsunbacked_bindingszunbacked binding opszCUDAGraph-unsafe custom opszdynamic shape ops)rrr`rar0r rSrVr-custom_should_partition_opsrrrrerNr rrshould_partitionr[r DeviceCopyrJrrZ&_uses_cudagraph_unsafe_unbacked_symintcudagraph_skip_dynamic_graphsr)rrrr rop_overload_packet_nameop_overload_namerrs rsrA zScheduler.should_partition\!s)) gu11@@ A%% %B %8DR8H 5 #%5'6+M+MM#v'I'II!"ejj&;&;<<<./?.@AA &&--886>>FK d. / "..u5!M "yy$$${{}oo'(- - dii /# dii 0$ 49914 8) !$)) ,0@@F F6 FM == 6 6-d3*rucTt}tjs|S|jD]}|j}|t |t jjjsA|j}|Pt|\}}|tjvr|tjvr|jD]g}tjjj!|}t#|t$j&t$j(fsW|j+|i|S)zc Collect output unbacked symints from ops in config.cudagraph_unsafe_unbacked_ops. )rr-cudagraph_unsafe_unbacked_opsrqrrr`rar0r rSrVrrarrrr)r* UNBACKED_INTUNBACKED_FLOATr)rrunsafe_symintsrr rrE rF syms rs&_get_cudagraph_unsafe_unbacked_symintsz0Scheduler._get_cudagraph_unsafe_unbacked_symints!s 4><33! !JJ ,DiiGgu'9'9'H'HI$$Bz8DR8H 5 #%5'v/S/SS$F,P,PP779 ,gg&&//4!#(9(94;N;N'OP"&&s+ ,' ,0ruc|j}|syt|}|D]I}tjjj |}|j D]}||vsd|ccSKy)Nz'uses cudagraph-unsafe unbacked symint: )rM rrarrrr&)rrrrK node_symbolsrL simplified_symfree_syms rsrC z0Scheduler._uses_cudagraph_unsafe_unbacked_symint!s~DDF5d;  PCWW--66s;N*77 P~-DXJOO P P ruci}|jtjj|jD]3}|j j D]\}}|j||<5|S)z~ Return a mapping from name strings to the corresponding graph inputs or base scheduler node outputs. )rrarrXrqr.rr)rrrIrrscheduler_buffers rsget_name_to_nodeszScheduler.get_name_to_nodes!srPR AGG001JJ ;D*.*>*>*D*D*F ;&&%5%:%: T" ; ;ruc ttjjDcic]\}}|| }}}ttjj Dcic]\}}|| }}}gtj_t|D]\}}|j rg}|jD]"}|j|j|$g} |jD]0} | j|j| j2tjj jt||| |jycc}}wcc}}w)z computes a mapping from partition input/output indices to graph input/output indices for each partition. N)rrarrXrpartition_mapsskip_cudagraph input_nodesrr output_nodesrrWconstant_names) rr signaturesrIrname_to_graph_input_indexname_to_graph_output_index partition_id signature input_mappingoutput_mappingrs rscompute_graph_partition_mapsz&Scheduler.compute_graph_partition_maps!sT(11E1E'F% ##tD#I% !% (11I1I1K'L& ##tD#I& "& "$'0'<  #L)'' M!-- J$$%>%B%B4%HI J N!.. W%%&@&D&DT]]_&UV W GG " " ) )! !",,  ! % & s E! E c dd dd}tjd|D}|jfd|jD||}t}|D]F}tj j j|}|j|jHtt|tjdS) ai Returns all symbol inputs which are required to be in scope to successfully perform codegen for this graph partition, including: - free symbols used in partition nodes - free symbols in partition input/node shapes, strides, and offsets. This is needed for recording cudagraphs for tensors with dynamic shapes. ct|tjr tSt|tjr t |St dt|)zW Gets symbols used in input node shapes, strides, and offsets. zUnsupported input node type: )rr0rLrrlrrrrs rsget_input_node_symbolszKScheduler.get_graph_partition_symbol_inputs..get_input_node_symbols"sN $ 2 23!|#D")),)$//*,I$t**VWWruc&td|DS)z Filters a set of symbols that are required for codegen. Skip symbols that are always internal to kernels, such as SymT.TMP, SymT.INDEX, and SymT.R0_INDEX. c3K|]N}t|tjtjtjtj fr|Pywr~)r)r*SIZEFLOATrI rJ rr9s rsrzVScheduler.get_graph_partition_symbol_inputs..filter_symbols..."sH !  ))++  sAAr)symbolss rsfilter_symbolszCScheduler.get_graph_partition_symbol_inputs..filter_symbols&"s     ruc32K|]}t|ywr~rrs rsrz>Scheduler.get_graph_partition_symbol_inputs..="s It,T2 Irc3.K|] }|ywr~r)rrre s rsrz>Scheduler.get_graph_partition_symbol_inputs..@"s Lt$T* Lr=rr)rz+ir.IRNode | sympy.Expr | ir.TorchBindObjectrOrderedSet[sympy.Symbol])rk ro rro ) rrrrrarrrr&rr attrgetter) rr partitionrX rl candidate_symbolsrr9 symplified_sre s @rs!get_graph_partition_symbol_inputsz+Scheduler.get_graph_partition_symbol_inputs "s X= X % X  -  % ,7Ijl6H6H Iy I7  !  L{7I7I7K L ++<=(2 " 2A77++44Q7L JJ|00 1 2 &(*=*=f*EFGGruc g}ttjj}j }dfd t t |t |D]2\}}t}|D]+} |j| jj-|j|} tjj|D cgc]} | jc} } t| j| j zD cgc]} t#| t$s | j& c} |z } tfd| D} t}|D]} |j| j(||z Dcgc]}||vr| }}| j|| Dcic] }||vr|||}}| Dcic] }||vr|||v }}| Dcgc] }||vr||vr|}}| j|tfd| D} | Dcgc]}|s||}}| Dcgc]!}|tjj*vs |#}}j-||}t/||||||}|j1|| j3|| z }5|dddScc} wcc} wcc}wcc}wcc}wcc}wcc}wcc}w)z Gets signature for each graph partition, including input nodes, output nodes, and whether deallocating an input within graph partition. cjj|d}|yt|jjt r'j j|dx}r|Syy)z Checks if buf_name resolves to a NoneLayout buffer (following mutation_real_name). Buffers with NoneLayout are not allocated so graph partition should not take them as inputs or outputs. NFT)rrrrrrDr)rrr3is_unallocated_bufferrrs rsrw zFScheduler.get_graph_partition_signature..is_unallocated_bufferY"sh ""&&x6C{#((//:6!% 7 7 ; ;Hd KK9K0;;ruc3VK|] }jj||"ywr~rrrb s rsrz:Scheduler.get_graph_partition_signature..",/''++D$7/rc3VK|] }jj||"ywr~ry rb s rsrz:Scheduler.get_graph_partition_signature.."rz rNr)rr,rr})rrarrrT ryrrr.r!rr/rrrrrQrr;rr"rprt rArr)rr partitionsskip_cudagraphsr[ unmet_output_namesrIrq rW output_namesrreturned_output_namesrrpartition_input_namesrrextra_input_namesrX input_deallocationextra_output_namesrY rZ symbol_inputspartition_signaturerw s` @rsget_graph_partition_signaturez'Scheduler.get_graph_partition_signatureM"s^ '(@(@(BC--/  ,*- Z (?";* g  %I~-7LL! A##D$8$8$=$=$?@ A%1$=$=>P$Q !'11<<.78d!!8K"-!2!2[5G5G!G)!W5 "%//1/% ! 5?L ! =$++DOO< =2L@!<'! ! " ( (): ;2<'l4((K2"<'d222" "2"<'D8L,L" " " ( (); <$./1/% !2,T2T"L"7$!''BSBS:SN!BB;M#:" #    1 2!6! J)"J.9J3J8 J=%!KKc|jjDcic] \}}||vr||}}}|jjDcic] \}}||vr||}}}|jDcgc]}|j |vr|} }|j Dcgc] }||vs| } }t |j|| ||j| Scc}}wcc}}wcc}wcc}w)z Updates the partition signature by removing buffers specified in removed_buffers. See [Note: Removed Graph Partition Arguments] ) rX rr rY maybe_get_namerZ rAr rW ) rrr_ rrr"rX r r rrY rZ s rs.clean_removed_buffer_from_partition_signaturesz8Scheduler.clean_removed_buffer_from_partition_signatures"s!* 5 5 ; ; = f?* &L  '99??A c?* #I  ".. ""$O;   '55 _9TD  '  # #     $ $    %    sCC1C C!&C!cp ddl t gg t|Dcic]\}}|| c}} d fd d fd }|D]5}t|jj  |< |dk(s. |7g}d}|t|kr sr} r0 j \}}|j||| r0r0 j \}}|j|||r0|dz }|t|kr rzr}|t|kDr td|Scc}}w)a Reorder nodes to minimize the number of partitions via a bfs topological sort. This is the optimal reordering such that the number of partitions cannot be reduced further. This may be sub-optimal for other metrics such as peak memory. This does not change relative orders of two cudagraphable nodes, nor the relative order of two non_cudagraphable nodes. rNc||f}j|rj|yj|yr~)rA heappush)rnode_with_indexcudagraphable_nodesheapq node_to_indexnon_cudagraphable_nodesrrs rsinsert_pending_nodeszHScheduler.reorder_for_minimizing_partition..insert_pending_nodes#s>,T2D9O$$T*6H2ODruc|jjD]*}|dkDsJ|xxdzcc<|dk(s#|,yr)r' succ_nodes)r succ_noder node_to_indegrees rsupdate_indegreezCScheduler.reorder_for_minimizing_partition..update_indegree#sT!]]55 4 ' 2Q666 +q0+#I.!3(3  4rurz Failed to schedule, while loop ran too long when reordering for minimizing the num of partitions r>) r rxrrr' pred_nodesheappoprrS)rrrqrIrr r num_itersr=r r r r r r s` @@@@@@rs reorder_for_minimizing_partitionz*Scheduler.reorder_for_minimizing_partition"sU 9=CEGI4=e4DEysDsE  E E 4 +D%()A)A%B T "%*$T* + -/ #e*$ #':)--(?@4%%* &--(;<4%%& NI#e*$ #': s5z ! ]Fs D2c ddlm}m}ttj j }|||j|jttj jj|\}}|j|}||||\}} ||tjjzkr|S|S)zx Reorder nodes to minimize the number of partitions if this only slightly increase peak memory. r)estimate_peak_memoryprepare_planning_info)rr r rrarrrrrXr!r r-r!cudagraph_partition_memory_budget) rrrqr r rdefault_peak_memoryr4reordered_nodesreorder_peak_memoryr=s rsrz0Scheduler.maybe_reorder_for_minimizing_partition?#s H"177#;#;#=> :O      # # qww++002 3  ; 77??F!5 7" Q !FMM$S$SS T# " ruc g}g}g}dd}|D]n}|j|du}|r*t|jdk(r|j|B|r||r|j|^|j|p||z|zS)a Reorder a node if it should be partitioned and has simple dependency: 1. move a partitioned node to the front if it has no dependency 2. move a partitioned node to the back if it is only used by OutputNode 3. otherwise do not reorder c|jD]0}|jD]}t|jtry2yr )r<rrrr )rrr s rsonly_output_userzPScheduler.reorder_for_partition_with_simple_dependency..only_output_usern#sC'') %99%C%chh ;$% %ruNrr*)rA rr3r)rrrqfrontmiddlebackr rrA s rsrz6Scheduler.reorder_for_partition_with_simple_dependency`#s*,*,(*  $D#44T:$F C(?(?$@A$E T"!&6t&< D! d# $v~$$rucg}d}g}g}|jD]S}|j|du}|r)||k7r$|j||j|g}|}|j|U|r"|j||j|tjj }|dkDrXt t||D]@\}\} } | r td| D} | |ks$d||<tjd|| |B|j||} |j| |j|| || fS)z Given a list of BaseSchedulerNodes, split into a list of graph partitions and compute partition input/output signatures. TNrc3@K|]}t|tsdywrN)rrrs rsrz,Scheduler.graph_partition..#s#')!-CD'rzFPartition %d has %d kernels, below minimum size %d, skipping cudagraph)r| r} )rqrA rr-rcudagraph_min_partition_sizerryrcudagraphs_logrr rb _log_graph_partitions) rrr| rW cur_partitionr} rnode_should_partitionmin_sizerrq skip kernel_countr[ s rsrzScheduler.graph_partition#sm+- ') JJ 'D$($9$9$$?t$K !3H!H!!-0&&~6 " 2N   & '    m ,  " "> 2==== a<(1#j/2R(S $$It#&'!*'$L $h.-1*&,,d($  "77!?8  ))*5 "":z::%%ruc `tjtjsyt dt j jD}|sytd|D}t||z }tjdt|||tt||D]\}\}}tjd|t||jrdndt|jt|j|jsm|D]} |j!| y)Nc32K|]}t|ywr~)r[)rrs rsrz2Scheduler._log_graph_partitions..#sOVF^Orc3:K|]}|jrdywr )rW rj s rsrz2Scheduler._log_graph_partitions..#s!Pq?O?O!!PszCCreated %d graph partitions: %d cudagraphable, %d non-cudagraphablez3 Partition %d: %d nodes, %s, inputs=%d, outputs=%dznon-cudagraphable cudagraphable)r rrrrrar device_typesrrrrryrW rX rY _log_non_cudagraphable_node) rrr| r[ has_gpu_devicecudagraphable_countnon_cudagraphable_countrrq r_ rs rsr zScheduler._log_graph_partitions#s **7==9 O!'':N:NOO !!PZ!PP"%j/4G"G Q  O  #  *33z:3N)O ; %A% 9  EI'0'?'?#_I))*I**+  ''%;D44T:; ;ruc|j|}|sy|j}|j|jjnd}d|g}t |jj }|j d||F|jddjd|jDd}|j d|tjd |dj||Z|jjd d}|r;|jjd D]} tjd | yyy) z)Log details for a non-cudagraphable node.Nzreason=zir=rEr]c32K|]}t|ywr~)r,)rrs rsrz8Scheduler._log_non_cudagraphable_node..#s2Pa3q62Prr@zfx=z %s: %srrHz %s)rA rrr_rrrrrdrr rrrstripsplit) rrrrrMrbpartsir_typefx_strrlines rsr z%Scheduler._log_non_cudagraphable_node#s1&&t, MMO 151F$))++-D6(#$tyy/** s7)_%  'q2P7<<2P)P(QQRSF LL3vh (\9dii6FG  !,,**=$?K'--/55d;>D"((=> ructd5tjjjr|j n|j |j cdddS#1swYyxYw)NzScheduler.codegen)rr`rar-r_codegen_partitions_codegenrqrqs rsrzScheduler.codegen#sX - . ??))99((*]]4::.    s AA&&A/cddlm}tjj}t |j }tjj5tjjdd|||tjjj}|j|ttjj|sJtjj|z }|j||}|tjj_tjjjtjj }tjjj#tjj$\} } dddtjjj' tjjj)||tjjj*j-|j.D cgc]} | j1c} y#1swYxYwcc} w)z,Codegen a partition given its inputs/outputsr)SubgraphPythonWrapperCodegenT partition_) is_subgraph subgraph_nameparent_wrapper_codepartition_signaturesN)rr rarrrroset_current_wrapper_codeinit_wrapper_coderrxr rr r write_prefixrgenerate is_inferencedefine_subgraph_launcher_fncodegen_partition_call allocatedrrY r) rrrq r_ r r graph_partition_idremoved_buffers_before_codegenremoved_buffers_during_codegen graph_namepartition_coder=rs rs_codegen_partition_wrapperz$Scheduler._codegen_partition_wrapper#s Bgg22!$"?"?@ WW - - / T GG % % *+=*>?$7%. & ./WW-D-D-I-I-K * MM) $agg224PQ QQ''*HH +KK9I9BAGG 5 GG - - /J ! 4 4 = =agg>R>R S NA? TB 88^T 334F R &&--)2)?)? @T]]_ @ I T TJ AsEI$I0$I-cLtjdfd }|S)Nc3KjjrytjjrZjjJdt j jjjj djrGtjjr(t j jjd_y#jrGtjjr(t j jjd_wxYww)Ndevice should have an index) %update_graph_partition_default_devicervrQrrrarrcodegen_device_guard_entercodegen_device_guard_exit)r| rrr[ srsctxz1Scheduler.use_default_device_context..ctx1$s  6 6z: N**/@++0002288D1D$$??//55 3..3D//444GG((BBD.2+ ..3D//444GG((BBD.2+s BEC; AE;AEE)rzIterator[None]) contextlibcontextmanager)rrr| r[ r s``` rsuse_default_device_contextz$Scheduler.use_default_device_context.$s&  " " 3 # 3*u ruc t|dk(r|djsydd} dd}d}t||D]\}}|jr||}n|yt||D]\}}|js|||ry||_y)Nrrc4|dj}|J|Srr)rq partition_devices rsget_cudagraph_partition_devicezWScheduler.update_graph_partition_default_device..get_cudagraph_partition_deviceX$s'(|668 #/ //# #ruc@|D]}|j}||k7syyr r)rq target_devicerrs rsall_on_target_devicezMScheduler.update_graph_partition_default_device..all_on_target_device]$s/" !*]*  !ru)rq rgrr)rq rgr rrr})rrW ryrv)rrr| r[ r r cudagraph_partition_devicerq r_ s rsr z/Scheduler.update_graph_partition_default_deviceI$s z?a  1 (D(D  $  $ 5A   &*"$' J$?  Iy++-KI-V*  & - $' J$?  Iy''0D51   'A#ruc|j\}}t|dkDrtddxxt|z cc<|j||5t ||D]V\}}t|dk\sJdt||j r|j |E|j||X dddt|j}tjjj||dkDrqtjjJ|ttjjk(s.Jd|dttjjyy#1swYxYw) z Split nodes into partitions and codegen each partition into separate functions. This allows further applying different optimizations (e.g., cudagraph) to each function. rr]cudagraph_partitionsz5Each partition must have at least one node but found NrzExpect z partition maps but got )rrrr ryrW r r rrorarrset_all_partition_namesrV )rrr| r[ rq r_ num_partitionss rsr zScheduler._codegen_partitionsx$sh "&!5!5!7 J z?Q  Z !7 8C O K 8  , ,Z D J(+J (C J$ 99~*KCPYNK[\*++MM),33IyI J Jd;;< 44^D A 77))5 55!S)?)?%@@ .))A#aggF\F\B]A^_ @  J Js A&E55E>ctjrddl}tj}t }t |D]}|jdk(r/|j|jjjk(rnQ|j|jf}||vs"Jd|jd|jd|j||j|_|j J|jrBtj"j$r(t&j(j*j-t&j(j*j/|D]}t0j3t4j6r4 t0j9d|j;|j=|jA|tjBrDt&j(j*jEd|jFjHD|jKx}rs||jk7s |jMs|jOr|jQ||jk7r$|jrctS|jjTrD|jV|jYt&j(j*j[||_tS|jTr|j\Jd d } |j_r5t |j`jc} | rte| d znd } t&j(j*jg|j\| |jh|j_r|j|jk|t&j(j*jmd |jFjHD||_7|jpjs|jt|jOrP|jwty|j{\} } } |j}|j| | | nf|jMr|j|nC|jrxtjt|}|j}|}d d lEmF}d d lGmH}d dlImJ}t||||fr|}ntdtU||j|nt|tr!|j}|j|nt|tr!|j}|j|nYt|ttfr!|j}|j|n"t|tsJ|jtj"jr|j}|j|jjs|j|jjs|jt|tsP|jK}|>|jTdk7r/|j}|jr|jQtd|j{Dr ||_d|_|j|jk7rU|jJtS|jjTr(t&j(j*j[d|_|jQy#t>$r(t0j9d|j;Y"wxYw)Nr_compile_innerzDuplicate stack frame :zs; did you add a decorator to one of the functions in this stack trace? If so, try using a context manager instead.z5Generating code for node %s with estimated runtime %fz6Generating code for node %s with estimated runtime 0.0c34K|]}|jywr~rtrus rsrz%Scheduler._codegen..$sD!$CHHDrvr rc34K|]}|jywr~rtrus rsrz%Scheduler._codegen..$sC Crv)CUDACombinedSchedulingr)XPUCombinedSchedulingz type(self)=rc3<K|]}t|tywr~r) rs rsrz%Scheduler._codegen..:%sJA:a/Jr)_r-"check_stack_no_cycles_TESTING_ONLYtorch._dynamo.convert_frame traceback extract_stackrrrfilename_dynamo convert_frame__file__linenorrvrrsrautotune_at_compile_timerarrwrite_get_raw_stream_headerregister_alignment_check_inputsrTrrrrrrhrSr9 size_assertscodegen_deferred_input_assertsrrrrrr% rQrcurrent_stream_idxgenerate_stream_ctx_exitr rrrvrrYr rgenerate_stream_ctx_switching!codegen_deferred_alignment_copiesrtrrr"rrrr#codegen_templater+ rrrrB codegen.cuda_combined_schedulingr rr#codegen.xpu.xpu_combined_schedulingr rrwcodegen_combo_kernelrcodegen_nested_reductionrcodegen_mix_order_reductionrr codegen_noderrdebug_sync_kernel codegen_syncrrrrready_to_flushr)rrrqr`rrDframerrr num_streamsunique_streamsrrrbackend_r rr rs rsr zScheduler._codegen$s>  4 4 .++-E7A|D!% JJ"22%--*E*E*N*NN~~u||4$,U^^, <<>E *D . IIO 224   t $""$$CCD(,(8(8(>(>D**v*d111~~''')JJLT000**/@++000 22> 99;,,FFH*0D'(5%||7V9VV7&' 779-78K8K8R8R8T-UN;IN 3a 7q(,,GG"LL' ;;++-$2E2E2Q2248 GG B BC$($4$4$:$:C !%D   % % , ,T__ =!484W4W)*51-  (99!8X!((."{{#=tD++F3T8V#%;=RS'G(KDJ=)9::,,T2D"78  (AA$GD"9:  (DDTJD#5}"EF  (55d;!$(>??? }}..  (557  ' ' . .t/D/D/F G  % % , ,T-E-E-G Hd$:;*& v-((0??AJJLJ9IJJ%)"%)"KE *N   $"="= =&&2 22 !4!4!9!9:$$>>@! U!IIP s3^44-_%$_%c|dj}|tj_||_|J|j |}|j ||S)rqr)rrarrrr#benchmark_combo_kernel)rrrnode_benchmark_resultsrrs rsr z Scheduler.benchmark_combo_kernelL%sZ1((* $!!!""6*--i9OPPruc|}|djtfd|DsJdtjsyddlm}dg}}i}t |D]\}}|j} |j| rtjd |j| \} } | | f||<tj| rtjd|y || z }|j|  |j||\} }}| |z d kxs|d k}tj!t"j$rP|| kDs|r%tjdt'|| z dn$tjdt)|| z d| |z |kxs|S#|$r.} d t| vrtjd Yd } ~ yd } ~ wwxYw#|$r-} d t| vrtjd Yd } ~ yd } ~ wwxYw)rrc3DK|]}|jk(ywr~r)rrrs rsrz4Scheduler.speedup_by_combo_kernel..c%sK44??$.Ks z$$U! " 2IC   d #9 $< *.*E*E!7+ 'CKY,9c  " "7== 1SyL  E#)C2   Ic #0 Y$4 4Q$ *c!f4$$]     &#a&0  Y s=AF9GG""G  G  GH"H?HHcp|j|}|jJ|jjSr~)rr get_layout)rrrrs rsget_buffer_layoutzScheduler.get_buffer_layout%s5x(xx###xx""$$ruc|jD]}|js|jjD]}tj j j|j}|s9t|dk(sHt|jttfri|jgk(s}tj jj!|jyr)rqr[rrrarrrrr@rrrDrCrczero_dim_cpu_tensor_listr)rrrrr"s rsruz$Scheduler.update_zero_dim_cpu_tensor%sJJ HD{{} ,,22 HDWW3377 BF+F3u< *"MMJ8I+J!#OO-388<":. .ruct|trJ|j|}tjj j ||_y)z6Code-gen to enter the Stream context assigned to node.)rN)rrrvrarrcodegen_cuda_stream_enterr)rrr node_streams rsgenerate_stream_ctx_enterz#Scheduler.generate_stream_ctx_enter%sId$:;;;))$/ #$77#7#7#Q#Q"$R$  ruc~|jJtjjj d|_y)z1Code-gen to exit from the current Stream context.N)rrarrcodegen_cuda_stream_exitrqs rsr z"Scheduler.generate_stream_ctx_exit%s2''333 557#' ruc&||jvsJt|trdn|j|}|j|k(ry|j|y|j||j |y|j |j |y)amGenerate stream entering and exiting to properly run node in a multi-stream scenario. Stream context switching is only generated if ``node``'s assigned stream is different from the previous node's stream. NopKernelSchedulerNodes have stream=None and inherit the enclosing stream context (or do nothing if no context is active yet). N)rvrrr r1 r )rrrrQs rsr z'Scheduler.generate_stream_ctx_switching%st*****$ 67 $$T*   " "f ,   $ $ 0V^   $ $ ,1C  * *4 0  ) ) +  * *4 0ru)rqzlist[ir.Operation]rr)rz!dict[str, SchedulerDonatedBuffer]rr)rr,rr)rr,rrfrr}r)rrrr)rr,rr)rrrrf)r str | Nonerr}r)rrfrr)rrrqrrtuple[float, str]r~rqrrzr}ryrrr,)r~rrrrr7 )rir.MultiTemplateBufferrr}) rir.OperationBufferrr9 rrrrrr)rrrr})rqrryrrz&tuple[LambdaFuture | None, ModuleType])rrfrrfrrk)rrfrrf)rrfrrfr\OrderedSet[BaseSchedulerNode]rrf)rrfrrfrrr\r; )r,dict[BaseSchedulerNode, list[PendingFusion]]r\r; rr) r.1list[tuple[BaseSchedulerNode, BaseSchedulerNode]]r,&dict[BaseSchedulerNode, PendingFusion]r/r< r\r; rjr})r\r; r,r> )r8r= r9r= )rqrrjr}rr)rqrrrrErrz!Iterator[list[BaseSchedulerNode]])rXrrr)rr)r{rrdrr[r}rz-tuple[ForeachKernelSchedulerNode | None, int]) rrrPrrdrr[r}rSzJCallable[[ForeachKernelSchedulerNode, list[BaseSchedulerNode], int], None]rrr)rqrrjr}rr= r+)rrfrrfrrrr})rrfrrfrz!tuple[str, ...] | OrderedSet[str]rr,r/r*)rF rfrrfrSrrr})rrfrrfrz,tuple[int, SchedulerNode, sympy.Expr] | None)rErfrJrfrr)rrfrrfrOrderedSet[str] | None) r7rfrrfrizCSequence[tuple[BaseSchedulerNode, NestedReduction.PointwiseDomain]]r%r}rr})FT) rrfrrfr%r}rr}rr})FTN) rrfrrfr%r}rr}ro r? rr})rrfrrfro r? rr})r r;rrfrrfrr})rr9rr9rr})rr8rr9r r}rr})rr8rr:r r*rr})r r8r r8rr}r)rr8r r}rr)TN) rErfrJrfr r}ro r? rr) rrfrrfrr}ro r? rr)...) rrfrrfr r}r zLiteral[False]rr}rr) rrfrrfr r}r z Literal[True]rr}rztuple[int, int, bool])TFT) rrfrrfr r}r r}rr}rzint | tuple[int, int, bool])r8r= rr= )rqrrr)r* rfrr)rrrBaseScheduling)rrrr@ r>)rr,rrrr})rrfrr5 )rro )r6dict[str, ir.IRNode | ir.TorchBindObject | sympy.Expr])r[ list[GraphPartitionSignature]rr)rq rgrX rA rro )r| list[PartitionType]r} z list[bool]rrB )r_ rArrrrA)rz9tuple[list[PartitionType], list[GraphPartitionSignature]])r| rC r[ rB rr)rq rgr_ rArr)r| rC r[ rB rz'contextlib.AbstractContextManager[None]rrrz%tuple[float, float, list[str | None]])rqrrr})rr,rz ir.Layoutr)rr5 )rrrrr:rNrwrrrrpropertyrsetterrrrrrrdrr|rCr{rrVrtr}r~r\rrrrrwr}rrrrrrrrrr&r2r6r>rkr0rJrrYrr]rrArrrrrr r r r rrJ r^ rf rh r&rrs rp r rr r r r r r2 r r r r rr r rrrr# r% r+ r4 r#r9 rrA rOrM rC rT rb rt r r r rrrr r rr r r r r r rZr( rur r- r1 r r rrs@rsrrs  h9T #6 p(S Q&&((7# ,"HMP^KZ+#Z , 6 S*4#&/<$6!F 808 8,%) *"  $ >  >*6 >  >'0' 'RIDV'8&'8+'8  '8  '8  '8R OS0AK / Z&Z/@Z Zx> !3   0 !'  3 "D2$PD23D2  D2LO?PO?@O? L O? 3 O?  O?bS2S@S0$K$$U$(C&CC ! CJ&1 + 8e.N( TC&,C&*C& C& 7 C&J'$*'$'$* '$  '$ '$ '$R?6 &6 6  ; 6 p,&,/@, ,\7&7/@7 7r.2&.2/@.2MP.2 .2`$&$/@$ $6< <!<< < <|M&M/@M M^76&76/@76 76r]K ]K!]K ]K~^ ^!^  ^@ ;(;); ;  ;z`&`/@` 5`D  )  5F    J J!J J"! '! !!  ! !  ! N"*.  !  $(   4"*.    !   $(    ,"*.=A uM uM!uM uM $( uM %; uM uMx >B G G!G %; G  GR3)3)(93)BS3) 3)j  OT== )=GK= =~==&/= =*OIOIh   ' 7J   44";!=A (#($( ( %; ( (\+/=A  !$(  %;  *  8;*-  !  (6  $(    7:*- $ $!$ $ (5 $ $( $ $$!.3*. ^K ^K!^K ^K (, ^K $( ^K %^K@d d!d  dLR &R /@R R h6 Q6 :6@4@4 4 8*4 ) &'*% 5$    +:    =~!!F% " ? '1' 'RBH BHLBH " BHHK -K @JK &K Z" *" )"  ! " H?&? !?B& !B%,% %@5& B5&n";'";2";  ";H>01  1 +1   1 f-;X 06-A--A;X-A -A^@rh Q4 Q . QN5`% H ( 1rucDeZdZdfd ZddZddZ ddZ ddZ ddZ ddZ ddZ dd Z d dd Z dd Z d d Zd!d ZddZd"dZddZ d#dZd$dZ d%dZ d&dZ d d'dZxZS)(r@ c0t|||_yr~)rr:r)rrrrs rsr:zBaseScheduling.__init__%s "rucR|jr|jjyyr~)rr# rqs rsfree_buffers_in_schedulerz(BaseScheduling.free_buffers_in_scheduler%s >> NN ' ' ) ructS)z0Return a set of .codegen.common.BackendFeature()rrs rsget_backend_featuresz#BaseScheduling.get_backend_features&s |ruct)zO Check whether node1 and node2 can be vertically fused or not. rrs rsr z BaseScheduling.can_fuse_vertical& "!ruct)zQ Check whether node1 and node2 can be horizontally fused or not. rrs rsr z"BaseScheduling.can_fuse_horizontal&rN ruc|j}t|tjsy|j syt|j tj rt|j jdk(xrkt|j jdtjxr8|j jdj|jk(Sy)av A Multi-Output Template (referenced in #144012) is a template node with MultiOutputLayout, and its output buffers are instances of MultiOutput. In this context, we verify whether node1 represents the Multi-Output Template and node2 corresponds to one of its outputs. If so, we further check if backend supports this fusion. Frr) rrr0r!r\rrBrrrlr)rrrrrs rsr z.BaseScheduling.can_fuse_multi_outputs_template&s..0 ,(9(9:557 ejj".. 1EJJ%%&!+Ouzz003RYY?OJJ%%a(113|7L7L7NN  ruc|js|jrtj||Stj ||r"tj ||r t ||Stj||r t||St|t r|j|St|tr|j|St|trLt|tr  UBL GG > >\  ru)rzScheduler | Noner)rrrzOrderedSet[BackendFeature]r+r)rCrrz"tuple[tuple[sympy.Expr, ...], ...])rrfrT rr rrr5 r~r8 )rz"FusedSchedulerNode | SchedulerNoderr)rrrr)rrrrrr6 )r~rrr7 r/rD )rrrr5 rr)rrrr:rJ rL r r r ryr$r rwr r r r r r% rrr}r r rg rrs@rsr@ r@ %s#*"&"/@" ""&"/@" "&/@ 49&9/@9 94"3" +" "( "4 "4 "  "$%) "* " "" " "" """ " "0" ""&/@ "4" ."#'2   rur@ r+)rz$torch._inductor.codecache.LocalCache)rrfrr,)rrfrzCallable[[Any], Any] | None)rrfrr/)rrrr,)rrfrrrr-rr)r)FusedSchedulerNode | GroupedSchedulerNoderr)rrh rrrrrr)r)rzlist[list[int]]rCrrrrr)rr9 r]r: rrr) rrrrrrrrrrHrztuple[int, int])rrqrrqrrrrrrrrrrHrr})rz ir.IRNoderro )rrfrro )rrfrr})rrfrr}rr})rrfrrfrr})rrfrrf) __future__rrWr rrCrr1 rrrrrrryrr[r rrrconcurrent.futuresrrrr r r r r rrtyping_extensionsrtorch.utils._ordered_setrr0rrcollections.abcrrrtypesrtorch._inductor.codegen.wrapperrrrrrrr`torch._inductor.async_compiletorch.utils._pytreer_pytreertorch._dynamo.utilsrr torch._inductor.autotune_processrtorch._inductor.codecacher r!torch._inductor.irr"torch._inductor.metricsr#r$torch._inductor.stream_utilsr%%torch.fx.experimental.symbolic_shapesr&torch.utils._sympy.functionsr'torch.utils._sympy.symbolr(r)r*torch.utils._tritonr+rXr,r-r.r/r1analyze_preserves_zero_maskr2codegen.commonr3r4r5 comm_analysisr6r7r8r9r:r;excr<r=fx_utilsr>r?r@rArBrCrDr$rErrFrG runtime.hintsrHrIruntime.runtime_utilsrJrKrLrrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r]r^r_r` virtualizedra getLoggerrrT_logginggetArtifactLoggerrrarr rrgrrhrirrkrrrrrr2rr rfryrornrrurrr rrRrrrrrrrr*rBrrrrrrr1rnrmrrrrr r$r&r)r+r-r/r@rr@ rrursr s"     ,3   (/)<< J51 $$$6E?7M8>1OO*DDDMM;:2$ J:FF&*g! ^^ - -h A NN44XO >>;; $11(LI 34 y4 T]t_ DDD* ((( * T"NN#N*       __D mm` h8h8h8V 1_11T 1T 1n 22,  ,#LT"  # *   *K *K4*K,*K *KZ"* 1"*J5.5A*%A*H : $: $ 8k**k*\hG0hGV\<.\<~<-+=<-~L:!3L:^ b,bP#%+#+ + + +\0%01C0 08 1(((( ( # (  (6       # @    >+9??,$ &"6: .2 4:$:5F: :"P@ /2/2/2d ===@{X1{X1|qAAru