^j&ddlmZddlZddlZddlZddlZddlZddlZddlZ ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlmZmZmZddlmZmZmZddlmZmZmZmZddlm Z m!Z!m"Z"ddl#Z#ddl$Z#ddl%m&Z&ddl'm(Z(dd l)m*Z*dd l+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1dd l2m3Z3dd l4m5Z5m6Z6m7Z7m8Z8m9Z9dd l:m;Z;ddle j~jddZAe"rddlBmCZCddlDmEZEmFZFmGZGddlHmIZIddlJmKZKddlLmMZMdZNe;eOdZPGddeQZRGddZSGddZTe*je*jzZWejGdd ZYejGd!d"ZZGd#d$eZZ[Gd%d&Z\Gd'd(Z]Gd)d*eZZ^Gd+d,e\e^Z_Gd-d.e]e^Z`Gd/d0eZZaGd1d2e\eaZbGd3d4e]eaZcGd5d6eZZdGd7d8e\edZeGd9d:e]edZfGd;de]eZZhGd?d@e\eZZiejdLdAZk dMdBZlGdCdDZmdNdEZndOdFZo dPdGZpGdHdIZqGdJdKZry)Q) annotationsN)CallableIterableSequence)FutureProcessPoolExecutorThreadPoolExecutor)byrefc_size_tc_void_pCDLL)AnyIO TYPE_CHECKING)get_interface_for_device) rand_strided)ir) CppCodeCache CUDACodeCache DLLWrapperget_hash PyCodeCache XPUCodeCache)Timer)do_bench_using_profiling get_gpu_typeget_ld_library_pathis_gpupython_subprocess_env)getArtifactLogger) OrderedSet.TORCHINDUCTOR_AUTOTUNE_POOL_INACTIVITY_TIMEOUT600) ModuleType) ChoiceCaller PartialRenderTritonTemplateCaller)config) benchmarker)VCUDA_VISIBLE_DEVICES autotuningc eZdZy)!NonzeroWorkspaceNotSupportedErrorN__name__ __module__ __qualname__k/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/autotune_process.pyr/r/Kr5r/ceZdZdZeddZe d ddZeddZddZdZ ddZ ddd Z ddd Z ddd Z dd Zdd ZddZddZy) TuningProcesszF Class to launch and interact with a benchmarking subprocess. ctjdtjtjj t fd} |y#t$rYywxYw)z4 Entry point for the child process. z3Started autotune subprocess %s. Visible devices: %sc tj\}}|y |rtjj ||}tj |[#t $r }|}Yd}~'d}~wwxYwN)r9recvosenvironupdate Exceptionsend)job extra_envresulte read_pipe write_pipes r6workloopz,TuningProcess.process_main..workloop_sq!.!3!3I!>Y;  )))4 UF""6:6!Fs(A A3'A..A3N)autotuning_logdebugr>getpidr?getr,EOFError)rGrHrIs`` r6 process_mainzTuningProcess.process_mainTsQ  A IIK JJNN/ 0 7  J   sA A('A(NcTtj||f||jyr<)pickledumpflush)objrHrDs r6rBzTuningProcess.sendss#  S)$j1r5c,tj|Sr<)rQload)rGs r6r=zTuningProcess.recvzs{{9%%r5c2||_|jyr<)devicestart)selfrXs r6__init__zTuningProcess.__init__~s  r5ctjjtjjtd}tj \}}tj \}}tj |d|_tj |d|_tj|_ |jj|jtjtj|dtj dt#|dt#|g}it%dt't(j*rdndd }|j,t#|j,|t.<t1j2||||f |_tj6|tj6|d |_y ) z4 Start the benchmarking subprocess. z__autotune_main__.pywbrbz --parent=z --read-fd=z --write-fd=01)TORCH_WARM_POOLLD_LIBRARY_PATH3TORCHINDUCTOR_PROFILE_WITH_DO_BENCH_USING_PROFILINGN)envpass_fdsT)r>pathjoindirname__file__pipefdopenrHrG selectorsDefaultSelectorselectorregister EVENT_READsys executablerLstrrrr)/profile_bandwidth_with_do_bench_using_profilingrXr, subprocessPopenprocesscloserunning)rZentrysubproc_read_fdwrite_fdread_fdsubproc_write_fdcmdrds r6rYzTuningProcess.startsw RWW__X68NO$&GGI!$&GGI!!))Hd37D1!113  t~~y/C/CD NN  } %_-. /#./0 1    #%   #24EEDG   ;; "(+DKK(8C$ %!'' %'78  ! !" r5cV|jxr|jjduS)z: True if the subprocess is still running. N)ryrwpollrZs r6alivezTuningProcess.alives%||; 1 1 3t ;;r5c|js|jtj||j|y)z8 Push a work item to the child process. rDN)rrYr9rBrH)rZreqrDs r6putzTuningProcess.puts/zz| JJL39Er5c |jj|s"td|jjt j |j\}}t|tr||S#t$r|jt$r|jt$r<tjd|jj|jwxYw)z Get a response from the child process. Raises TimeoutError on timeout; raises EOFError if the subprocess crashes. zTimeout in autotune subprocess z.Unexpected exception in autotune subprocess %s)rnselect TimeoutErrorrwpidr9r=rGkillrNrxrArJ exception isinstance)rZtimeoutrE_s r6rMzTuningProcess.gets  ==''0"%DT\\EUEUDV#WXX%**4>>:IFA fi (L !  IIK   JJL    $ $@$,,BRBR  IIK   s AA55A7C,c|jr tjd|j|r|j yy)zC Signal the child process to shut down gracefully. N)rr9rBrHwaitrZrs r6shutdownzTuningProcess.shutdowns2 ::<   tT__ 5  IIK r5cx|jr|jj|jy)z5 Wait for the child process to exit. N)rrwrrxrs r6rzTuningProcess.waits& ::< LL    r5c|jj|jj|jjd|_y)z" Close resources. FN)rnrxrGrHryrs r6rxzTuningProcess.closes;    r5c|jrDtjd|jj|jj |j y)z6 Send a SIGKILL to the child process. z)Sending SIGKILL to autotune subprocess %dN)rrJerrorrwrrrxrs r6rzTuningProcess.killsF ::<  ;     LL    r5cH|jd|jy)z8 Gracefully restarts the child process. TrN)rrYrs r6restartzTuningProcess.restarts 4   r5)rG IO[bytes]rHrreturnNoner<)rTrrHrrDdict[str, str] | Nonerr)rGrrr)rX int | Nonerbool)rrrDrrr)g^@)rfloatrr)T)rrrrrr)r1r2r3__doc__ staticmethodrOrBr=r[rYrrrMrrrxrrr4r5r6r9r9Os<LP '4I  &&+Z< F6 r5r9cJeZdZdZddZed dZddZd dZ d dZ y) TuningProcessPoolz Maintains a pool of TuningProcesses to benchmark kernels in parallel across devices. By default, we create one TuningProcess per device and set the sub-process environment to make only that device visible. cV|j}tjd||Dcgc]}t|c}|_t j |_|jD]}|jj|tt||_ ycc}w)z, Start the child processes. z$Sub-process autotune device list: %srX max_workersN) get_device_listrJrKr9 processesqueueQueue process_queuerr lenexecutor)rZdevicesrXps r6r[zTuningProcessPool.__init__ s&&(CWMFMM6-v6M9> &A    " "1 % & +s7|D NsB&cltjsdgSt}t|}|j }t t jvrNt jt jdDcgc] }t|}}t||ksJ|Stt|Scc}w)zD Gather the list of devices to be used in the pool. N,) r)autotune_multi_devicerr device_countr,r>r?splitintrlistrange)gpu_typedevice_interfacecountdrs r6rz!TuningProcessPool.get_device_list s ++6M>3H= --/ 2:: -')zz2F'G'M'Mc'RS!s1vSGSw<5( ((NE%L!! Ts7B1c|jj|jD]}|jd|jD]}|jy)z5 Signal all child processes to exit. FrN)rrrr)rZrs r6rzTuningProcessPool.shutdown5sQ   #A JJEJ " # A FFH r5cR|jJddg}|Dcic])}|tjvs|tj|+}}|jj }|j |jj | |j tj|jj |Scc}w#t$rBtjd|dtdcY|jj |St$rl}tjd|dt|}d |vsd |vr|jtdcYd}~|jj |Sd}~wwxYw#|jj |wxYw) z Entry point for the thread-pool helper threads: Wait for an open TuningProcess, remove it from the queue, execute the benchmark in that subprocess, and return the TuningProcess to the queue. NTORCHINDUCTOR_CACHE_DIRTRITON_CACHE_DIRrzTimed out benchmarking choice 'z['. It will be ignored. Please debug the root cause in case the choice can bring perf gains.infzFailed to benchmark choice 'cudaErrorLaunchFailurecudaErrorIllegalAddress)bmreqr>r?rrMr benchmarkr)+max_autotune_subproc_result_timeout_secondsrwarningswarnrrArsr)rZchoiceenv_varsvrDrwprocess_exception error_msgs r6targetzTuningProcessPool.target@s ||'''-/AB/7K!1 ?Q 1 %K K$$((* FLL**i @ ,;;BB4    " "7 +=L MM1&:WW  < "    " "7 +! MM.vh7WW  -.I(I5, 9!<     " "7 +!    " "7 +sGCC C-F5F FAF F!F FF F&c xtt||jj|j|}|S)z> Benchmark each choice in a separate process. )dictziprmapr)rZchoicesresultss r6rzTuningProcessPool.benchmarkis/s7DMM$5$5dkk7$KLMr5Nr)rzSequence[int | None])rr'rrrzlist[TritonTemplateCaller]rz!dict[TritonTemplateCaller, float]) r1r2r3rr[rrrrrr4r5r6rrsC E&""( ',R +  + r5rcpeZdZUded<ded<ded<ded<d ed <d Zd ed <e ddZddZy ) TensorMetaz torch.devicerXz torch.dtypedtypeztorch._prims_common.ShapeTypesizesztorch._prims_common.StrideTypestridesroffsetNz str | Nonenamec ddlm}t|tr4|Dcgc]}|j |}}t d|DsJ|S|}t|t jrt jd|}|j}|J|j}|Jt||tjjj|j!tjjj||tjjj#|j%j&|j)Scc}w)Nr)#get_strides_with_layout_constraintsc3<K|]}t|tywr<)rr.0xs r6 z*TensorMeta.from_irnodes..sAQz!Z0Afake)rlayout)rXrrrrr) torch._inductor.select_algorithmrrr from_irnodesallrLayoutBuffer get_dtype get_devicerr+graphsizevarsoptimization_hintsget_sizeoptimization_hint get_layoutrget_name)clsirnodesrrrEnoderrXs r6rzTensorMeta.from_irnodess# Y gx (>E F!1!1!!4 FF FA&AA AAM dBII &99&6D    "!!!''""55dmmoFGG$$773D977##55doo6G6N6NO  !GsE&ct|j|j|j|j|j S)N)rXr extra_size)rrrrXrrrs r6 to_tensorzTensorMeta.to_tensors2 JJ LL;;**{{   r5)rz)LayoutOrBuffer | Sequence[LayoutOrBuffer]rTensorMeta | list[TensorMeta])r torch.Tensor)r1r2r3__annotations__r classmethodrrr4r5r6rr{sP   (( ++ KD* ? &  < r5rcxeZdZdZ d dZ d dZd dZdd d dZdd d dZy) BenchmarkRequesta1 Only handle triton template benchmark for now. The extern kernel benchmark can be done inside the same process since they usually don't cause crash. Important: Instances of this class and subclasses have to be serializable across process boundaries. Do not put CUDA Tensors in here! c ||_t|tr |g|_n||_rEttt fr/t dkDrtfdDsJd|_n|_||_ d|_ y)Nr(c3dK|]'}dD] }td|t||k(")yw))rXrrrrrN)getattr)rrattroutput_tensor_metas r6rz,BenchmarkRequest.__init__..sG Q.q148GAt.s!PA!++-!Prrz0Skipping op due to nonzero workspace requirementrz6InChildProcess %s: load %f, create tensor %f, bench %f)rJ isEnabledForloggingDEBUGtimer r rr rrr/inforrrr*benchmark_gpu_with_cuda_graphrrK) rZrrrKstart_tscreate_tensor_elapser load_elapseres bench_elapses r6rzBenchmarkRequest.benchmarks ++GMM: yy{H ;))d.E.E \ E}%* **!!P9O9O!PPM))335C #'99;#9 yy{H " $%T%%}>#> "iikH4 99;--!??C#dmmB<<<#yy{X5 $$L(     !35 $##$VWU|#,    !3 $2    !s+F "BG )G3G GG G) r rsr rr rr Iterable[Any]rrrrrrrzCallable[[], None]rrrrtorch.Tensor | Nonerr) r1r2r3rr[rrrrr4r5r6rrs//9/: / " /  />"*"1=" "  $( "%"! "  "$(1"$1"!1"  1"r5rcNeZdZdZ d ddZdd ddZy) _TestBenchmarkRequestz Supports unit testing. Defined in this file instead of the test file so the TuningProcess sub-process can unpickle these objects. NcJ||_||_||_||_||_yr<)rErXsleepexccrash)rZrErXr2r3r4s r6r[z_TestBenchmarkRequest.__init__ s'    r5rcr|jr?rMr,rsr2r#r3r4rqexitrErs r6rz_TestBenchmarkRequest.benchmark.sx ;; "::>>"6=T[[AQQ QQ :: JJtzz " 88((N :: HHQK{{r5)NNNF) rErrXrr2z float | Noner3zException | Noner4rr-)r1r2r3rr[rr4r5r6r0r0sq!" $        HL * 1D   r5r0c$eZdZdd ddZy)GPUDeviceBenchmarkMixinNrctdg||D}t|dks Jd|td|Dd}t|}t|dk(rtt |}n|j }|j |5tj||}|jddd|S#1swYSxYw)Nc3K|]i}t|tjrMt|jj r.|jj |jj kywr<)rtorchTensorrrXtypeindexrtensors r6rz3GPUDeviceBenchmarkMixin.do_bench..CsR$ &%,,/v}}))* ##/ MM  $ sA/A1r(zCan not mix devices c3K|]9}t|jjr|jj;ywr<)rrXr>r@s r6rz3GPUDeviceBenchmarkMixin.do_bench..Ls4 &--,,- "" s?Acudar) r!rnextritercurrent_devicerXr*r synchronize) rZrrrdevice_idx_set device_typer device_idxr)s r6rz GPUDeviceBenchmarkMixin.do_bench=s $$ /M/3/$  >"a'P+??O)PP' +    4K@ ~ ! #d>23J)88:J  $ $Z 0 +'';?C  ( ( * +  + s (CC r-r1r2r3rr4r5r6r9r9<s* $( %!   r5r9c$eZdZdd ddZy)CPUDeviceBenchmarkMixinNrc,tj|Sr<)r* benchmark_cpurs r6rz CPUDeviceBenchmarkMixin.do_bench`s ((,,r5r-rKr4r5r6rMrM_s* $( -%-! -  -r5rMceZdZdZ d dfd Z d dZd dZdZd dZxZ S) TritonBenchmarkRequestz Represents a standalone benchmark request for a Triton Template. Important: Instances of this class have to be serializable across process boundaries. Do not put CUDA Tensors in here! ct|||||||_||_||_||_| |_| |_| |_| |_ | |_ ||_ ||_ d|_ yr<)superr[ module_pathmodule_cache_key num_stages num_warpsnum_consumer_groupsnum_buffers_warp_specmatrix_instr_nonkdim waves_per_eukpackworkspace_sizeworkspace_zero_fill_benchmark_module)rZr r r rrTrUrVrWrXrYrZr[r\r]r^ __class__s r6r[zTritonBenchmarkRequest.__init__qsw$ &79KZX& 0$"#6 %:"$8!( ,#6 -1r5ctj|j|jd}||_t j d|j|jt||jj}t|j}|jsddl m}tj |jftj"|j$}|j&r|j)|j+|}|||<d|j,_i} ddl} d| j3|j4vrd| d<|j$j6dk(rd} nP|j$j6} t9| } | j;|j<j$j*} t?t||jtj@jBjDjFr!tIjJ|g|||i| d | iStIjJ|g|||i| | d d S) NFset_sys_modulesz"benchmark module key: %s, path: %sr)WORKSPACE_ARG_PLACEHOLDERrrXwarmupcpustreamT)rh benchmark_run)&rload_by_key_pathrUrTr_rJrKrr runrrr]rrdr<emptyuint8rXr^zero_r?__self__with_bandwidth_infoinspect signature parametersr>rget_raw_streamr r _inductorruntimetriton_heuristicsDebugAutotuner functoolspartial)rZrrmod run_methodrrdworkspace_tensorworkspace_index warmup_argrqrhrIrs r6rz"TritonBenchmarkRequest.make_run_fns=**  ! !   !  "% 0  ! !    S$"2"2377 $//*    * R${{$$&kkzz   '' &&((../HIO*:J '27 /  w((4?? ?#(Jx ::??e #F**//K7 D %44''..44F  C)) * OO # # 5 5 D D $$      $$    " r5c|j}d|_tjj|jd}||}|t ||j d}t |dd}||t |dd}| | |dnr|pt |ddxsdD]$}t t |dddd}||&t |ddxsdD]$}t t |dddd}||&tjD cgc]} t | d d|jk7r|  c} tjddtjj|jdycc} w) Nrelease_benchmark_artifacts _release_static_launchers_except launchersr4rorxcompile_resultskernelri) r_rmodules_no_attrpoprTrr moduleslinemaps) rZr{ cached_modrrrelease_static_launcherslauncherrxcompile_resultmodules r6rz%TritonBenchmarkRequest.cleanup_run_fns$$!% 0044T5E5EtL ;C ?S$"2"2D9F*15t+ '+6+-'.:D( $,3,8(.,4 ' R @ FB H#GHj$$GRVWE( '.f6G&L&RPR N#$?$E(  &--" vz40D4D4DD " A   !1!148 " s#E1c> tj|j|jd}||_t ||j }|j|jdj|_ |jy#|jwxYw)NFrbr) rrjrUrTr_rr  precompilern_regsr)rZr{rs r6rz!TritonBenchmarkRequest.precompile s "..%%  %C &)D "S$"2"23F     **1-44DK    !D   !s A7B BcTd|jd|jd|jS)Nself.kernel_name=z, self.module_path=z, self.module_cache_key=)r rTrUrs r6__str__zTritonBenchmarkRequest.__str__s2#$""$$8t'7'7&99RDz8ExternKernelBenchmarkRequest.benchmark..Ws D-0r5cSr<r4rsr6rz8ExternKernelBenchmarkRequest.benchmark..Zs m8Lr5)numelrrrSrrr<_C_dynamoguardsassert_size_strider sizestridecopy_rr*r%r)rtr)rZrrout_newrr`s ` @r6rz&ExternKernelBenchmarkRequest.benchmarkGs ?syy{a/   3}#5#:7$m== =##%DM*G  ''::U388:.cjjl0C '"--"@@0EE/0LMM((}bA Ar5cyr<r4rs r6rz'ExternKernelBenchmarkRequest.precompile] r5cddlm}t||j}|jr t j |fi|jS|S)Nr)extern_kernels)rrrr rryrz)rZrrs r6rz(ExternKernelBenchmarkRequest.to_callableas@ D ^T%5%5 6 ;;$$R74;;7 7 r5c"d|jdS)NzExternKernelBenchmarkRequest())rrs r6rz$ExternKernelBenchmarkRequest.__str__ms.t/A/A.B!DDr5)NT)r rsr rr rrr+rrsrzdict[str, Any] | Nonerrrrr,)rrrr.rr) r1r2r3rr[rrrrrrrs@r6rr$s)- $ / /9 /: / " /  /& / /  / 9* 91= 9  9RVB,  Er5rc eZdZy)ExternKernelGPUBenchmarkRequestNr0r4r5r6rrq r5rc eZdZy)ExternKernelCPUBenchmarkRequestNr0r4r5r6rrwrr5rcheZdZdZ dfd Z ddZddZd dZxZS) SubgraphBenchmarkRequestz Benchmark request for subgraph choices. Pre-compiles the subgraph in the main process and stores the module path/cache key for loading in subprocess. cTt|||||||_||_||_yr<)rSr[rTrUsym_input_values) rZr r r rrTrUrr`s r6r[z!SubgraphBenchmarkRequest.__init__s2 &79KZX& 0 0r5ctj|j|jd|jfdS)NFrbc.jgSr<)call)rr{rsr6rz6SubgraphBenchmarkRequest.make_run_fn..ssxx C"2 C] CDr5)rrjrUrTr)rZrrr{rs `@@r6rz$SubgraphBenchmarkRequest.make_run_fns>**  ! !   !   00DDr5cyr<r4rs r6rz#SubgraphBenchmarkRequest.precompilerr5c<d|jd|jdS)NzSubgraphBenchmarkRequest(z, r)r rTrs r6rz SubgraphBenchmarkRequest.__str__s&*4+;+;*PPQRRr5)r rsr rr rrr+rTrsrUrsrz list[int]rrr,rr) r1r2r3rr[rrrrrs@r6rr}s 1 19 1: 1 " 1  1 1$ 1  1 E* E1= E  E Sr5rc eZdZy)SubgraphGPUBenchmarkRequestNr0r4r5r6rrr7r5rc eZdZy)SubgraphCPUBenchmarkRequestNr0r4r5r6rrr7r5rceZdZdZ d d fd ZdZ d dZddZdZddZ ddZ dd Z dd Z xZ S)CUTLASSBenchmarkRequestae A class to handle CUDA (CUTLASS) benchmark requests. This class is for managing the lifecycle of a CUDA kernel benchmark, including compiling the source code, managing workspace memory, and executing the kernel. Important: Instances of this class have to be serializable across process boundaries. Do not put CUDA Tensors in here! cLt|||||||_d|_d|_d|_d|_d|_d|_||_ |dk(rtnt|_ t||_|jj|jd\|_|_y)NrFxpuso)rSr[ source_coder] workspaceDLL_workspace_size_updatedhash_key source_filerIrr codecache_clsrrwrite)rZr r r rrrIr`s r6r[z CUTLASSBenchmarkRequest.__init__s &79KZX&#$.2&*',$ "&-8E-A\} 8 E*.*<*<*B*B   d+ ' t'r5ctjd||jj|jdtjd|y)z Precompile the CUDA source code to populate the CUDACodeCache. This may happen in a separate thread pool. Precompiling %srDone precompiling %sN)rJrKrcompilerrs r6rz"CUTLASSBenchmarkRequest.precompilesB .5 ""4#3#3T:3T:r5c  |j|jt||gzDcgc]}t|j }}t j d|j|j|j|j||jt|jj|jj}t|j|j}td}|j dkDrht#j$|j dzdzt"j&|j(|_t|j*j }t-j.|g||jd||} ||Scc}w#t0$r,} t3|  fd} |j5| cYd} ~ Sd} ~ wwxYw)zg Create a function to run the CUDA/XPU kernel with the given input and output tensors. zqmake_run_fn: self.kernel_name=%s, self.source_file=%s, self.hash_key=%s, self.DLL=%s, args=%s, self.extra_args=%srreNctr<) RuntimeError)err_msgsr6raise_runtime_errorz@CUTLASSBenchmarkRequest.make_run_fn..raise_runtime_error s"7++r5)ensure_dll_loadedupdate_workspace_sizerr data_ptrrJrKr rrrrrrtrFrr]r<zerosfloat64rXrryrzrrsr) rZrrrAargs stream_ptrr| workspace_ptrretrFrrs @r6rz#CUTLASSBenchmarkRequest.make_run_fns   ""$:>}:MQTPU:UV*+VV        MM HH  OO   ! ! 0 01F1F1U1U1W X TXXt'7'78      ""[[$$q(Q.mmzzDN %T^^%<%<%>?M   __         ' E [WH '!fG ,    !& & 's# F2)F77 G,!G'!G,'G,c |jry|jttj d|j D}t |dzDcgc] }td}}t|jj|jj}t|j|j}t}|g||jt!|d||jj#|j$|_t(j+d|j&|j|j,|j.|j||jd|_ycc}w)Nc34K|]}|jywr<)r)rmetas r6rz@CUTLASSBenchmarkRequest.update_workspace_size..sG$))Gsr(zupdate_workspace_size called: new workspace size=%d, self.kernel_name=%s, self.source_file=%s, self.hash_key=%s, self.DLL=%s, args=%s, self.extra_args=%sT)rrrrfromkeysr rr rrtrFrrr r rr rGvaluer]rJrKrr)rZunique_input_countrrrr|c_workspace_sizes r6rz-CUTLASSBenchmarkRequest.update_workspace_sizesi  ' '    MMG0F0FG G )..@1.D(EF1FF  ! ! 0 01F1F1U1U1W X TXXt'7'78 #:  __          ))+.44 h          MM HH  OO (,$;GsFc|j9|jj|jd\|_|_|_|jj y)Nr)rrrVrrropenrs r6rz)CUTLASSBenchmarkRequest.ensure_dll_loaded9sJ 88 8<8J8J8O8O  $9 5DHdmT%5  r5cl|j!|jjd|_d|_yr<)rrxrrs r6rz&CUTLASSBenchmarkRequest.cleanup_run_fn@s( 88  HHNN DHr5cTd|jd|jd|jS)Nrz, self.source_file=z, self.hash_key=)r rrrs r6rzCUTLASSBenchmarkRequest.__str__Fs0#$""$$8t'7'7&99JDMM;KLLr5cX|jj}d|d<d|d<d|d<|S)NrrFr)__dict__copyrZstates r6 __getstate__z$CUTLASSBenchmarkRequest.__getstate__Is7 ""$e !k+0'( r5c:|jj|yr<)rr@rs r6 __setstate__z$CUTLASSBenchmarkRequest.__setstate__Ps U#r5)rC)r rsr rr rrr+rrsrIrsrrr,rr)rdict[str, Any])rrrr)r1r2r3rr[rrrrrrrrrrs@r6rrs "  9 :  "       0;6*61=6 6p$,L M$r5rcZeZdZ dfd ZdZ ddZddZxZS)CppBenchmarkRequestcft|||||||_t||_d|_yr<)rSr[rrrr)rZr r r rrr`s r6r[zCppBenchmarkRequest.__init__Xs5 &79KZX& - -1r5ctjd|tj|jdtjd|y)NrrgrIr)rJrKrrVrrs r6rzCppBenchmarkRequest.precompilees< .5$**>3T:r5c\tj|jd|_t ||gzDcgc]}|j }}t jd|j|j||jt|j|j}td|jDsJtjgt|tt |jzz|_t!j"|g||jScc}w)NrgrzJmake_run_fn: self.kernel_name=%s, self.DLL=%s, args=%s, self.extra_args=%sc3PK|]}t|tj ywr<)rctypes c_ulonglong)rargs r6rz2CppBenchmarkRequest.make_run_fn..{sR3:c6#5#56Rs$&)rrVrrrrrJrKr rrrrr rargtypesryrz)rZrrrArr|s r6rzCppBenchmarkRequest.make_run_fnls $$T%5%55I04]0Cse0KLf!LL X    HH  OO  TXXt'7'78 R$//RRRR%112 ID12 2      __  !MsD)c d|jS)Nr)r rs r6rzCppBenchmarkRequest.__str__s#$""$%%r5) r rsr rr rrr+rrsrrr,r)r1r2r3r[rrrrrs@r6rrTsi 2 29 2: 2 " 2  2  2; * 1=  6&r5rcPeZdZdZ dfd Z ddZxZS)CuteDSLBenchmarkRequestz;Benchmark request for CuteDSL (CUTLASS Python DSL) kernels.ct||||||j}tj|\|_|_yr<)rSr[ finalize_allrrrUrT)rZr r r rrfinalized_coder`s r6r[z CuteDSLBenchmarkRequest.__init__sC &79KZX$1132=2C2CN2S/t/r5c X tj|j|jd}ddlm}|j d|}t||s?t|Dcgc]}tt||s|}}td|d|t||  fd}|Scc}w) z Create a function to run the CuteDSL kernel with the given input and output tensors. Similar to TritonBenchmarkRequest.make_run_fn but for CuteDSL kernels. Frbr() MAIN_SUFFIXrz-Could not find CuteDSL main kernel function 'z'. Available callables: c~td}|jjj}gd|iS)NrCrh)rrtrXr?)rrhr kernel_funcrs r6 run_kernelz7CuteDSLBenchmarkRequest.make_run_fn..run_kernels@7? %44SZZ5E5EFFB BsB6B Br5) rrjrUrTcodegen.cutedsl.cutedsl_kernelrr hasattrdircallablerr) rZrrr{rmain_func_namer availablerrs `` @r6rz#CuteDSLBenchmarkRequest.make_run_fns**  ! !   !  @ ,,-Q{m<sN+*-c(S$hwsD?Q6RSIS??OOghqgrs c>2  C Ts B';B') r rsr rr rrztuple[Any, ...]rr&rrr,)r1r2r3rr[rrrs@r6rrsiE T T9 T: T $ T # T  T*1= r5rcXt}tj|j|Sr<)ratexitror)pools r6get_tuning_process_poolr s  D OODMM" Kr5c4tj|S)zO Do benchmarking in a subprocess and return the perf number (latency). )r r)rs r6benchmark_in_sub_processr"s # $ . .w 77r5ceZdZUdZdZded<ejZded<dZ ded <d Z e d Z e d Zdd ZddZddZdZddZddZddZdZe dZy)AutotuneProcessPoolzr Singleton pool manager for running autotuning (precompilation + benchmarking) in a separate process. NzAutotuneProcessPool | None _instancezthreading.Lock_lockFr_shutdown_for_inactivityct|j|_d|_d|_|j |_yr<) _init_pool_pool_warmup_future_warmup_start_time _init_timer_timerrs r6r[zAutotuneProcessPool.__init__s0151B 2604$($4$4$6 r5c|j9|j5|j ||_ddd|jS|jS#1swY|jSxYw)z*Get or create the singleton pool instance.N)r%r&rs r6 get_instancez AutotuneProcessPool.get_instancesY ==  *==($'ECM *}}s}}  *}}s AA&ctjsJd|j*|j|_|j |_|jS)zGet the process pool.zFTo use AutotuneProcessPool, pipeline_max_autotune_gemm must be enabled)r)pipeline_max_autotune_gemmr*r)r-r.rs r6rzAutotuneProcessPool.poolsQ00 T 0 :: *DJ**,DKzzr5cJtdkDrtt|jSy)Nr) AUTOTUNE_POOL_INACTIVITY_TIMEOUTr_on_inactivity_timeoutrs r6r-zAutotuneProcessPool._init_timers! +a /94;V;VW Wr5cR|j|jjyyr<)r. record_callrs r6_record_activityz$AutotuneProcessPool._record_activitys! ;; " KK # # % #r5ctjdt|j5|j#|jj dd|_d|_dt_dddy#1swYyxYw)NzAAutotuneProcessPool shutting down due to inactivity (timeout=%ds)FrT) rJr$r5r&r*rr.r$r'rs r6r6z*AutotuneProcessPool._on_inactivity_timeoutsq O , ZZ @zz% ###/! DK <@  8 @ @ @s AA22A;ctjd}td|}tj|j t jd|S)a Get or create the process pool. Uses ProcessPoolExecutor with 'spawn' context for CUDA safety. ProcessPoolExecutor is lazily initialized - workers are not spawned until the first submit() call, making this property non-blocking. spawnr()r mp_contextz2AutotuneProcessPool created (workers spawn lazily))mp get_contextrrro _shutdownrJr$)rZctxrs r6r)zAutotuneProcessPool._init_pool sHnnW%"  'PQ r5c|j|j5|jtj|_|j j ttjjjj|_|jj|jtj!dddd|jS|jS#1swY|jSxYw)z Submit a warmup job to eagerly spawn workers and initialize CUDA. This is optional - call it early to hide spawn latency. Returns the warmup future which can be ignored or awaited. N)fp32_precisionzWarmup job submitted)r+r&r# perf_counterr,rsubmit_init_autotune_subprocessr<backendsrCmatmulrCadd_done_callback_on_warmup_completerJr$rs r6warm_upzAutotuneProcessPool.warm_up s    & @&&..2.?.?.AD+*.))*:*:1',~~':':'A'A'P'P+;+D'''99$:R:RS"''(>? @"""t""" @"""s B2C++C?c$d}|j!tj|jz } |j}tj d|||j y#t$r}tjd||d}~wwxYw)z/Callback invoked when the warmup job completes.NzEAutotuneProcessPool warmup completed successfully in %.4f seconds: %sz4AutotuneProcessPool warmup failed after %.4f seconds) r,r#rDrErJr$r9rAr)rZfuturewarmup_elapsed_timerErFs r6rJz'AutotuneProcessPool._on_warmup_complete4s"  " " ."&"3"3"58O8O"O  ]]_F   W#   ! ! #   F# G  s7A)) B2B  Bcjj|g|i|}j|jfd|S)z-Submit a job to the pool and return a Future.c$jSr<)r9)rrZs r6rz,AutotuneProcessPool.submit..Mst/D/D/Fr5)rrEr.rI)rZrrrrMs` r6rEzAutotuneProcessPool.submitIsA!!!"6t6v6 ;; "  $ $%F G r5c|j!|jjd|_|j$|jjdd|_yy)zShutdown the pool on exit.NFr)r.quitr*rrs r6r@zAutotuneProcessPool._shutdownPsN ;; " KK   DK :: ! JJ  U  +DJ "r5c|jC|j5|j!|jjd|_dddyy#1swYyxYw)z+Explicitly shutdown the singleton instance.Nr%r&r@r0s r6shutdown_instancez%AutotuneProcessPool.shutdown_instanceYsU == $ )==,MM++-$(CM ) ) % ) )s .AA)rz Timer | Noner)r Future[Any])rMrVrr)r1r2r3rr%r threadingLockr&r'r[rr1propertyrr-r9r6r)rKrJrEr@rUr4r5r6r$r$s -1I)0*INN,E>,%*d*7  &@"(#(*))r5r$cHtjxrtj Sr<)r)r3r$r'r4r5r6use_pipelined_autotuningr[cs!)) =#<< <r5cddl}|jjr|jdd||jjj _y)z9 Warmup function run in the autotune subprocess. rNr(rCrT)r<rC is_availablerrGrHrC)rCr<s r6rFrFjsD zz  Af%0>ENN- r5c |j}|S#t$r&tjd|dt dcYSwxYw)a* Run autotuning benchmarks in a subprocess. This function is submitted to AutotuneProcessPool and runs in isolation to prevent GPU contention with the main compilation process. Args: picklable_choices: List of picklable choice information Returns: timing zFailed to benchmark choice %sT)exc_infor)rrArJwarningr)benchmark_requesttimings r6run_autotune_in_subprocessrcysQ ",,.  +   U|s,AAceZdZUdZdZded<ejZd d dZ e d dZ dZ d ddZ e dd Zy)PrecompileThreadPoolz Thread pool for running precompilation asynchronously. This allows the main compilation process to continue while precompilation happens in background threads. NzPrecompileThreadPool | Noner%c&t||_y)Nr)r _executor)rZrs r6r[zPrecompileThreadPool.__init__s+ Dr5cddlm}|j?|j5|j|||_ddd|jS|jS#1swY|jSxYw)Nr)get_num_workers)rrir%r&)rris r6r1z!PrecompileThreadPool.get_instances`D ==  ;==($'(9$:CM ;}}s}} ;}}s AA2ctj}tj|j|}|j j |g|i|Sr<) contextvars copy_contextryrzrkrgrE)rZrrrrAs r6rEzPrecompileThreadPool.submitsG&&(   sww +$t~~$$R9$9&99r5c:|jj|S)Nr)rgrrs r6r@zPrecompileThreadPool._shutdowns~~&&D&11r5c|jE|j5|j#|jjdd|_dddyy#1swYyxYw)NFrrTr0s r6rUz&PrecompileThreadPool.shutdown_instancesX == $ )==,MM+++7$(CM ) ) % ) )s 0AA))rr)rre)F)rrr)r1r2r3rr%rrWrXr&r[rr1rEr@rUr4r5r6reresZ.2I*1 INN EE: 2))r5recVeZdZdZiZeddZeddZe ddZ y) AsyncAutotunera Handles asynchronous autotuning of kernel choices in a separate process. This class manages the lifecycle of autotuning: 1. Accepts precompiled choices from the main process 2. Submits benchmarking work to AutotuneProcessPool 3. Returns results via a Future Usage: autotuner = AsyncAutotuner(choices) autotuner.start() # Kicks off async benchmarking timings = autotuner.get_results() # Blocks until complete c(|j|zSr<)r)r inputs_keys r6get_choice_hashzAsyncAutotuner.get_choice_hashs :--r5c|D]}tj||}|tjvr,t|ddJdtj j t|j}|tj|<y)z Start asynchronous autotuning in a subprocess. This method: 1. Extracts picklable benchmark requests from choices 2. Submits benchmarking work to AutotuneProcessPool 3. Returns immediately (non-blocking) rNzbmreq is None for choice) rqrtchoice_hash_to_futurerr$r1rErcr)rrrsr choice_hashautotune_futures r6rYzAsyncAutotuner.starts PF(88LKnBBB67D1= * =2>>@GG* O APN 0 0 = Pr5ci}|D]<}tj||}tj|j||<>|S)z Get autotuning results, blocking until complete. Args: timeout: Maximum time to wait in seconds. None means wait forever. Returns: Dict mapping ChoiceCaller to benchmark timing )rqrtrvrE)rrrstimingsrrws r6 get_resultszAsyncAutotuner.get_resultssP YF(88LK,BB;OVVXGFO Yr5N)rr%rsrsrrs)rlist[ChoiceCaller]rsrs)rr|rsrsrzdict[ChoiceCaller, float]) r1r2r3rrvrrtrrYr{r4r5r6rqrqsc ..PP6(69 "r5rq)rrrr)rCrsrr)rarrr)s __future__rrrkr dataclassesryr!multiprocessingr>r>rQrrlrurqrWr#rcollections.abcrrrconcurrent.futuresrrr r r r r typingrrrr<torch._inductor.async_compiletorch._dynamo.device_interfacertorch._dynamo.testingrtorch._inductorrtorch._inductor.codecacherrrrrr$torch._inductor.compile_worker.timerrtorch._inductor.utilsrrrrrtorch._loggingr torch.utils._ordered_setr!rr?rMr5typesr$rr%r&r'rr)runtime.benchmarkingr* virtualizedr+r,r1rJrAr/r9rrrLayoutOrBuffer dataclassrrr0r9rMrQrrrrrrrrrrrcacher r"r$r[rFrcrerqr4r5r6rs"     88NN22)) $C.7-/ $'JJNNCUK$   -."8\:  ttnoodRYY& . . . b i"i"i"X,D  F--pU-pUf 79O  79O JE#3JEZ 9  9 (S/(SV "9;S  "9;S a$57Ga$H4&13C4&n/57G/d  8 '8&8S)S)l ' @')')TCCr5