^j UddlZddlZddlZddlZddlmZmZddlmZmZddl m Z ddl m Z ddl mZmZddlmZmZmZddlZddlmcmZddlmcmZddlmZdd lm Z dd l!m"Z"dd l#m$Z$ejJjMe'd Z(d Z)dZ*dZ+edZ,edZ-de.fdZ/de.ej`zdzde.fdZ1egejddfZ3da4e3dze5d<de3dzde3dzfdZ6ejndedfdZ8dee,e-fdee,e-ffdZ9iZ:e;e.edeffe5d<ddde.dedefd ed:d#Z?deeee,fe-fdeeee,fe-ffd$Z@Gd%d&ZAd'ZBd(ZCd)ZDe=d*eBd+e=d,eCd+e=d-eDd+d.ed/ed0eEd1edeFf d2ZGGd3d4eAZHGd5d6eHZIGd7d8eIZJdeAfd9ZKeKZLy);N)CallableIterator)cached_propertywraps)chain)median)Any Concatenate) ParamSpecSelfTypeVar)get_interface_for_device)counters) DebugMode) OrderedSet benchmarking)cudaxpumtia _CALLABLEiPTreturnctDcgc]"}tt|jr|$}}t |dksJt |dk(rdS|j Scc}w)Nrr)GPU_BENCHMARK_DEVICE_TYPESgetattrtorch is_availablelenpop) device_type avail_gpuss o/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/runtime/benchmarking.py_get_default_gpu_device_typer% sj6  5+ & 3 3 5 J z?a  _)6?z~~/?? s'A"r"c| tSt|tjr |jStj|jSN)r% isinstancerdevicetyper"s r$_normalize_gpu_device_typer,*s@+--+u||, << $ ) ))_gpu_benchmark_lock_contextcontext_factoryct}|a|S)agOverride the process-local GPU benchmark lock context. This lets benchmark harnesses provide the context used by Inductor GPU benchmark calls. Some benchmark helpers delegate to other benchmark methods, so harness contexts should support nested entry from the same thread. Returning the previous context lets callers restore it in tests. r.)r/previouss r$set_gpu_benchmark_lock_contextr36s+H"1 Or-c#fKt}|dy|5ddddy#1swYyxYww)z;Optionally enter the registered GPU benchmark lock context.Nr1)r/s r$maybe_gpu_benchmark_lockr5Fs62O    s1% 1.1fnc|tdtjdtjdtffd }|S)NargskwargsrcTt5|i|cdddS#1swYyxYwr')r5)r8r9r6s r$wrapperz#gpu_benchmark_lock..wrapperRs+ % ' 't&v& ' ' 's')rrr8r9rr6r;s` r$gpu_benchmark_lockr=Qs= 2Y'qvv''a'' Nr-._BENCHMARK_DISPATCHF)overrider?ct|tr|s tdt|s t d|s|t vrtd|d|t |<y)a Register a device-type specific benchmarker. Args: device_type: torch.device.type string (e.g., "cuda", "cpu", "mps", "xpu"). fn: callable(self, _callable, *, warmup, rep, **kwargs) -> Any override: allow overriding an existing registration. zAdevice_type must be a non-empty string matching torch.device.typezfn must be callablezBenchmarker for device_type 'z' already registeredN)r(str ValueErrorcallable TypeErrorr>)r"r6r?s r$register_benchmarkerrEasg k3 '{ O   B<-..  '::+K=8L M  (* $r-c ddlmjjdk(rSdtt t t dfzt zdtt t t dfzt zffd tjdttd tttfdtt t t dfzt zffd }|S) Nr)configms.rct|ttfrt|fd|DSjj }t|t sJ|dk(r |rd|z SdS|dk(rddl}|jStd|)Nc3.K|] }|ywr').0valdistorts r$ zCmay_distort_benchmarking_result..distort..s7SGCL7sinverseg?randomrzUnrecognized distort method ) r(listtupler* test_configsdistort_benchmarking_resultfloatrS RuntimeError)rIdistort_methodrSrGrOs r$rOz0may_distort_benchmarking_result..distorts b4- (487B77 7,,HH"e$$$ Y &!38 *s * x ' ==? "!=n=MNO Or-r8r9c$|i|}|Sr'rL)r8r9rIrOr6s r$r;z0may_distort_benchmarking_result..wrappers  r{r-) torch._inductorrGrVrWrTrXrU functoolsrr dictrA)r6r;rGrOs` @@r$may_distort_benchmarking_resultr_|s& 66"< P K%s + +e 3P euUCZ( (5 0P"__RCy$(cN euUCZ( (5 0 Nr-cbtjjjr t dy)NaIn the deterministic mode of Inductor, we will avoid those benchmarkings that would cause non-deterministic results. Only benchmarkings in the vetted scenarios are allowed. Examples include autotuning for triton configs of pointwise kernels. When you see this exception, you can do one of the following two things: 1. if the benchmarking you are doing does not introduce any non-determinism, you can just add is_vetted_benchmarking=True to your benchmark_gpu call. That would solve the issue. 2. if the benchmarking you are doing indeed introduces non-determinism, you'll need to disable such feature in deterministic mode or find an alternative implementation that is deterministic. )r _inductorrG deterministicrYrLr-r$may_ban_benchmarkingrcs, ++   ,r-c tdtdtjdtjdt ffd }|S)am Wraps `fn` to increment the appropriate dynamo counters. It is expected that `fn` is a method of `Benchmarker` or one of its subclasses; typing limitations prevent us from declaring this directly. NOTE: If you're tempted to add a dynamo_timed call here, this function can be called enough that the dynamo_timed overhead is not negligible. selfr8r9rc|jjdj}tdd|xxdz cc<|g|i|S)N.inductorz benchmarking.r) __class____name__r)rer8r9 fn_qual_namer6s r$r;ztime_and_count..wrappersS..112!BKK=A }\N;<A<$((((r-)rr rr8r9rr<s` r$time_and_countrlsF 2Y)c)!&&)AHH))) Nr-cTeZdZdZdeddfdZdededejfdZ e dded e d efde ed fdzde eefdzd eejzdzd edefd Ze ddede gefdededef dZe deded edefdZe edede gefd edefdZy) Benchmarkerzo A device-agnostic benchmarking utility for measuring the runtime of inductor generated callables. rerNcyr'rLres r$__init__zBenchmarker.__init__s r-fn_args fn_kwargscd}t||jD]`}tj|D]F}t |t j s| |j}-|j|k7s=tdb| td|S)NzcCan't safely infer the device type of `fn` with multiple device types in `fn_args` and `fn_kwargs`!zCan't safely infer the device type of `fn` with no device types in `fn_args` or `fn_kwargs`. Use a direct benchmarking method instead e.g. `Benchmarker.benchmark_cpu` or `Benchmarker.benchmark_gpu`.) rvaluespytree tree_leavesr(rTensorr)rB)rerrrsinferred_device arg_or_kwargarg_or_kwarg_leafs r$ infer_devicezBenchmarker.infer_devices/3!'9+;+;+=> L&,%7%7 %E !!"3U\\B"*&7&>&>O&--@$}     "N  r-r6.r)r9c | d}|(t|trtj|n|}n5| | t d|xs t }|xsi}|j |i|}t|tjsJ|xs t }|xsi}t|dk(rt|dk(r dtffd }n| | dtf fd }|jdtj}|jdtj} tj5tj!|j"} | | ||f|| d |cdddS|tjd k(r|j$|f|| d |cdddS|j&|f|| d |cdddS#1swYyxYw) a7Benchmark `fn(*fn_args, *fn_kwargs)` and return the runtime, in milliseconds (the actual runtime calculation is dictated by the benchmarking implementation, but may be one of [mean, median, minimum, etc.]). Functions as a convenience wrapper around device-specific implementations, like `benchmark_cpu` and `benchmark_gpu`. Raises `ValueError(...)` if we can't safely infer the device type of `fn`; for example, if multiple device types are found in `fn_args` and `fn_kwargs`, or if no device types are found. To bypass device inference, provide the device to the `device` parameter. WARNING: if `fn` mutates `fn_args` or `fn_kwargs`, benchmarking may fail unexpectedly. For example, if `fn` clears a mutable object, subsequent invocations of `fn` during benchmarking will fail. In such cases, `fn` should handle cloning its arguments internally. If device inference is required, `Benchmarker.infer_device` can be used prior to calling this method without any arguments for `fn_args` and `fn_kwargs`. Arguments: - fn: The function to benchmark. - fn_args: The function's arguments. - fn_kwargs: The function's kwargs. Keyword Arguments: - device: Which device to use for benchmarking. If not provided the device will be attempted to be inferred from `fn_args` and `fn_kwargs`. - **kwargs: The benchmarking implementation's kwargs. Returns: - The runtime of `fn(*fn_args, **fn_kwargs)`, in milliseconds. NzJ`fn_args` and `fn_kwargs` cannot both be None if `device` is not provided.rrcSr'rL)r6sr$ _callablez(Benchmarker.benchmark.._callables t r-ciSr'rL)_args_kwargsr6sr$rz(Benchmarker.benchmark.._callable%s5,G,,r-warmupreprrcpu)r(rArr)rBrUr|r r r!inductor_config inductor_default_autotune_warmupinductor_default_autotune_repr_benchmarking_inductorr>getr* benchmark_cpu benchmark_gpu) rer6rrrsr)r9ryrrr benchmark_fnrrs ` @@r$ benchmarkzBenchmarker.benchmarksJ04  (263(? V$V 9#4 `(G!RI/d//FIFO/5<<888$UWO  w<1 Y1!4 s EG -s -Ho&V&VWjj M MN - - / S6I6M6M$$7L'#D)VFVvV S S%,,u"55)t)))VFVvV S S&4%%iRCR6R S S Ss$.F2-F2F22F;rrrchdtdttffd }||t||S)aBenchmark the CPU callable, `_callable`, and return the median runtime, in milliseconds. Arguments: - _callable: The CPU callable to benchmark. Keyword Arguments: - warmup: Optionally, the duration, in milliseconds, to run `_callable` before benchmarking starts. - rep: Optionally, the duration, in milliseconds, to run `_callable` during benchmarking. Returns: - The median runtime of `_callable`, in milliseconds. rIrcg}tj} tj}tj}|j||z tz||z tz|kDr |S]r')time perf_counterappendMILLISECONDS_PER_SECOND)rItimings run_start_tstart_tend_trs r$run_forz*Benchmarker.benchmark_cpu..run_forOspG++-K++- ))+3JJK[(,CCrINr-)intrTrXr)rerrrrs ` r$rzBenchmarker.benchmark_cpu;s2(  U  gcl##r-r8ctr')NotImplementedError)rer8r9s r$rzBenchmarker.benchmark_gpu^s!!r-c x|tjjtjj}tjj |d5|dddtjj|j |j fi|S#1swYDxYw)a Benchmark a GPU callable using CUDA graph capture and replay. This captures the callable into a CUDA graph and benchmarks the graph replay, which eliminates kernel launch overhead for fair comparison between different implementations. thread_local)capture_error_modeN)rr synchronize CUDAGraphgraphrreplay)rerr9 cuda_graphs r$benchmark_gpu_with_cuda_graphz)Benchmarker.benchmark_gpu_with_cuda_graphbs    ZZ))+ ZZ  j^  L  K   !t!!*"3"3>v>>   s %B00B9)NNN)d)rj __module__ __qualname____doc__r rqr rr)r|rlrrUr^rArXrrrrr=rrLr-r$rnrns  t  Ssu||0+/+/,0 USUS S#X USsCx4'USS>D( US ell"T) US  US USUSnOR $ $'C0 $:= $IL $  $ $D"D"""""??BG$??  ??r-rnc .|j|f||d|S)Nr)rrefrrkws r$_default_cpu_benchr}s! 4  a >C >2 >>r-c R|jdd|j|f||d|S)Nr"rr setdefaultrrs r$_default_cuda_benchrs/MM-( 4  a >C >2 >>r-c R|jdd|j|f||d|S)Nr"rrrrs r$_default_xpu_benchrs/MM-' 4  a >C >2 >>r-rTrr kineto_eventsprofiler_eventsrprofiler_device_typec Zddlmtdtddffd |Dcgc]0}|jt k(r|j jk(r|2}}t||k7r"td|dt dt|d |D] }| g}|D]g}|j t k(s|j jk(s9|j|j|jfi|r{|D]v}|j jk7r!|j}|j} |D]1\} } || k\s | | ksj|jvxd } |D]}|j} |j}|j!}|j |k(sG|d k7sM| vs | dk(sW|vs\|j d k7sp| |j|jz d z z } | Scc}w)Nr DeviceTypeeventrc|jjk7ryj|j|jD] }| yr')r"CPUaddid cpu_children)rchildrbenchmark_event_idscollect_cpu_event_idss r$rzB_get_callable_device_kernel_time_us..collect_cpu_event_idssG     . )'' )E !% ( )r-z Expected  z profiling events. Found z events.rRgpu_user_annotationz Context Sync@@)torch.autogradrrr name_CALLABLE_PROFILE_EVENT_NAMEr"rr rYrstart_nsend_nsrcorrelation_idlinked_correlation_id activity_type)rrrrrbenchmark_eventscallable_kineto_windowsevev_startev_end win_startwin_enddevice_time_usrrrrrrs @@@r$#_get_callable_device_kernel_time_usrsL *+5<)S)T)%  ::5 5    /   #uA:;<)*+8 5  "%e$%68I GGI5 5 JNN2 # * *BKKM299;+G H I B~~:>>1{{}HYY[F&= " 7y(Vw->'++B,=,=,?@   NK % ; ; =--/++-    #7 7!66%)<<)Q.>EX3X . u||~0@@FJ JNK" ss5H(cxeZdZedededeffdZee e d dedegefde dede f dZ y ) TritonBenchmarkerrer.cN ddlm}|S#t$r}td|d}~wwxYw)z"Lazily import Triton's `do_bench`.r)do_benchzrequires TritonN)triton.testingr ImportErrorr)reres r$triton_do_benchz!TritonBenchmarker.triton_do_benchs4 @ / @%&78a ? @s $ $ris_vetted_benchmarkingr9c |s ttj|jj}t |j D] }||vs||=  d|vr|j|fi|dSd|vr|j|fi|S|j|fi|ddiS#t$rI}t|j}d|vr&tjd|tdcYd}~Sd}~wwxYw) aBenchmark the GPU callable, `_callable`, and return the runtime, in milliseconds. Arguments: - _callable: The GPU callable to benchmark. Keyword Arguments: - quantiles: Optionally, a tuple of floats denoting the requested quantiles. - return_mode: Optionally, the requested return mode. Currently, Triton's `do_bench` supports min, max, mean, and median return modes. - **kwargs: Additional kwargs passed to Triton's `do_bench`. Returns: - The runtime of `callable`, in milliseconds. If `kwargs["quantiles"]` is specified, this is the first requested quantile. Else, if `kwargs["return_mode"]` is specified, this is the requested return mode. Otherwise, this is the median. quantilesr return_moderzinvalid configurationz9Skipping benchmark due to invalid configuration error: %sinfN) rcinspect signaturer parametersrTkeys ExceptionrAlowerloggerwarningrX)rerrr9do_bench_paramskwargr error_strs r$rzTritonBenchmarker.benchmark_gpus4& "!++D,@,@ALL&++-( "EO+5M " f$+t++I@@CC&(+t++I@@@'4'' RVRR R A I&)3OU|#  s0B%7B%B%% C7.=C2+C71C22C7N)F)rjrrrr rr rr_rlr=boolrXrrLr-r$rrsdxS'9% (---BG$-!%- -  -%-r-rceZdZededefdZ ddedeejzdzdefdZ ddededeejzdzde e e e ffdZded e e e e fdefd Zeee dded ege fd ed edededede ej*dzdedeejzdzde dee ezfdZy)InductorBenchmarkerrerc"|jS)z7Get the L2 cache size, in bytes, of the current device.)get_device_cache_sizerps r$ L2_cache_sizez!InductorBenchmarker.L2_cache_sizes))++r-Nr"cd|jvr|jdSt|}t|}|j}|j |}dD]}t ||d}|s|cSy)z>Get the L2/global cache size, in bytes, of the current device.r)rlast_level_cache_sizeN)__dict__r,rcurrent_deviceget_device_propertiesr)rer"device_interfacer)propsattr cache_sizes r$rz)InductorBenchmarker.get_device_cache_size"s} dmm +==1 10= 3K@!002 66v>> "D d3J!! "!r-itersctt|}t|Dcgc]&}|jd|jdf(c}Scc}w)z#Get `iters` pairs of device events.T) enable_timing)rr,rangeEvent)rerr"r_s r$get_event_pairsz#InductorBenchmarker.get_event_pairs3sc4 &{ 3 5\  !&&T&: &&T&:    s+A event_pairsc ht|Dcgc]\}}|j|c}}Scc}}w)zDGet the minimum timing, in milliseconds, for a group of event pairs.)min elapsed_time)rer  start_event end_events r$get_event_pairs_min_timingz.InductorBenchmarker.get_event_pairs_min_timingBs=/: *K((3    s. restimation_itersmemory_warmup_itersbenchmark_itersmax_benchmark_durationr grad_to_nonerr9c h|s tt| } t| } | j|| jt j |j | dztj| } | j|j|| } | D]L\}}||D] }d|_ | j|j||jN| j|j| }|dkDr"tt|t||zd}t|D]}| j|j|| } | D]L\}}||D] }d|_ | j|j||jN| j~ |dk(r%| Dcgc]\}}|j!|}}}|S|dk(r|j| }t||St#d |d cc}}w) aBenchmark a GPU callable using a custom benchmarking implementation. Arguments: - _callable: The callable to benchmark. Keyword Arguments: - estimation_iters: Optionally, the number of iterations to run `_callable` during runtime estimation. - memory_warmup_iters: Optionally, the number of iterations to flush the L2 cache before starting benchmarking. - benchmark_iters: Optionally, the number of iterations to run `_callable` during the benchmarking. - max_benchmark_duration: Optionally, the maximum duration of the benchmarking, in milliseconds. An estimated duration is calculated based on the values of `memory_warmup_iters` and `benchmark_iters`, along with the estimated runtime of `_callable` and various other factors, and we then shrink `benchmark_iters` to fit in the allotted maximum duration. - return_mode: Return mode for benchmark results. Options are "min" (default), "all" (returns all measurements). - grad_to_none: Optionally, a list of tensors whose gradients should be cleared before each benchmark iteration. - is_vetted_benchmarking: in deterministic mode, we only allow benchmarking in vetted cases. - **kwargs: Additional kwargs that may be passed to the fallback. Returns: - If return_mode="min": The minimum runtime of `_callable`, in milliseconds. - If return_mode="all": List of all runtime measurements, in milliseconds. dtyper)r+Nrrallr Unsupported return_mode: z. Use 'min' or 'all'.)rcr,rrremptyrrzero_rgradrecordrmaxr rr rB)rerrrrrrrrr"r9rbufferr r rxestimated_timingr all_timingsbenchmarked_timings r$rz!InductorBenchmarker.benchmark_gpuMs_\& "0= 3K@ $$&  $$&  & &{ 3q 8))   **+;*U &1  "K'%"A!AF" LLN    K      $$&::;G a !OS)?CS)S%TUWXO *+ A LLN **? *T &1  "K'%"A!AF" LLN    K      $$&  % /:*K((3K  E !!%!@!@!M ');< <+K=8MN sH.r')rrr NFN)rjrrrr rrrArr)rrTrUr rrXrr_rlr=rrxrrrLr-r$rrs,D,S,, >B!!!$u||!3d!:! !$JN      -05<<-?$-F  eCHo       !%eCHo!6    %!"#&"&( 26',15yyBG$yy! y  y !$ yy5<<(4/y!%y5<<'$.yy e y%yr-rceZdZdZee ddedegefde de de de d e d e d e e jdzd e e jzdzd edefdZy)TorchProfilerBenchmarkerzABenchmarker that uses torch.profiler for GPU kernel benchmarking.Nrerrrrrrrrr"r9rc t| } t| } | j|| jtjj rd} n|j | } tj| dztj| } | j|j|| }|D]L\}}||D] }d|_ | j|j||jN| j|j|}|dkDr"tt|t||z d}t!|D]}| j| j#}t%tj&j(|}tj&j+tj&j(j,|gd 5}t!|D]Q}||D] }d|_ | jtj&j/d 5|dddS ddd| jdd lm}t%||}d }j&j4j7D]U}|j9t:k(s|j=|k(s/||j?|jAz d z z }W|dkr>tC|j&j4j7|j7||}|dkrtEd|d|}||z d z }~ |dvr|StGd|d#1swY~xYw#1swY.xYw)aBenchmark a GPU callable using torch.profiler. Arguments: - _callable: The callable to benchmark. Keyword Arguments: - warmup: Ignored (kept for API compat). Warmup is handled by the estimation phase which runs estimation_iters with cache flushing. - rep: Optionally, the maximum number of iterations to run during benchmarking. - estimation_iters: Optionally, the number of iterations used to estimate the runtime of `_callable` for dynamic rep adjustment. These iterations also serve as warmup for the callable. - memory_warmup_iters: Optionally, the number of buffer.zero_() iterations to run after estimation to bring the cache into a steady state before the profiled benchmark phase. - max_benchmark_duration: Optionally, the maximum duration of the profiled benchmark phase, in milliseconds. The rep count is reduced if the estimated total would exceed this budget. - return_mode: Return mode for benchmark results. Options are "min", "mean" (default), or "max". - grad_to_none: Optionally, a list of tensors whose gradients should be cleared before each benchmark iteration. - **kwargs: Additional kwargs that may be passed to the fallback. Returns: - The runtime of `_callable` in milliseconds, computed according to return_mode. rrrr+NrrF) activities record_shapesrrrRrz&TorchProfilerBenchmarker: '_CALLABLE' zS event not found in raw kineto results, and no correlated device events were found.)r meanrrz. Use 'min', 'mean', or 'max'.)$r,rrrversionhiprrrrrrrrrr rupperrprofilerProfilerActivityprofilerrecord_functionrrkineto_resultseventsrrr"rrrAssertionErrorrB)rerrrrrrrrr"r9rbuffer_size_bytesr r r rr! estimated_msrdevice_type_upperprofile_activityprof _DeviceTypercallable_gpu_time_us kineto_event total_time_us avg_time_mss r$rz&TorchProfilerBenchmarker.benchmark_gpusT1= 3K@ $$&  $$& ==   1  $ : :; G   "%))K    **+;*U &1  "K'%"A!AF" LLN    K      $$&66{C ! c#s#9L#HIJANC*+ A LLN  (--/"5>>#B#BDUV ^^ # #//33  $  3Z +)&!%& ^^33K@ K    $$& =&{4EF" MM88??A L!!#'CC ,,.2FF$ '')L,A,A,CC)$   1 $#F ,,335 $ $  1 $ 89J8KLRR  - $s*f4   0 0 +K=8VW e    s%*AM78M* M7*M4 /M77N)r&rr% r&r,NN)rjrrrrlr=r rr rrArTrrxr)rXrrLr-r$r(r(sK !#%&(!2615ZZBG$ZZ Z  Z ! Z!$ZZ5<<(4/Z5<<'$.ZZ ZZr-r(c~tjr tStjr t St Sr')ruse_torch_profiler_benchmarkerr(use_experimental_benchmarkerrrrLr-r$_make_default_benchmarkerrEns/55'))33"$$  r-)rN)M contextlibr]rrcollections.abcrrrr itertoolsr statisticsrtypingr r typing_extensionsr r r rtorch._inductor.configrarGrtorch.utils._pytreeutils_pytreervtorch._dynamo.device_interfacertorch._dynamo.utilsrtorch.utils._debug_modertorch.utils._ordered_setr_logginggetArtifactLoggerrjrrrrrrrAr%r)r,AbstractContextManager_GpuBenchmarkLockContextr.__annotations__r3contextmanagerr5r=r>r^rrEr_rcrlrnrrrrrXrrrr(rE benchmarkerrLr-r$r[s .,#66 00$$C(-/  ) )(N C4* cN CL@c@*C%,,,>,E*#*$B (I(I$(O$OP?C5<C -4 $   (4.8AqD>hq!tn68T#xS1127 **c* *  *6c(:xS?QD  S!V$a'( k#q&!1$%*w?w?v?? ? U.>V04@U.>KKK K K  K\; ;|m+m`_2_D;() r-