^jC jUddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZmZddlmZddlmZmZmZmZmZmZddlmZmZddlZddlZddl Zddl!m"Z"dd l#m$Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+dd l,m-Z-ddl.m/Z0ddl1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7ddl8m9Z9m:Z:ddl;mZ>m?Z?ddl@mAZAddlBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNddlOmPZPmQZQmRZRmSZSmTZTmUZUmVZVeriddlWmXZXmYZYmZZZm[Z[ddl m\Z\ddl]m^Z^ddl_m`Z`maZambZbmcZcddldmeZeddlfmgZgmhZhmiZiddl@mjZjdd lkmlZled!ZmeXehdzgegfZneoelZpeqejzZseqZtejjewd"ZxejewZzdd#Z{ejGd$d%Z}Gd&d'ej$Z~Gd(d)eZeHd*+Gd,d-eZGd.d/ZejGd0d1ZejGd2d3ZejGd4d5ZejGd6d7ZejGd8d9ZeezezezezZiZd:ed;<Gd<d=ZiZd>ed?<d@aiZdAedB<iZdCedD< d ddEZGdFdGeZ ddHZ ddIZddJZ d ddKZddLZddMZej.ddNZ ddOZ ddPZdQZddRZej:ej<ej>ej<iej@ejBejDejFejHejJejLejNejPejRejTf Dcic]}||c}ZdSedT< d dUZd!dVZ d"dXZ d#dYZd$dZZGd[d\ZGd]d^e0Z/Gd_d`Ze jfdae jhbZd%dcZGdddee>eeReZejGdfdgZed&idhee"jtdidjkdlee"jtdmdndopdqee"jtdrdsdtpduee"jtdvdwdxpdyee"jtdzd{d|pd}ee"jtd~dd}dee"jtdddpdee"jtdddddee"jvdddpdee"jtddkdee"jtddkdee"jtddkdee"jtddkdee"jtdddddee"jtddddee"jtdddpdee"jtddkdee"jtddkdee"jtdddpdee"jtdddpdee"jtddkdee"jtddkdee"jtddkdee"jtddkdee"jtddkdee"jtdd¬kdee"jtdĄdŬkdee"jtdDŽdȬkdee"jtdʄdˬkdee"jtd̈́dάkdee"jtdЄdѬkdee"jtdӄdԬkdee"jtdքd׬kdee"jtdلdڬkdee"jtd܄dݬkdee"jtd߄dkdee"jtddkdee"jtddkdee"jtddkdee"jtddkdee"jtddkZded<d'dZGddeDZGddeGZGddeZejGddZGddZe«ZGddZGddZedeqZedWeeŐZereejeTeeeŐdfzfZGddeeefZGddZGd d eeeZejGd d Zej.d(d ZGddZGdde?Zycc}w()) annotationsN)ABCabstractmethod)autoEnum)chain)AnycastClassVarGeneric NamedTuple TYPE_CHECKING)SelfTypeVar)ELEMENTWISE_TYPE_PROMOTION_KIND)_pytree) ConfigModule) OrderedSet)Max)int_oo) PythonPrinter)free_symbol_is_typesymbol_is_typeSymT) bound_sympy ValueRanges)configmetrics)DtypePropagationOpsHandler)BasicMathOpsMixinDefaultHandler)ShapePropagationOpsHandler) boolean_opsDeferredLineBasegenerate_assertget_current_backendIndentedBuffer ir_dataclass ScopedDict sympy_dotsympy_index_symbol sympy_subs triton_typeunique) NullHandlerops OpsHandlerOpsValue ReductionType StoreModeV)CallableIteratorMutableMappingSequence) GraphModule)CustomGraphModulePass)Buffer ChoiceCaller FixedLayoutIRNodeLoopBody)BaseScheduling Scheduler SchedulerNode)BlockShapeTypePythonWrapperCodegen_Tschedulecxtjtjrtj d|yy)NzData type propagation: %s) schedule_log isEnabledForloggingDEBUGdebug)msgs i/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/codegen/common.pydata_type_loggerrTYs*  /6<0cZeZdZUdZded<ded<d dZed dZd dZed d Z y )FileBackedGraphModulez Output of FX wrapper codegen. Exposes the same methods as ModuleType, but these map back to a GraphModule instead of Python source. r;gmzCallable[..., Any] compiled_fnc&tjddd|_tjtj |jj |j5}|j|jdddy#1swYyxYw)Nzw+z.pyF)modesuffixdelete) tempfileNamedTemporaryFileatexitregisterosremovenamewritevalue)selffs rS __post_init__z#FileBackedGraphModule.__post_init__hsi!33eE    4==#5#56 ]] a GGDJJ    s "BBc.|jjSN)r^rdrgs rS__file__zFileBackedGraphModule.__file__rs}}!!!rUc |j|Srk)rYrgargss rScallzFileBackedGraphModule.callvst&&rUc.|jjSrk)rXcoderls rSrfzFileBackedGraphModule.valueysww||rUNreturnNonerustr)rp list[Any]rur ) __name__ __module__ __qualname____doc____annotations__ripropertyrmrqrfrUrSrWrW^sF O## ""'rUrWc<eZdZdZdZdZeddZeddZy) WorkspaceZeroModerrGrc||k(s|tjk(r|S|tjk(r|Std|d|d)NzWorkspaceZeroMode.combine(, ))r UNINITIALIZEDNotImplementedErrorabs rScombinezWorkspaceZeroMode.combinesK 6Q+999H !// /H!$>qe2aU!"LMMrUcF|rtjStjSrk)r ZERO_ON_CALLr) zero_fills rS from_boolzWorkspaceZeroMode.from_bools $11 1 ...rUN)rrrrrur)rboolrur) rzr{r|rrZERO_PER_GRAPH staticmethodrrrrUrSrr~s9MLNNN//rUrc4eZdZdZeddZeddZy) CodegenSymbolzP An IR object possibly corresponding to a variable in the wrapper code. cyrkrrls rSget_namezCodegenSymbol.get_name rUcyrkrrls rS get_examplezCodegenSymbol.get_examplerrUNrwruztorch.Tensor | sympy.Symbol)rzr{r|r}rrrrrUrSrrs/    rUrT)frozenc&eZdZUdZded<ded<ded<ded <d Zded <ejZd ed <e dddZ e d dZ e d!dZ e d!dZ d"dZeZd#dZd$dZd%dZed%dZeZeZeZd&dZd'dZd'dZd(dZd)dZd*dZy)+ WorkspaceArga2A temporary buffer used for a single kernel, then discarded. Not registered as a traditional buffer since there are no users, so it would be dead code eliminated. Args: nbytes: The size of the buffer in bytes. zero_fill: Whether the buffer should be initialized to zero. sympy.Exprcountr zero_mode torch.devicedevicerx outer_namews_ptr inner_name torch.dtypedtypecP|ttjjSrk)nextr6graph workspace_id)prefixs rS unique_namezWorkspaceArg.unique_names!$qww334566rUc|j|jk(xr4|j|jk(xr|j|jk(Srk)rrrrs rScan_joinzWorkspaceArg.can_joins@ LLALL ( XQWW-? XAHHPQPXPXDX rUct|j|jztj|j|j|j |j |j|jSN)rrrrrr) rrrrrrrrrrs rSjoinzWorkspaceArg.joinsS''AGG#'// Q[[I''88||||   rUc|j|jk(r2|j|jk(r|j|jk(sJtt |j |j t j|j|j|j|j|j|jSr) rrrrrrrrrrrs rSmaximumzWorkspaceArg.maximums GGqww 188qxx#7ALLALL|jjSrk) get_layoutrrls rSrzWorkspaceArg.get_examples ,,..rUcfddlm}||j|j|jgdgS)Nr)r?rG)rrsizestride)irr?rrr)rgr?s rSrzWorkspaceArg.get_layouts.$;;****3   rUc"|jSrk)rrls rSlayoutzWorkspaceArg.layouts  rUc6tjjSrk)sympySZerorls rS get_offsetzWorkspaceArg.get_offsetsww||rUc|jgSrk)rrls rSget_sizezWorkspaceArg.get_sizes |rUc8tjjgSrk)rrOnerls rS get_stridezWorkspaceArg.get_strides }rUc|jSrk)rrls rSrzWorkspaceArg.get_names rUcy)NFrrls rS get_is_pinnedzWorkspaceArg.get_is_pinnedsrUcgSrkrrls rSget_inputs_that_alias_outputz)WorkspaceArg.get_inputs_that_alias_outputs rUN) workspace_)rrxrurx)rrrrrur)rrrrrur)rur)rurr)rur?)rur)rulist[sympy.Expr]rwrur)ru list[str])rzr{r|r}r~rtorchuint8rrrrrrrget_device_or_errorrrrrrget_output_specmaybe_get_output_specmaybe_get_layoutrrrrrrrrUrSrrs    OJE;$77        %/ !!!O&!rUrceZdZddZddZy)TritonScratchWorkspacec ||_||_yrk)r_generate_dtype_str)rgrgenerate_dtype_strs rS__init__zTritonScratchWorkspace.__init__ s #5 rUc"|jSrk)rrls rSrz)TritonScratchWorkspace.generate_dtype_str s''))rUN)rintrCallable[..., str]rw)rzr{r|rrrrUrSrrs 6*rUrcpeZdZUded<ded<ded<ej j Zded<dZd ed <y) TensorArgrxrdbufferrrroffsetN str | Nonealias_of) rzr{r|r~rrrrrrrUrSrrs. I K FJ%HjrUrc4eZdZUded<ded<eddZy)SizeArgrxrdrexprcyrkrrls rSrzSizeArg.alias_ofsrUNrur)rzr{r|r~rrrrUrSrrs I  rUrceZdZUded<y) ConstexprArgrxrdNrzr{r|r~rrUrSrr$s IrUrc6eZdZUded<ded<ded<ded<y) TMADescriptorArgrxrdapi_typezlist[sympy.Expr] | None block_shapetorch.dtype | NonerNrrrUrSrr)s IM(( rUrc>eZdZUded<ded<dZded<dZded<y) DeviceCodegenSchedulingConstructor schedulingWrapperConstructorwrapper_codegenNWrapperConstructor | Nonecpp_wrapper_codegenfx_wrapper_codegen)rzr{r|r~rrrrUrSrr1s&%%''59294818rUrzdict[str, DeviceCodegen]device_codegensceZdZddZddZddZddZddZddZddZ ddZ dd Z dd Z dd Z dd Zdd ZddZddZddZddZddZ d ddZy)DeviceOpOverridesctrkrrgrds rSimport_get_raw_stream_asz*DeviceOpOverrides.import_get_raw_stream_as?!!rUctrkrrg device_idxs rS set_devicezDeviceOpOverrides.set_deviceBrrUctrkrrls rS synchronizezDeviceOpOverrides.synchronizeErrUctrkrr s rS device_guardzDeviceOpOverrides.device_guardHrrUctrkrrls rScurrent_streamz DeviceOpOverrides.current_streamKrrUc |dS)Nz.native_handler)rg stream_names rS stream_handlezDeviceOpOverrides.stream_handleNsn--rUctrkrrls rScpp_device_guardz"DeviceOpOverrides.cpp_device_guardQrrUctrkrrls rScpp_aoti_device_guardz'DeviceOpOverrides.cpp_aoti_device_guardTrrUctrkrrls rScpp_stream_guardz"DeviceOpOverrides.cpp_stream_guardWrrUctrkrrls rScpp_aoti_stream_guardz'DeviceOpOverrides.cpp_aoti_stream_guardZrrUctrkrrls rScpp_getStreamFromExternalz+DeviceOpOverrides.cpp_getStreamFromExternal]rrUctrkrrls rS kernel_headerzDeviceOpOverrides.kernel_header`rrUctrkrrls rS kernel_driverzDeviceOpOverrides.kernel_drivercrrUctrkrrls rScpp_stream_typez!DeviceOpOverrides.cpp_stream_typefrrUctrkrrls rSaoti_get_streamz!DeviceOpOverrides.aoti_get_streamirrUctrkrrls rScpp_kernel_typez!DeviceOpOverrides.cpp_kernel_typelrrUctrkrrls rScpp_device_ptrz DeviceOpOverrides.cpp_device_ptrorrUctrkrrls rStma_descriptor_helpersz(DeviceOpOverrides.tma_descriptor_helpersrrrUNctrkr)rgidx workspacers rS cpp_scratchzDeviceOpOverrides.cpp_scratchus "!rUrdrxrurx)r rrurxrw)rrxrurxrk)r/rr0rrrruztuple[list[str], str] | None)rzr{r|rr rrrrrrrrrr!r#r%r'r)r+r-r1rrUrSrr>s""""".""""""""""""QU""#9"CM" %"rUrzdict[str, DeviceOpOverrides]device_op_overrides_dictFz'dict[str, CustomGraphModulePass | None]custom_backend_passeszdict[str, ConfigModule | None]custom_backend_codegen_configsct||||t|<|t|<|r)t|tr|t usJd|dt |t |<y)Nzdevice_custom_config=z: cannot be the same as the default inductor config config=)rrr4 isinstancerrr5)rdevice_schedulingdevice_wrapper_codegendevice_cpp_wrapper_codegendevice_fx_wrapper_codegendevice_custom_passdevice_custom_configs rSregister_backend_for_devicer>sv,"! OF %7&! +\ :$F2 %#%%`Y_Xa b   3 .B"6*rUceZdZeZeZeZeZeZeZ eZ eZ eZ eZ y)BackendFeatureN)rzr{r|rFOREACH BUCKETIZEINPLACE_BUFFERSMASKED_SCATTER_WITH_INDEXSCANSORTTUPLE_REDUCTIONPREFER_STORE_LOOP_ORDERTRITON_TEMPLATESREDUCE_TO_SINGLE_ELEMENTrrUrSr@r@sLfGIfO $ 6D 6DfO"fv#vrUr@c:| tStt|tjr |j }n7t|t sJt ||}tj|}t|}|sJ|d}|j|Srk) rinit_backend_registrationr7rrtyperxget_scheduling_for_deviceget_backend_features)r device_typescheduling_ctorrs rSrOrOs~|&%,,'kk &#&4V 4& k*/ j@r|n|| t9d|||dt;dfd| || t9d t;d|| t9dt;d| | || t9dd|it;dfd| t9dt;d|| || t9dt;d| | || tBjDjG}|dk7rLt9|@ddl$m%} |d }|d!}|d"}|d#}|r|r|rt;|||||yyyyyy#tL$rYywxYw)$z Register the backend for different devices, including the scheduling for kernel code generation and the host side wrapper code generation. rG) CppScheduling) CppWrapperCpu)CppWrapperCpuArrayRef) CppWrapperGpu) CppWrapperMps)CUDACombinedScheduling)HalideScheduling)MetalScheduling)PallasScheduling)PythonWrapperMtia)TritonSchedulingrH)WrapperFxCodegen)XPUCombinedSchedulingcpuN)cpphalidetritonpallasc6tj|Srk)r cpu_backend)r cpu_backendss rSz+init_backend_registration..?|F,>,>? KrUcuda)rqrprrc6tj|Srk)r cuda_backend)r cuda_backendss rSrvz+init_backend_registration..!sA}V-@-@A*MrUtpuxpurrc6tj|Srk)r tpu_backend)r tpu_backendss rSrvz+init_backend_registration..?rwrUmpsmtia privateuseoner)_get_custom_mod_func SchedulingrICppWrapperCodegenrl)'roracpp_wrapper_cpurbcpp_wrapper_cpu_array_refrccpp_wrapper_gpurdcpp_wrapper_mpsrecuda_combined_schedulingrfrprgrrhrrripython_wrapper_mtiarjrqrkwrapperrI wrapper_fxirrlxpu.xpu_combined_schedulingrmrNr>r aot_inductorallow_stack_allocationr_C_get_privateuse1_backend_name torch.utils.backend_registrationr RuntimeError)rarbrcrdrerfrgrhrirjrkrIrlrmprivate_backendrr8rrrrur{rs @@@rSrLrLs #.@..@($(6(-.B '/ &&&  $  K ""99 "  !(0-&& $  M    !'/#    !'/#  !    !'/ &  $  K  !'/#      !(0#       hh<<>O?* %o 6 >I  4\ B 23IJO"67J"K !56H!I  _9L+#%#'& :M_  ? +$   s55F00 F<;F<cLddlm}g|t||j|S)Nr)FlexibleLayout)rrr+contiguous_strides)index index_varssizesrs rSindex_prevent_reorderingrms, $ UU TIj.*K*KE*RS TTrUc|t|<yrk)r3)rdevice_op_overridess rSregister_device_op_overridesrxs(;V$rUcttryddlm}ddlm}ddlm}ddlm}ddlm}td|day)NrG)mps_device_op_overrides)CpuDeviceOpOverrides)rr|T) _device_op_overrides_initializedrcpu_device_op_overridesrrxrrr}r)rrrmtia_op_overridesxpu_op_overridess rS_initialize_device_op_overridesr~s/()=)><!(<(>?'+$rUcht|tsJt|tt|Srk)r7rxrMrr3rs rSget_device_op_overridesrs, fc "0DL0 "#% #F ++rUzdict[torch.dtype, torch.dtype]DTYPE_TO_COMPUTATION_DTYPEcr|tvrtjS|dvrd|vr|dS|dS|dvrtjS|dvrtjS|dk(rd|vr|dS|dS|dk(rd|vr|dS|dS|d vr$|d}t j j|S|d k(rd|vr|dS|d Sy ) zK Given op name and a list of input dtypes, deduce the output dtype )to_dtype index_expr value_exprr)randrandn) get_index randint64 load_seed reductionrGconstant)loadstorestore_reductionto_dtype_bitcastN)r$rrfloatint64r6rr)op_namerpkwargsbuf_names rSdeduce_output_dtype_by_namers +-zz   #*V"3vgAbA  {{   {{ K ")V"3vg@a@ J ")V"3vgAbA   7ww  ** & &")V"3vgAbA rUctdk(xrGtjjjdk7xrt tj dS)Nrqrntriton_tensor_ndim)r'r6rget_current_device_or_throwrMhasattrkernelrrUrS _is_runtime_triton_assert_targetrsF) 4 GG / / 1 6 6% ? 4 AHH2 3rUCSEVariableTypect}tjjr,t r"|j d|dt |dytjjr|dk(rddlm }m }t||sJt||tjk(r|jrd|d}n.d |d |d }n$d |d}|jrd |d}d|d||d}|j d|dyyy)Ntl.static_assert(z .dtype == rrorG)CppCSEVariable DTYPE_TO_CPPzIsVecMaskType::valuezstd::is_same_v || std::is_same_vz decltype(z typename z ::value_typezstd::is_same_vzstatic_assert(z);)r'r test_configsruntime_triton_dtype_assertr writeliner.static_cpp_dtype_assert cpp_utilsrrr7rMrris_vec)rvarrbackendrr is_same_dt c_var_types rS check_dtypers"#G77 , .,SEK*R89!:J 4rUc|JtjjrRtrGt |dk7rdj d|Dn|dd}|j d|d|dyyy) NrGrc32K|]}t|ywrkrx).0ds rS zcheck_shape..s,c!f,sr,rz .shape == ()))rrruntime_triton_shape_assertrlenrr)rrshape shape_strs rS check_shapers   77 , .14E aDII,e, ,azQR^  ,SEYKrJK / 8rUc jt}|dk(r$d}|jd|d|d|d|d|d yy) NrqzNaN or Inf foundztl.device_assert((z == ) & (z != float('inf')) & (z != float('-inf')), 'z'))r'r)rrrrRs rS check_nanr sS!#G(  T#eC58McURghkglln o rUc`eZdZd dZd dZd dZd dZd dZddZe ddZ e ddZ y )DataTypePropagationc||_d|jji|_|jj D]\}}|j|j|< yNroot)body root_blockrgraphs subblocksitems)rgrkvs rSrzDataTypePropagation.__init__sT DOO))<  NN((* %DAqWWDKKN %rUc|j}|Dcgc]9}t|tjjs(|j dk7s8|;}}t |dk(rytd|D}|sytjtj|Dcgc])}|jtjj+c}Scc}wcc}w)N placeholderrc3K|]K}tj|jvxr)|jtjjduMywrk)OptimizationContextkeymetar)rns rSrzBDataTypePropagation.deduce_node_dtype_by_inputs..$sS)  # #qvv - B*../55TA B) sAA)all_input_nodesr7rfxNodeoprall functoolsreduce promote_typesrrrr)rgnodeinputsr input_nodesall_input_nodes_propagateds rSdeduce_node_dtype_by_inputsz/DataTypePropagation.deduce_node_dtype_by_inputss%% Auxx}}!=!$$-BWA   { q %() !) & " *   T$++->>+ ;; ! !"3 455d; ;8 ++ L    //55rUcn|jsJd}|jD]}tj|jvr|jtj}n t}|j ||_||jtj<|j dk(s|j }|S)Nr)nodesrrrrrr )rgr graph_dtyperopt_ctxs rSr z#DataTypePropagation.propagate_graphTs{{{*. KK ,D"&&$))3))$7$;$;<-/ 2248GM18DII)-- .{{h&%mm  ,rUc>|j|jdSr)r rrls rS propagatezDataTypePropagation.propagatefs##DKK$788rUc.||jSrk)r)clsrs rSpropagate_loopbodyz&DataTypePropagation.propagate_loopbodyis4y""$$rUcddlm}ddlm}t ||sJt |t |j |sJt |j tj|j S)NrrA)rE) loop_bodyrB schedulerrEr7rM_bodyrr!)r rrBrEs rSpropagate_scheduler_nodez,DataTypePropagation.propagate_scheduler_nodemsX(-$ .:T :.$**h/Adjj1AA/"55djjAArUN)rrBrurv)r torch.fx.Noderur)rr'rur)rztorch.fx.Graphrur)rur)rrBrur)rrErur) rzr{r|rr rrr r classmethodr!r&rrUrSrrsJ% * 6:$9%%BBrUrcDeZdZddd dfdZddfd ZxZS)rT)simplifypc|r]t|tjrCttj dr)tj j j|}t|%|S)Nsizevars) r7rExprrr6rr-r*superdoprint)rgrr*r+ __class__s rSr0zPythonPrinter.doprintxsK  44*9U77##,,T2Dwt$$rUct|tjrd|j|dSt||||SN(r)r7rMod_printr/ parenthesize)rgitemlevelstrictr1s rSr7zPythonPrinter.parenthesizes@ dEII &t{{4()+ +7'eV< ?@@rUc tjdtj}tj|tj |tj tj|SrL)r1rrrarBr`rQneg)rEones rSsigmoidzOpDecompositions.sigmoidsCll1ekk*{{3SWWSWWQZ-@ ABBrUcrtj|tjdtjSNr)r1rrrrarDs rSreluzOpDecompositions.relus"{{1cll1ekk:;;rUcVtjtj|||Srk)r1r`rIrEyzs rSfmazOpDecompositions.fmaswwswwq!}a((rUc.tj||SrkrHrErls rSmul_rnzOpDecompositions.mul_rnswwq!}rUc.tj||Srkr1rBrps rSdiv_rnzOpDecompositions.div_rns{{1a  rUcTtjtj||Srk)r1rfloorrrs rS floor_to_intzOpDecompositions.floor_to_int||CIIaL%00rUcTtjtj||Srk)r1rceilrws rS ceil_to_intzOpDecompositions.ceil_to_ints||CHHQK//rUcTtjtj||Srk)r1rtruncrws rS trunc_to_intzOpDecompositions.trunc_to_intryrUc tj||}tjtj|tjdt j tjtj|tj|}tj|tj|||Srh) r1modand_nerrrasignbitwherer`)rrrconds rS remainderzOpDecompositions.remaindersy GGAqMxx FF1cll1ekk2 3 FF3;;q>3;;q> 2 yyswwq!}a00rUcTtjtj||Srk)r1rroundrws rS round_to_intzOpDecompositions.round_to_intryrUN)rfOpVarTrurrErrur)rErrlrrmrrurrErrlrrur)rrrrrurrrrrrur)rzr{r|r}rr@rFrJrOrRrTrZr\r^rbrfrirnrqrtrxr|rrrrrUrSr>r>s@@ CC<<CCRRQQMMAACC<<))!!1100111111rUr>z[a-z0-9_.]+|\([^)]*\)|)flagsc|ddk7st|dkryd}t|ddD]3\}}|dk(r|dz }n |dk(r|dz}|dk(s!|t|dz k7s3y|dk(sJy)Nrr4rFrGrT)r enumerate)stringrichars rS_all_in_parensrs ayC3v;? EVABZ(4 3; QJE S[ QJE A:!s6{Q.  A:: rUc^eZdZed"dZed#dZed$dZed%dZed&dZed&dZ ed&dZ ed&dZ ed&d Z ed'd Z ed(d Z d) d*d Z d+dZd,dZ d- d.dZd/dZd0dZ d1dZ d2dZ d3dZ d4 d5dZd6dZd&dZdej8d ddd d7dZd8dZd9dZed:dZ e!d;d Z"e!d> rrps rSbitwise_right_shiftzOpOverrides.bitwise_right_shift.rrUc.tj||Srkrsrs rS int_truedivzOpOverrides.int_truediv3s {{1a  rUcTtj|tj|Srk)r1rrInteger)rdrs rSrzOpOverrides.load_seed;sxxemmF344rUTc*tt|Srk)r,rx)rgrrcheckwrap_negs rSindirect_indexingzOpOverrides.indirect_indexing?s"#c(++rUcDtt|jd)Nz,: check_bounds should be handled by CSEProxyrrMrzrgrrloweruppers rS check_boundszOpOverrides.check_boundsHs'"Dz""##O P  rUcDtt|jd)Nz$: load should be handled by CSEProxyrrgrdrs rSrzOpOverrides.loadOs%!Dz""##G H  rUNcDtt|jd)Nz%: store should be handled by CSEProxyrrgrdrrfr[s rSrzOpOverrides.storeTs'"Dz""##H I  rUcDtt|jdNz3: device_assert_async should be handled by CSEProxyrrgrrRs rSdevice_assert_asynczOpOverrides.device_assert_async[%!Dz""##V W  rUcDtt|jd)Nz/: store_reduction should be handled by CSEProxyrrgrdrrfs rSrzOpOverrides.store_reduction`s%!Dz""##R S  rUcDtt|jd)Nz): reduction should be handled by CSEProxyrrgr src_dtypereduction_typerfs rSrzOpOverrides.reductiones'"Dz""##L M  rUcDtt|jd)Nz$: scan should be handled by CSEProxyrrgdtypes combine_fnvaluess rSscanzOpOverrides.scanps'"Dz""##G H  rUcDtt|jd)Nz$: sort should be handled by CSEProxyrrgrrstable descendings rSsortzOpOverrides.sort}s'"Dz""##G H  rUcDtt|jd)Nz): bucketize should be handled by CSEProxyrrgr boundariesboundary_indicesindexing_dtyperightsortersorter_indicess rS bucketizezOpOverrides.bucketizes'"Dz""##L M  rUcDtt|jd)Nz2: halide_clamp only implemented for Halide backendr)rgrfrrs rS halide_clampzOpOverrides.halide_clamps%!Dz""##U V  rUcDtt|jd)Nz): dot only implemented for Triton backendr)rgrErls rSdotzOpOverrides.dots%!Dz""##L M  rUrG) constraintsris_purepack input_dtypescDtt|jd)Nz<: inline_asm_elementwise only implemented for Triton backendr)rgasmrrrrrrs rSinline_asm_elementwisez"OpOverrides.inline_asm_elementwises'"Dz""##_ `  rUcDtt|jd)Nz.: ops.output should not appear at codegen timeAssertionErrorrMrzros rSrzOpOverrides.outputs%Dz""##Q R  rUcDtt|jd)Nz3: ops.placeholder should not appear at codegen timerrgrs rSrzOpOverrides.placeholders%Dz""##V W  rUc0dfd }|_d|_|S)NcJtt|jd)Nz does not implement ops.r)rgrprrds rS unimplementedz1OpOverrides._unimplemented..unimplementeds*%:&&''?vF rUT)rgrrpr rr rur)rzis_unimplemented)rdrs` rS_unimplementedzOpOverrides._unimplementeds   "& )- &rUcpt||d}tt|d}| xs||k(xs t|ddS)NrF)getattrr2)r rdfn default_fns rS_is_unimplementedzOpOverrides._is_unimplementeds? S$ %Zt4 vSz)SWR9KU-SSrUcP|dvsJ|tjD]\}}t||}|/|j|s&t |||j |C||j vsJd|d|j||_t ||t|y)N)rqrocppvecrprzmultiple definitions of z on ) pointwise_overrides_datarrrsetattrr__dict__rzr)r r funcnamedataimpls rS_initialize_pointwise_overridesz+OpOverrides._initialize_pointwise_overridessEEMvME6<<> ;NHd4(D|((2C3+=+=h+GHs||3.xjS\\NK3!) X|D'9: ;rU)rrrur)rfzbool | float | intrrrurr)rrrurrr)rdrxrrrurTT) rrrsympy.Expr | intrrrrru sympy.Symbol rrrrrrrrrurv)rdrxrrrurrk) rdrxrrrfrr[r5rurvrrrRrxrurv)rdrxrrrfrrurv) rrrrrr4rfOpVarT | tuple[OpVarT, ...]rur )rtuple[torch.dtype, ...]rzFCallable[[tuple[OpVarT, ...], tuple[OpVarT, ...]], tuple[OpVarT, ...]]rtuple[OpVarT, ...]rur ) rr rr rrrrrur NN)rrr.tuple[str, sympy.Expr, sympy.Expr, sympy.Expr]rrrrrrrtuple[str, sympy.Expr] | Nonerz OpVarT | Nonerur)rfrrrrrrur)rrrrxrrrrrrrrrztuple[torch.dtype, ...] | Nonerur)rprrurv)rrrur)rdrxruzCallable[..., OpVarT]rdrxrur)r rxrurv)$rzr{r|rrrrrrrrrrrrrrrrrrrrrrrrrrCrrrrr(rrrrUrSrrs>  **..BBBBBBCCCC!! 55 , ,, ,  ,  ,  &0 9= FJ   NR  * 39 AJ         &  +  %    '     #       '  #        $15(,    C  !  $    .  &       #'"]]7;              5       TT  ; ;rUrc|eZdZUded<ded<dZded<dZded<ejZd ed <dZ ded <dZ ded <y) OverridesDatarxrdrroNzCallable[..., str] | Nonerqrrtype_promotion_kindrpr) rzr{r|r~rqrrDEFAULTrrprrrUrSrrsQ I (,F %,(,F %,'//8)-F %,%)C ")rUrairy_aicd|dS)Nzairy_ai_forward(rrrDs rSrvrvs(1-rUspecial_airy_ai)rrord bessel_j0cd|dS)Nzbessel_j0_forward(rrrDs rSrvrv*1#Q/rUcd|dS)Nz libdevice.j0(rrrDs rSrvrv=1-rUspecial_bessel_j0)rrorqrd bessel_j1cd|dS)Nzbessel_j1_forward(rrrDs rSrvrvrrUcd|dS)Nz libdevice.j1(rrrDs rSrvrvrrUspecial_bessel_j1 bessel_y0cd|dS)Nzbessel_y0_forward(rrrDs rSrvrvrrUcd|dS)Nz libdevice.y0(rrrDs rSrvrvrrUspecial_bessel_y0 bessel_y1cd|dS)Nzbessel_y1_forward(rrrDs rSrvrvrrUcd|dS)Nz libdevice.y1(rrrDs rSrvrvrrUspecial_bessel_y1digammacd|dS)Nz calc_digamma(rrrDs rSrvrv s aS*rUc |dS)Nz .digamma()rrDs rSrvrv s A3j)rU)rrorrdrRcd|dS)Nz calc_erfcx(rrrDs rSrvrv A3a(rUcd|dS)Nzlibdevice.erfcx(rrrDs rSrvrvs+A3a0rU special_erfcxrncd|d|d|dS)Nz std::fma(rrrrks rSrvrvsis"QCr!A6rUcd|d|d|dS)Nzfmadd(rrrrks rSrvrvss"QCr!A6rUcd|d|d|dS)Nztl.fma(rrrrks rSrvrvs2aS1#Q7rU)rrorrqrdrqcd|d|dS)Nr4) * (rrrps rSrvrv*s1QCuQCq)rUcZtjjr d|d|dSd|d|dS)Nr4r6rzlibdevice.mul_rn(r)rversionhiprps rSrvrv+s< ==   s%s!, 2aS *rUigammacd|d|dSNz calc_igamma(rrrrps rSrvrv3<s"QCq1rUigammaccd|d|dSNz calc_igammac(rrrrps rSrvrv8=2aS2rUgammainccd|d|dSr<rrps rSrvrv=r=rUspecial_gammainc gammaincccd|d|dSr@rrps rSrvrvBrArUspecial_gammaincci0cd|dS)Nzcalc_i0(rrrDs rSrvrvG1orUcd|dSNzlibdevice.cyl_bessel_i0(rrrDs rSrvrvH3A3a8rUc |dS)Nz.i0()rrDs rSrvrvIs A3erU)rrorqrrdi0ecd|dS)Nz calc_i0e(rrrDs rSrvrvN !A&rUc |dS)Nz.i0e()rrDs rSrvrvOs A3frU special_i0ei1cd|dS)Nzcalc_i1(rrrDs rSrvrvTrJrUcd|dSNzlibdevice.cyl_bessel_i1(rrrDs rSrvrvUrMrU special_i1i1ecd|dS)Nz calc_i1e(rrrDs rSrvrvZrQrU special_i1elog_ndtrcd|dS)Nzcalc_log_ndtr(rrrDs rSrvrv_sqc+rUspecial_log_ndtrmodified_bessel_i0cd|dS)Nzmodified_bessel_i0_forward(rrrDs rSrvrve3A3a8rUcd|dSrLrrDs rSrvrvfrMrUspecial_modified_bessel_i0modified_bessel_i1cd|dS)Nzmodified_bessel_i1_forward(rrrDs rSrvrvkrarUcd|dSrWrrDs rSrvrvlrMrUspecial_modified_bessel_i1modified_bessel_k0cd|dS)Nzmodified_bessel_k0_forward(rrrDs rSrvrvqrarUspecial_modified_bessel_k0modified_bessel_k1cd|dS)Nzmodified_bessel_k1_forward(rrrDs rSrvrvvrarUspecial_modified_bessel_k1ndtrcd|dS)Nz calc_ndtr(rrrDs rSrvrv|s 1#Q'rU special_ndtrndtricd|dS)Nz calc_ndtri(rrrDs rSrvrvr/rU special_ndtri polygammac *|d|d|d|d|d|d S)Nz == 0 ? calc_digamma(z) : (z == 1 ? trigamma(z) : calc_polygamma(rrrrps rSrvrvs8S%aSaS0A!DWXYWZZ\]^\__a brUscaled_modified_bessel_k0cd|dS)Nz"scaled_modified_bessel_k0_forward(rrrDs rSrvrv:1#Q?rU!special_scaled_modified_bessel_k0scaled_modified_bessel_k1cd|dS)Nz"scaled_modified_bessel_k1_forward(rrrDs rSrvrvrxrU!special_scaled_modified_bessel_k1spherical_bessel_j0cd|dS)Nzspherical_bessel_j0_forward(rrrDs rSrvrvs4QCq9rUspecial_spherical_bessel_j0zetacd|d|dS)Nzzeta(rrrrps rSrvrvs52aS*rU special_zetachebyshev_polynomial_tcd|d|dS)Nzchebyshev_polynomial_t_forward(rrrrps rSrvrv:1#Rs!DrUspecial_chebyshev_polynomial_tchebyshev_polynomial_ucd|d|dS)Nzchebyshev_polynomial_u_forward(rrrrps rSrvrvrrUspecial_chebyshev_polynomial_uchebyshev_polynomial_vcd|d|dS)Nzchebyshev_polynomial_v_forward(rrrrps rSrvrvrrUspecial_chebyshev_polynomial_vchebyshev_polynomial_wcd|d|dS)Nzchebyshev_polynomial_w_forward(rrrrps rSrvrvrrUspecial_chebyshev_polynomial_wlegendre_polynomial_pcd|d|dS)Nzlegendre_polynomial_p_forward(rrrrps rSrvrv9!BqcCrUspecial_legendre_polynomial_pshifted_chebyshev_polynomial_tcd|d|dS)Nz'shifted_chebyshev_polynomial_t_forward(rrrrps rSrvrvB1#Rs!LrU&special_shifted_chebyshev_polynomial_tshifted_chebyshev_polynomial_ucd|d|dS)Nz'shifted_chebyshev_polynomial_u_forward(rrrrps rSrvrvrrU&special_shifted_chebyshev_polynomial_ushifted_chebyshev_polynomial_vcd|d|dS)Nz'shifted_chebyshev_polynomial_v_forward(rrrrps rSrvrvrrU&special_shifted_chebyshev_polynomial_vshifted_chebyshev_polynomial_wcd|d|dS)Nz'shifted_chebyshev_polynomial_w_forward(rrrrps rSrvrvrrU&special_shifted_chebyshev_polynomial_whermite_polynomial_hcd|d|dS)Nzhermite_polynomial_h_forward(rrrrps rSrvrvs82aSBrUspecial_hermite_polynomial_hhermite_polynomial_hecd|d|dS)Nzhermite_polynomial_he_forward(rrrrps rSrvrvrrUspecial_hermite_polynomial_helaguerre_polynomial_lcd|d|dS)Nzlaguerre_polynomial_l_forward(rrrrps rSrvrvrrUspecial_laguerre_polynomial_lzdict[str, OverridesData]rctfdtjjtjjtjj tjj fDS)Nc3&K|]}|v ywrkr)rrErds rSrz$is_buffer_removed..s   )anyr6rremoved_buffersrinplaced_to_removerds`rSis_buffer_removedrsU  GG # # HH $ $ GG & & HH ' '   rUc4eZdZdZdfd ZddZddZxZS) DeferredLinezHA line that can be 'unwritten' by adding name to V.graph.removed_bufferscVt||||_t|trJyrk)r/rrdr7r%)rgrdliner1s rSrzDeferredLine.__init__s+  d$45555rUcFt|js |jSyrk)rrdrrls rS__call__zDeferredLine.__call__s +99 rUc.t|j|Srk)rrd)rgrs rS _new_linezDeferredLine._new_linesDIIt,,rU)rdrxrrxr)rrxrur)rzr{r|r}rrrr;r<s@rSrrsR6  -rUrceZdZdddZy) BracesBuffercHtjdfd }|S)Nc3KtD](}jdxjdz c_*t D](}xjdzc_jd*dt D](}jdxjdz c_*tD](}xjdzc_jd*yw)N{rG})ranger_indent)_rrgs rSctxz BracesBuffer.indent..ctxs6] "s# !  "F7^ $ ! s# $ F7^ "s# !  "6] $ ! s# $sC C#)ruIterator[None]) contextlibcontextmanager)rgrrs`` rSindentzBracesBuffer.indents$  " " $ # $u rUN)rG)rrruz'contextlib.AbstractContextManager[None])rzr{r|rrrUrSrrsrUrc"eZdZUded<ded<y)InplacedBufferrxrr other_namesNrrrUrSrrs OrUrc.eZdZUded<dZded<ddZy) ArgNamerxrdFr is_constexprcB|j|jrdSdS)Nz : tl.constexprr)rdrrls rS full_namezArgName.full_names*))$2C2C.LMMLMMrUNrw)rzr{r|r~rrrrUrSrrs IL$NrUrceZdZddZy) RemovedArgcy)NREMOVEDrrls rS__str__zRemovedArg.__str__"srUNrw)rzr{r|rrrUrSrr!srUrceZdZe ddZddZddZeddZddZddZ ddZ e jf ddZ dd Zdd Zdd Zd d Zd!d Zd"dZd#dZ d$ d%dZ d&dZd'dZd(dZd)dZy)* KernelArgsc~|j|t}t|tr|t |x||<}|S|Srk)r\rr7rr)rodictrdresult new_results rS_lookupzKernelArgs._lookup*sD $)99T7#; fj )*0#e*'> >E$K*  rUcJi|_i|_i|_i|_g|_yrk) input_buffersoutput_buffersinplace_buffersr-workspace_argsrls rSrzKernelArgs.__init__6s)-/;=GI/1 24rUc djdjtt|j|j |j |jgS)NzKernelArgs({})r)formatrmaprrrrr-rls rS__repr__zKernelArgs.__repr__=sS&& II**++,,    rUc"t|tSrk)r7rrs rS_buffer_is_marked_removedz$KernelArgs._buffer_is_marked_removedLs$ ++rUc:tjjr4tjjjj ||}|tjj vsJ|||j vrtt|j |S||jvr'tt|j|jS|jdr|jd|j|S|jd|j|S)Nseedin_ptr)r6rr$mutation_real_namer\rrr rxrrrrrrrs rSinputzKernelArgs.inputQs 77  77$$77;;D$GD1772228D82 4&& &T0067 7 4'' '(<(R!R  /01[)C03D  ,03D  -! s E3E8cxt|tj|tjj tj |}t|jD]\}}tj||rJ|j}tj|||j|<|j|j|fcS|j|jk7r|j|jk7rJ||jj||j|jdfS)aZ Allocate or extend a workspace buffer of nelem elements. This function manages the allocation of a workspace buffer. It either creates a new WorkspaceArg or extends an existing one. Note: - Calling this function will in-place mutate the args by adding or updating a WorkspaceArg. - The codegen for generating the Python argdefs and call_defs will check this field and allocate the buffer accordingly. - A new argument "ws_ptr" will be present in the generated code. Args: nelem (sympy.Expr): The number of elements to allocate. zero_fill (bool): Whether to initialize the buffer to zero. dtype (torch.dtype): the dtype of the workspace tensor Returns: Tuple[str, str, int]: A tuple containing: - "ws_ptr": A string identifier for the workspace pointer. - "workspace_{i}": agraph level unique identifier for the workspace tensor. - offset: An integer representing the item offset in the workspace. )rrrrrr)rrrr6rrrrrrrrrrr)rgnelemrrargr existing_argrs rSr0zKernelArgs.workspaces8'11)<77668#//1   ))<)<= OA|$$\37%++)5):):<)M##A&#.. 0G0GOO''3>>9 ++s~~=  >   ""3'~~s~~q00rUc tjj}t|tj t jdd|jd|j|}|jD]*}|j|jk(s||k(r#J||f|jj||jS)a Lazily allocate a graph-wide semaphores buffer with at least min_size. This is a single buffer shared by all kernels and zero initialized once at graph start. Each kernel must leave the buffer zeroed on exit. Warning: multiple calls to this function will return the same buffer. Args: min_size: the number of int32 semaphores required Returns: name of the semaphores buffer sem_ptr semaphores_r)rrrrrr) r6rrrrrruint32rMrrrr)rgmin_sizecurrent_devicerrs rS semaphoreszKernelArgs.semaphoress<<>'66,, $^%8%8$9>;O;O:PQ!  !// @L&&#..8l*?S,,??* @ ""3'~~rUcft|tsJt||ftj|}||j vr|j |S|j j vr0tfd|j j D|j |<S)Nc3FK|]}|jsdyw)rGN)r)rrrds rSrz)KernelArgs.seed_offset..sU1!,,tBTQUs!!)r7rrMrrr-rsum)rgrdrfs ` rS seed_offsetzKernelArgs.seed_offsets%%;U U';;% e$ DMM !==' ' 4=='') )&U (<(<(>UUVW  $ e rUct|tjsJt||f|jdk(rd|j |<y|j d|j |S)Nrks)r7rSymbolrMrdr-rrs rSrzKernelArgs.sizesX$ -AT D/AA- 99 "(DMM$ ||D$--66rUct|jj|jj|jjSrk)rrkeysrr-rls rS call_nameszKernelArgs.call_namessA    # # %t':':'?'?'A4==CUCUCW  rUc|jj|d}|t|ts |jS|j j|d}|t|ts|S|j j|dS)z; Returns inner name of a given outer name. N)rr\r7rrrr)rgrdinplacedrs rSarg_namezKernelArgs.arg_names}''++D$7   8Z(H&& &))--dD9  ":k:+N !!%%dD11rUc|Srkr)rgrrs rS wrap_ptr_argzKernelArgs.wrap_ptr_args rUct|Srkr)rgrs rS wrap_size_argzKernelArgs.wrap_size_args 4yrUNcddlm}|ddlm}|}g}g}g}t|jj D]}t |tr|jd}|j} tjj|} || } |j| d| |j|j|| |j| d|jj!D]\}} ||jvrtjj|} || } |jd| d| |j|j|| |jd| d|j"j!D]\}} ||jvst | tr%tjj|} || } |j| d| |j|j|| |j| d|j$j!D]\}} t |t&j(r@t+|t,j.r&|jd| |jd n+|jd|d | |jd||j|j1|tjj2stjj2j5||j6rJd |||fS) NrG) INDEX_TYPE)rrz* *zconst z const float z const float zWorkspace not supported on CPU )rrrr/rrr7rrrr6rrrrrrrr-rr rrUNBACKED_FLOATr wrapper_codeensure_size_computedr) rgdtype_to_cpp_typerr call_argsarg_defs arg_typesrouterinnerr cpp_dtype maybe_inners rS cpp_argdefszKernelArgs.cpp_argdefss  *  $ / ,   t33::<= .H(J/((,E''EGG%%e,E)%0I OOykE73 4   T..ue< =    {!_ - .!..446 4LE5,,,GG%%e,E)%0I OOfYKr%9 :   T..ue< =   vi[2 3 4#'"5"5";";"= . E;,,, ; 0SGG%%e,E)%0I OOykK=9 :   T..ue< =    {!_ - .!MM//1 ALE5%.>++4,ug 67  /& AeW =>  6*!67   T//6 7ww##$$99%@ A&&I(II&I--rUc g}g}g}g}t|jjD]}t|tr|j t |j|j |jd|j tjj|jd|j t|j|jdtjj|jdt|jj|j jD]\}}||jvst|tr%|j t ||j ||j tjj||j t||tjj||j"jD]\}}|j t ||j ||j t%||j t'||tjj(stjj(j+||j,D]m}|j t |j|j |j.|j ||j |j0o||||fS)Nr)rdrr)r/rrr7rrrrrr6rrrrrrrr-rMrrrrrr) rgrrrprecompile_argsrr r!rs rSpython_argdefszKernelArgs.python_argdefs-s#%! ! /1t33::<= H(J/ OOGH$7$78 9   X11"5 6   QWW..x/C/CB/GH I  " "!,,#//3''++H,@,@,DE  "    $ $ &    % % '  LE5 ,,, 5*0M OOGEN +   U #   QWW..u5 6  " " ''++E2  "!MM//1 ALE5 OOGEN +   U #   T%[ )  " "75%#8 9ww##$$99%@  A&& (C OOGCNN3 4   S^^ ,  " "3 '   SYY '  ( OY>>rUc#Kt|jjD]}t|tr|j D]}|t jjvs|t jjvr<||jvr|j||jf||jvsvtt|j||jfywrk)r/rrr7rrr6rrrrrrr rx)rgrothers rSaliaseszKernelArgs.aliases`st33::<= UH(J/!-- UQWW777 ; ;;D...,,U3X5H5HHHD///sD$7$7$>?ATATTT U Us B9C,<0C,ct|jj|ttxr.t|j j|ttSrk)r7rr\rrrrs rS is_removedzKernelArgs.is_removedosK    # #D' 2J N--11$@*M NrUclt}t|jjD]1}t |t r|j |jd3|jjD]5\}}||jvst |t r%|j |7|S)Nr) rr/rrr7rr`rrr)rg live_outsrr r!s rSlive_output_bufferszKernelArgs.live_output_buffersws%/\ t33::<= 4H(J/ MM(..r2 3 4!//557 !LE5,,, 5*0M MM%  !rU)rrxrz*dict[_T, str | RemovedArg] | dict[_T, str]rdrJrurxrtrw)rdr rurr2)rrxrrxrurv)rrrrrrruztuple[str, str, int])rrrurx)rdrxrfrrurx)rdrrurx)ruz Iterator[str])rdrxrur)rrxrrrurx)r SymbolLikerurxrk)rzdict[torch.dtype, str] | Noneruz&tuple[list[str], list[str], list[str]])ruz?tuple[list[ArgName], list[str], list[KernelArgType], list[Any]])ruzIterator[tuple[str, str]]r)ruzOrderedSet[str])rzr{r|rrrrrrrrrrr0rrrr rrrr$r'r*r,r/rrUrSrr)s  9    5  ,, @B4:HM{{-1-1,0-19D-1 -1^8 7 2BF4.!>4. /4.l1? H1?f UN rUrc`eZdZdZ d d fd Zd dZd dZd dZd dZd dZ xZ S)raDA CSEVariable is just a name for an expression but it is useful to be able to annotate them on a backend dependent basis. To do so, the backends can simply overload `Kernel.create_cse_var` The "CSEVariable.update_on_args" method gives you a hook for annotations See example of TritonCSEVariable in triton.py ct|t|tsJt |||_||_d|_||_||_ yrL) r/rr7rrMrdbounds use_countrr)rgrdr3rrr1s rSrzCSEVariable.__init__sL &+.<V <.    rUc|jSrkrrls rSrzCSEVariable.__str__s yyrUc,t|jSrk)hashrdrls rS__hash__zCSEVariable.__hash__sDIIrUcXt|txr|j|jk(Srk)r7rrd)rgr)s rS__eq__zCSEVariable.__eq__s!%-I%** 2IIrUcyrkr)rgrdrprs rSupdate_on_argszCSEVariable.update_on_argss rUcN|jjd|jdSr3)r1rzrdrls rSrzCSEVariable.__repr__s$..))*!DII=::rUr)rdrxr3ValueRanges[Any]rrrrFrw)rur)r)objectrur)rdrxrpr rr rurv) rzr{r|r}rrr8r:r<rr;r<s@rSrrsU%) $  ! "   J ;rUr AugmentedKeyT)default)boundrA.cPeZdZdZ d ddZddZddZddZddZddZ dd Z dd Z dd Z e jd d ddd  ddZe jddf ddZe jddf ddZy)CSEz Common subexpression eliminationNc||_||_i|_||_|xsi|_|xsi|_|xst j|_t|_ |xsi|_ yrk) rr\_cache name_prefix store_cachereduction_cache itertoolsriter_buffer_idsrinvalidated_stores varname_map)rgrr\rG iter_buffersrHrIrMs rSrz CSE.__init__sm  FH &ALARPR  !r 6B5VY__EV3=<7B7HbrUc6g|jjD]2\}}||vs |j|=|jj|4|r9|jjDcic] \}}||vs ||c}}|_yi|_ycc}}wrk)rHrrLr`rF)rg keep_varsrdtmprrs rS invalidatezCSE.invalidates44++1134 2ID#)#$$T*''++D1 2 ,0KK,=,=,?RDAq1 >1a4RDKDKSs 1 B>Bc t||j|j|j|j|j |j |jS)N)rr\rGrNrHrMrI)rMrr\rGrKrHrMrIrls rSclonez CSE.clonesPtDz;;;;((--(((( 00  rUc|j}t|j|_t|j|_t|j|_|S)zNReturn a copy of using ScopedDict so changes to *_cache aren't visible in self)rTr*rFrIrH)rgnew_cses rS scoped_copyzCSE.scoped_copysH**,#DKK0",T-A-A"B()9)9:rUc"tt|S)z@Override this method to augment cache key with backend specifics)r r@rg cache_keys rS augment_keyzCSE.augment_keysM9--rUc@||j|j|<yrkrFr[)rgrZrs rSputzCSE.puts36 D$$Y/0rUc<|j||jvSrk)r[rFrYs rScontainsz CSE.containss *dkk99rUcX|jj|j|dSrk)rFr\r[rYs rStry_getz CSE.try_gets"{{t// :DAArUc>|j|j|Srkr]rYs rSr\zCSE.gets{{4++I677rUT)r3re assignmentrrc t|tr |j}|s|sJt|trE|jj ||_|xj dz c_tt|St|tr|j}n1t|tr |j}nt|tsJ|}|j|} ||sd}| s|j|||} |j!|| |rt"j$j&r+t"j$j&j)|dt|trP|r |j+|j,| d|j/||j+|j0| St|trM|sJ|j+|j3|j,| d|j|j0| S|r |j,| d||j0} n||j0} |j+| |rPt4j6j8st4j6j:r|t=dk7r t?|| || S| jj || _| xj dz c_| S)NrGrT) only_oncez =z = ro) r7r3rfrr3tightenr4r rr(getvaluer%rrxrbnewvarr^r6r current_nodecodegen_originating_inforrsplicer\rrrrrr'r) rgrrr3rerdrrrZrrs rSgeneratez CSE.generatesl dH %::D "" dK (++--f5DK NNa N. . n - I . / IdC( ((Ill9% =E++feU3C HHY $88((HH))BB$CdN3!((DKK=R)@AMM$'$$T[[1: 9 &67%%:$$$++se3tyyk$++'WX4 -""&++se3tfT[[MJ"& }5$$T*#"//KK%22JJ!-/1U:#FC7  ++F3CJ MMQ M rUc|jt|j}tjj ||||}||j |<|Srk)rGrrKr6rcreate_cse_varrM)rgr3rrvar_namers rSriz CSE.newvarIsT &&'T-A-A(B'CDhh%%huE%(" rUctj|jvfdtjj |||}||j<|S)NcdS)Nzduplicate name: rrsrSrvzCSE.namedvar..\s4DTF2KrU)r _check_valuerMr6rro)rgrdr3rrrs ` rSnamedvarz CSE.namedvarTsU  (( (*K hh%%dFE5A!$ rU)rrrQNNNN)rrxr\rxrGrxrNzitertools.count[int] | NonerHz+MutableMapping[str, CSEVariableType] | NonerIz9MutableMapping[ReductionCacheKey, CSEVariableType] | NonerMz!dict[str, CSEVariableType] | None)rPzOrderedSet[CSEVariable]rurvrur)rZrxrur@)rZrxrrrurv)rZrxrur)rZrxruzCSEVariableType | None)rZrxrur)rr(rz@str | CSEVariable | OpsValue | IndentedBuffer | DeferredLineBaser3r>rerrdrrrrrFrur)r3r>rrrrFrur) rdrxr3r>rrrrFrur)rzr{r|r}rrRrTrWr[r^r`rbr\runknownrmrirtrrUrSrDrDs* 48CG9=III I 2 I A II7I.  .7:B8$7;#6#6#8$( $KKOK ! K  KK"KK K^$7;#6#6#8$( $   "    $7;#6#6#8$( $  ! "     rUrDc0eZdZdfd ZddZddZxZS)CodeGencTt|tj|_yrk)r/rr ExitStack exit_stackrgr1s rSrzCodeGen.__init__ds $..0rUc:|jj|Srk)r{ __enter__rls rSr~zCodeGen.__enter__hs !!# rUc>|jj|||yrk)r{__exit__)rgexc_typeexc_valexc_tbs rSrzCodeGen.__exit__ls   7F;rUrtrurr rr rr rurv)rzr{r|rr~rr;r<s@rSrxrxcs1d"Z$xZ%S)?Kernelrrx newvar_prefixr\Nz$Callable[[], OpsHandler[Any]] | None overridesc4t||rtxjdz c_|xs t |_t |_t |_t |_ d|_ d|_ d|_ d|_ t|j|j |_t%|_t%|_d|_d|_d|_d|_t%|_t%|_i|_d|_d|_y)NrGFr)r/rrgenerated_kernel_countrrpr(loadscomputestoresatomic_add_foundnum_load num_store num_reductionrDrr\csermust_keep_buffersstore_buffer_names _load_mask _load_otherrjnode_to_boundsrrinplace_update_buffersmin_elem_per_thread kernel_name)rgrpincrease_kernel_countr1s rSrzKernel.__init__us   * *a / *(JL #% %' $& % .1$2D2Ddkk.R2<,3=<&*/326LP0: 3=< 79##$ '+rUc#K|j}||_|jjj|_ d||_y#||_wxYwwrk)rjr%r3 get_boundsr)rgrpriors rSset_current_nodezKernel.set_current_nodesO!! "jj//1<<> &  %D D sAAAA AAc#K||}|dux}r t}|j}|j}|j}|j}||_||_||_|j |_ d||_||_||_||_|r |rJdyy#||_||_||_||_|r |rJdwwxYww)Nz$unexpected store inside swap_buffers)r(rrrrrW) rglbcbsbdisallow_storesrrrrs rS swap_bufferszKernel.swap_bufferss :B Dj (? (!B ,,hh   ??$ F DJ"DL DKDHEEEv2 DJ"DL DKDHEEEv2sA/C 2B6)C *C  C cy)zOverride kernel emission. Return True if overridden, False to use default. External template handlers (e.g. Helion) can override this method to implement custom kernel emission to the wrapper. Fr)rgrsrc_coder node_schedule kernel_pathget_kernel_metadatas rSemit_kernel_overridezKernel.emit_kernel_overridesrUctrkrrs rSrz Kernel.loadrrUc|j} |j|_|j||||_S#||_wxYw)z+A load the depends on an index we have read)rrr)rgrdrrs rS indirect_loadzKernel.indirect_loads8  DJ99T5)DJDJs "8 Actrkrrs rSrzKernel.store_reductionrrUctrkrrs rSrz Kernel.store "!rUcDtt|jdrrrs rSrzKernel.device_assert_asyncrrUctrkrrs rSrzKernel.reduction "!rUctrkr)rgrdrrf extra_metas rSpartial_accumulatezKernel.partial_accumulaterrUctrkrrs rSrz Kernel.scans "!rUctrkrrs rSrz Kernel.sort rrUctrkrrls rS var_rangeszKernel.var_ranges rrUct)z3 See [Note: Inductor bucketize op] rrs rSrzKernel.bucketize s "!rUctrkrrls rSassert_functionzKernel.assert_function# s!!rUc vt|tr t|}t|tsJt||t|tsJ|t|tsJ|r|rd|d|d|d|d }|d|d|}n|r |d|}|}n |sJ|d|}|}|r d|d|d}|jd|d|dS) Nr4z <= rz < rz) | ~(z, "index out of bounds: z"))r7rrxrMr)rgrrrmaskr cond_prints rSindirect_assertzKernel.indirect_assert' s c; 'c(C#s#.T#Y.#} 5# 666} 5# 666 UugT#eC5E7! ?  %%a&:&:4&@A rUcH|jt| |||yrk)remove_kernel_local_buffersr/r)rgrrrr1s rSrzKernel.__exit__V s  ((* 7F3rUctjjsytfd|jD}t|jD]c}||j vs||j jvs+j||s>|xjdzc_ j|eD]}||j jvrw|j j|}t|trEtfd|jD}|r|j!||j"j||j%|y)z Any buffers that are both created and have a last use in the same kernel can be removed. Note that V.graph.scheduler can be None when codegening triton template kernels. Nc3tK|]/}|jvrj|j1ywrk) name_to_bufdefining_op_name)rrr$s rSrz5Kernel.remove_kernel_local_buffers..e s;& i+++  ! !# & 7 7 9& s58rGc3&K|]}|v ywrkr)rrnames_to_removes rSrz5Kernel.remove_kernel_local_buffers..{ sKaQ/1Kr)r6rr$rrrrpr$can_buffer_be_removed_through_fusionrr`rr7rrrremove_inplace_bufferr remove_buffer)rgfused_node_namesrdrrcrr$s @@rSrz"Kernel.remove_kernel_local_buffersZ s3GG%%  %& ..&   ,6<++ *DD222 7 77BB*!###D) *$ )Dtyy000ii//5c:.K3??KK..t4''++D1""4( )rUctjd|t|jj|<|j j |y)Nzremove_buffer(%r))rXrQrrprrr`rs rSrzKernel.remove_buffer s; %t,)0   &   &rUctjd|t|jj|<|j j |y)Nzremoving_inplace_buffer(%r))rXrQrrprrr`rs rSrzKernel.remove_inplace_buffer s9 /6*1 !!$'   &rUc t|ttfr|Dcgc]}|j|c}Stj j j|}t|jd}|Dcic]f}t|tjtjtjtjfr||j j#|h}}t%||Scc}wcc}w)Nc|jSrkr)ss rSrvz(Kernel.rename_indexing.. s !&&rU)r)r7listtuplerename_indexingr6rr-r*sorted free_symbolsrr UNBACKED_INTSIZEPRECOMPUTED_SIZErrprr-)rgrrEsorted_symbols replacementss rSrzKernel.rename_indexing s edE] +5:;D((+; ;  ))%0 2 28HI$  %%II))''  tyy~~a    %..!<  s C4;A+C9ct|i|Srk)r)rgrprs rSrozKernel.create_cse_var sD+F++rUcZ|y|jj|jS)zC Returns arg name of a given input or output node. N)rprr)rgrs rSrzKernel.arg_name s' <yy!!$--/22rU)NT)rpzKernelArgs | Nonerrrurv)rrErurr)rr(rIndentedBuffer | Nonerrrur)rrxrrxrrxrurrdrxrrrurrdrxrrrfrrurvrk rdrxrrrfrr[r5rurvr  rrrrrr4rf%CSEVariable | tuple[CSEVariable, ...]rur) rdrxrr4rfrrdict[str, Any]rurvrr rzUCallable[[tuple[CSEVariable, ...], tuple[CSEVariable, ...]], tuple[CSEVariable, ...]]rtuple[CSEVariable, ...]rur rr rrrrrrrur)ruzdict[sympy.Symbol, sympy.Expr]rrrrrrrrrrrrrzCSEVariable | Nonerurrw) rzCSEVariable | strrrrrrzCSEVariable | str | Nonerurxr )rrrurxrurrt)rdrxrurv)rz6list[sympy.Expr] | tuple[sympy.Expr, ...] | sympy.Exprrur)rpr rr rur)rr@rur)&rzr{r|rr~r\rrrrrrrrrrrrrrrrrrrrrrrr~rrrrrrorr;r<s@rSrrpsM3FC6:I3:MQ#,%#,EI#, #,J&&%)$( F F "F " F  FF:     ""SW"" *"3>"FO" "  """& " 5 " / """&" " # "  ""'" " ( " !""'"(" "  " ! ""15-1 " "C "& " $ "  ". "+ "  """*. V VV V ' V V<""&0"9="FJ" " "4&)P'' /K/ /0,3rUrc8eZdZUdZded<dZded<dZded <y) rrz ClassVar[str]rNrrrrxops_name)rzr{r|rr~rrrrUrSrr s!"C" $E $HcrUrcb ddl}|j|jS#t$rYywxYw)Nr) undefined)jinja2 EnvironmentStrictUndefined ImportError)rs rS jinja2_envr s?!!,,"   s " ..ceZdZdZe d d dZeddZe ddZdddZe ddZ e ddZ dd Z dd Z dd Zy)KernelTemplatezj Base class for defining kernel templates. Children classes: TritonTemplate, CUTLASSTemplate c|jd}t|dkDr|ddDcgc] }d|z|z|zc}|dddj|Scc}w)NTrGrr) splitlinesrr)source num_indentsindents_spacinglinesrs rSindent_except_firstz"KernelTemplate.indent_except_first sd!!$' u:>INqrAE&4<E!"Iwwu~sAct}|ytj|jd<ddlm} |j |S#|$r}Gdd|}|||d}~wwxYw)Nrr)TemplateSyntaxErrorc(eZdZdfd ZddZxZS)IKernelTemplate._template_from_string..DetailedTemplateSyntaxErrorct||j|j|j|j ||_yrk)r/rmessagelinenordfilenameoriginal_error)rgr r1s rSrzRKernelTemplate._template_from_string..DetailedTemplateSyntaxError.__init__ s>G$&..&--&++&//  +9D'rUcFd|jd}|d|jdz }t|jdr|jjj d}|dz }t d|jdz }tt||jdz}t||D]s}||jdz k(rN||dzd ||dz }t|jd s=|d d |jjdz zzd zz }c||dzd||dz }u|S)NzError in template at line  zError message: rz Context: rrrGz: --> columnz rz^ z: ) r r rr rsplitmaxminrrr)rg error_inforstartendrs rSrzQKernelTemplate._template_from_string..DetailedTemplateSyntaxError.__str__ sA#=dkk]"!MJODLL>"DDJt22H= $ 3 3 : : @ @ F"l2 #At{{Q 7!#e*dkkAo>!&uc!2 KA DKK!O3 *QveAhZr.J J #*4+>+>#I$.(/*-1D1D1K1Ka1O*P)Q*/)0%&J !+QveAhZr.J J  K&%rU)r rrurvrw)rzr{r|rrr;r<s@rSDetailedTemplateSyntaxErrorr s 9&rUr)rrrfiltersrr from_string)renvrers rS_template_from_stringz$KernelTemplate._template_from_string sjl ;-;-O-O )*.# 8??6* *"! 8 &.A &>.a0a 7C! 8sAA!AA!ctjjt|tt fr.|Dcic]!}|j |j#c}n |j |jidfd }|Scc}w)Nc>j|}||S|Srk)r\)rdr_get_dtype_reallookups rSrz1KernelTemplate._fake_get_dtype..get_dtype s'ZZ%F! "4( (rU)rdrxrur)r6rrr7rrr) fake_outsrrrr s @@rS_fake_get_dtypezKernelTemplate._fake_get_dtype sr''++ i$ /AJK#cllncmmo5KF((*I,?,?,ABF ) Ls&B Nc ||_||_yrk)rd_hash)rgrdr7s rSrzKernelTemplate.__init__ s  rUc|jS)a  entry point to override for templates to ensure a uid e.g. through a prefix the purpose of this is that every KernelTemplate/ExternKernelChoice is unique in the system, but reproducible e.g. restarting pytorch should yield the same id rrls rSuidzKernelTemplate.uid syyrUc|jS)a source hash for a Template. Templates can optionally provide a src hash to make it easier to cache/validate that a template has not changed from one version to another. Override this if that detection is different for your specific Template )r$rls rSsrc_hashzKernelTemplate.src_hash' szzrUc Xg}|j|fi|}|t|dk(r|dSy)z Maybe generates a new ChoiceCaller and returns it, or None if generation fails. kwargs: Additional kwargs to be passed to self.generate() to generate a new ChoiceCaller. NrGr)maybe_append_choicer)rgr temp_choicesrs rSchoice_or_nonezKernelTemplate.choice_or_none2 s> #% ))),A&A >c,/14? "rUc  |j|jdi|y#t$rQ}tj d|t |tj tjk|cYd}~Sd}~wwxYw)a% Maybe generates a new ChoiceCaller and appends it into existing choices. Returns None if success, otherwise returns the error. choices: A list of ChoiceCallers. kwargs: Additional kwargs to be passed to self.generate() to generate a new ChoiceCaller. Nz3Cannot Append Choice: %s. KernelTemplate type is %s) stack_infor) rrmrrXinforMgetEffectiveLevelrOINFO)rgchoicesrrs rSr*z"KernelTemplate.maybe_append_choice> sn  NN=4==262 3"  HHET 002W\\A   H s!$ A>AA93A>9A>c t)zM Generates a ChoiceCaller instance from the given arguments. r)rgrs rSrmzKernelTemplate.generateU s "!rU))rrxrrrrrurx)rrxrur )r!zlist[Buffer] | BufferruzCallable[[str], torch.dtype]rk)rdrxr7rrurvrwr)rr ruzChoiceCaller | None)r2ryrr ruzNotImplementedError | None)rr rur>)rzr{r|r}rrrr"rrr&r(r,r*rmrrUrSrr s >?"%8; *8*8X( %"  ,/ #."rUrc6eZdZdZdZdfd ZddZddZ d ddZ ddZ ddZ ddZ d dd Z dd Z dd Zdd Z dd Z ddZ d dZ d! d"dZxZS)#rz~A ops handler that proxies calls to `kernel` and its handler and returns `CSEVariable`s with correct shape and dtype. cbt|ddlm}||_||_||_y)NrValueRangeAnalysis)r/rr3r8 vr_analysisrparent_handler)rgrr:r8r1s rSrzCSEProxy.__init__d s+ /-/ ,rUc d |jgi t|ji}t}t }t t|}d ddk(r dk(r|j |jnydk(rT dk(rOtjjjjtjdj dn dvrt|}|i |i dvr Jd d  fd } tj | |S) Nmaskedrqro)rqror)rqrorct ttfr n }t ttfr,t dkDrt dttfr n } dz t|tr+dk(r|j ||_|j ||_tjjjtjj| }|j tjjstjj r'|%t#tjj||tjj$rKt'rA|j |j n|}|%t)tjj||tj*r$t-tjj||S)NrrGror3rr)r7rrrrrrr6rrrmrr<rrrrrrrrruntime_triton_nan_assertsr) r var_dtype var_shapecsevarshape_to_checkrprr3rrdr output_idx output_shapes rSdo_csez!CSEProxy._default..do_cse s lT5M:Z(! lT5M: %)|Au >Z("  !OJ![)e#'AG77?'AGXX\\**  "" +F  ! !$f 5##??&&>>'AHH,,fi@##??46171Iy!- 0 0&.I00!((**F3MrU)rzstr | CSEVariablerur)_bound_variablerr:r r#r'rrr6 interpreterrjrr\rrpytreetree_map)rgrdrprrf dtype_handler shape_handlershape_opdtype_oprFrr3rrDrEs ``` @@@@@rS_defaultzCSEProxy._defaultl s@%%%d>#''e  L 0 0}d3H#T4V4L#T4V4L ' '+ ++ 3 3 jvu--rUc ddlm}ddlm}ddlm}t tj|rtjSt tj|rtjSt tjtrtjStjj j|k(r|jjt |jjt s$Jt#|jj|jjj% tjSt&j(rjt+||r^t- fddDrtjS|rJd d}t/t1||}t3|j4||StjS) z If the variable comes from an FX node, we forward the bound we have already computed Else, if the variable when codegen'ing another op, we try to compute its bounds rr7)TritonTemplateKernelrG)CUTLASSTemplateKernelc3:K|]}|jvywrk)r )rrfx_nodes rSrz+CSEProxy._bound_variable.. sV11&Vs) set_indirectrrct|tr |jSt|tjr t |S|Srk)r7rr3rr.rrDs rS arg_to_boundz.CSEProxy._bound_variable..arg_to_bound s2a-88O5::.&q>)HrU)rEr rur )r3r8select_algorithmrQcutlass.kernelrRr7r6rrrvrHr0rjr rdictrMr\rcompute_all_boundsrrrrrr9) rgrdrprr8rQrRrW arg_boundsrTs @rSrGzCSEProxy._bound_variable sm 0;9 ahh 4 5&&( ( ahh 5 6&&( ( amm[ 1&&( (--,, >>T !dkk&@&@&Ldkk88$?  **B ?;;--11';;N;N;PQ Q  & &73Et+LV0UVV"**,,  : c,56J274++T2J? ?""$$rUcpt|trtj|}t|tjsJt ||f|j jdkr|rtj|tj|tj}|j jdk\r0tj|d}tj|||}n|}|s |j nt!j"}|r|j t!j"k7rt|tj$r|j t!t& dz}t!|j|z|j|z}|j jdk\r"|j t!dt&z} || z}|j(j*j-|j(j.|||j0|j2}|j4j7|||} t9|ro|j jdk\ } t|tj$ xs|j j|k } |j(j;| || | | S)Nrrr>)r7rrrr.rMr3rr1r`rrlongrltrrrvNumberrrrrmrrrr:rr&r) rgrrrrstmr_ new_bounds neg_boundspos sympy_var assert_lower assert_uppers rSrzCSEProxy.indirect_indexing s$ dC ==&D$ +?d4j$-??+ ::  a ggc3>>$ #CD::##q(QB))BS1C ,49L9L9NJJJ+"5"5"77tU\\2 !ZZ+vgr*BB ($$t+Z-=-=-D ::##q(**{1f'==C!+c!1J++//** ##!iiii +C''99#tUK 5 ! # 0 0A 56L)$ ==   4'BL KK $ $YlL QrUc>|jj||||Srk)rrrs rSrzCSEProxy.check_bounds1 s{{''dE5AArUc||jjjvr)tjjj |t |tjr|jj||S|jjj}||vr||S|jj||}|jdk(r|jxjdz c_ |SrL)rrrLr6rr`rrTMPrrHrr4r)rgrdrrHouts rSrz CSEProxy.load6 s 4;;??55 5 HH & & * *4 0 udhh /;;,,T59 9kkoo11 ; t$ $kktU+ ==A  KK A %  rUcl||jjj|<|jjr{|tj j vr^|jjj|}|jD]%}||jjj|<'yyyrk) rrrHrjr6rname_to_buffer get_output get_mutations)rgrdrfr other_names rS_update_store_cachezCSEProxy._update_store_cacheG s,1 ##D) ;; # #0F0F(F++**55d;C!//1 @ :? ++J7 @)G #rUc2|jjj||dk7r|j|||tj j vr?|jj|||||jxjdz c_yy)N atomic_add)r[rG) rrr`rqr6rrrrrs rSrzCSEProxy.storeN s{ &&**40 <   $ $T5 1 qww.. . KK  dE5t  < KK ! !Q & ! /rUc<|jj||yrk)rrrs rSrzCSEProxy.device_assert_asyncY s ''c2rUc6|jj|yrk)rrros rSrzCSEProxy.partial_accumulate] s& &&-rUc"|jjj||j|||tj j vr<|jxjdz c_|jj|||SyrL) rrr`rqr6rrrrrs rSrzCSEProxy.store_reduction` so &&**40   u- qww.. . KK ! !Q & !;;..tUEB B /rUc||jxjdz c_|jj||||SrL)rrrrs rSrzCSEProxy.reductionh s4 !!Q&!{{$$UI~uMMrUc<|jj|||Srk)rrrs rSrz CSEProxy.scanr s{{ F;;rUc>|jj||||Srk)rrrs rSrz CSEProxy.sort} s{{ CCrUc D|jj|||||||S)a [Note: Inductor bucketize op] Inputs: ------- values: the values to be bucketized. boundaries: a tuple containing (a) the name of the boundaries tensor (which must be sorted, unless the sorting tensor is present), (b) the length of the tensor in the last dimension (i.e. the length of one set of boundaries), (c) the number of elements in the underlying storage (i.e. the length of the flattened tensor, ignoring striding), and (d) the stride of the tensor in the last dimension. boundary_indices: indices into a flattened version of the boundaries tensor, of the same size and shape as "values". Each index points to the first element in the set of boundaries to be used for the corresponding value. indexing_dtype: the dtype to use when indexing into the boundaries tensor. This must be int64 or int32. This additionally specifies the dtype of the return value. right: see "Details" below. sorter: an optional tuple containing (a) the name of an optional sorting tensor, used to access unsorted boundaries without reordering the boundaries tensor, and (b) the stride of the tensor in the last dimension. The values in the sorting tensor are used as indices into the *last* dimension of the boundaries tensor, with all other indices matching. The size of the sorting and boundaries tensors must be equivalent. sorter_indices: must be present if the sorting array is present; see "boundary_indices" for the equivalent definition for the boundaries tensor. Output: ------- The buckets each value belongs in, within a given set of boundaries. 0 indicates a position before the first boundary, and len(boundaries_set) represents a position after the last boundary. Details: -------- Given a value and a set of boundaries, calculate the bucket that each value belongs to. This works differently in 1-D and N-D cases. for values [[-1, 0, 1, 2], [3, 4, 5, 9]], boundaries [0, 4, 4, 8], right=True return = [[ 0, 1, 1, 1], [1, 3, 3, 4]]. for values [[-1, 0, 1, 2], [3, 4, 5, 9]], boundaries [[0, 4], [4, 8]], right=True return = [[ 0, 1, 1, 1], [0, 1, 1, 2]] Note that in the N-D boundaries case, the shape of "values" and "boundaries" must match in every dimension _except_ the last. When right == False, bucket i refers to range (boundaries[i], boundaries[i+1]]. When right == True, bucket i refers to range [boundaries[i], boundaries[i+1]). Boundaries must be non-decreasing, or a sorter must be provided which would re-index offsets in a non-decreasing order (e.g. the second output of torch.sort(offsets)). Otherwise, the result is undefined. )rrrs rSrzCSEProxy.bucketize s1L{{$$         rU)rz Kernel[Any]r:zOpsHandler[Any])rdrxrpztuple[Any, ...]rrrur )rdrxrpr rr rur>r) rrrrrrrrrurr r)rdrxrfrrurvrkrr )rpr rurvrrrrrr)rzr{r|r}rdrrOrGrrrrqrrrrrrrrr;r<s@rSrr] s D-V.p.%h ; ;; ;  ;  ;zBB&0B9=BFJB B "@SW ' ' * '3> 'FO '  '3.CNNN& N 5 N / N <' <  <( < ! <D'D(D D  D ! D 15-1N N CN & N $ N  N .N +N  N rUr)rRrxrurv)NNNN)rrxr8rr9rr:rr;rr<CustomGraphModulePass | Noner=ConfigModule | Nonerurv)rtorch.device | str | NoneruzOrderedSet[BackendFeature])rr}rSr@rur)rrxruzSchedulingConstructor | None)FF)rrxrWrrXrrur)rrxrur{)rrxrur|rt)rSequence[sympy.Expr]rr~rr~rur)rrxrrrurv)rrxrur)rrxrpr rr rurr)rr(rrrrrurv)rr(rrrrFrurv)rr(rrrurv)rrxrurrr)rur ) __future__rr`r dataclassesenumrrJrOrYrrbrer^abcrrrrrtypingr r r r r rtyping_extensionsrrrrtorch.fxtorch._prims_commonr torch.utilsrrItorch.utils._config_modulertorch.utils._ordered_setrtorch.utils._sympy.functionsrtorch.utils._sympy.numbersrtorch.utils._sympy.printersr_PythonPrintertorch.utils._sympy.symbolrrrtorch.utils._sympy.value_rangesrrrrrdtype_propagationr ops_handlerr!r"shape_propagationr#utilsr$r%r&r'r(r)r*r+r,r-r.r/ virtualizedr0r1r2r3r4r5r6collections.abcr7r8r9r:r;custom_graph_passr<rr=r>r?r@r#rBr$rCrDrErFrrIrJrrMrrxr r0r_logginggetArtifactLoggerrzrM getLoggerrXrT dataclassrWrrrrrrrrr KernelArgTyperr~rr3rr4r5r>r@rOrTrNrZr]r_cacherLrrrrbfloat16rfloat16rrCfloat64int8int16rarruint16ruint64rrrrrrrr>compile IGNORECASErrrrrZ INT_TO_FLOATrrrrrrrrrrrr@rrrReductionCacheKeyrDrxrrrrrrs0rSrsY "    #JJ+ ?)3/,-GOOD:;:    LL$9>>$DD2- B$i$&6%7%GH23u||#JF~~//*E g!=  >/ /(  C  Td=ddN**        999y(725EE T ,.).;";"|:<6;#( AC>CAC >C8=A;?7;04B B,B/B!: B 9 B 5 B.B B4 &T &3 %33$3 %30>3 3U @E   " 8<  -6 wwtU U$U U U; ;&7; ; ,&, NNEKK MM5;;> JJ MM MM JJ KK KK KK KK LL LL LL    u >:,( ( (( (V: :!0:9D: :8 L  L!0 L9G L  L aBaBH=N=&_1_1D"rzz";2==Q  X;#%5z#X;v  * * * 6:r6 ;HH -  r6 ;HH /-  r6;HH /- r6$;HH /- %r60;HH /- 1r6< ;HH *)   =r6L ;HH (0   Mr6X ;HH 66 8  Yr6z ;CC )+  {r6L ;HH 1  Mr6V ;HH 2  Wr6`;HH 1 ar6j;HH 2 kr6t;HH %8$  ur6B ;HH &%   Cr6N;HH %8  Or6Z ;HH &  [r6d;HH + er6p%;HH 88 ) qr6|%;HH 88 ) }r6H%;HH 8 )Ir6R%;HH 8 )Sr6^ ;HH '  _r6h ;HH (  ir6r;HH c  sr6B,;HH ? 0Cr6L,;HH ? 0Mr6X&;HH 9 *Yr6b ;HH *  cr6l);HH D -mr6v);HH D -wr6@);HH D -Ar6J);HH D -Kr6T(;HH C ,Ur6^$1;HH L 5$_r6h$1;HH L 5$ir6r$1;HH L 5$sr6|$1;HH L 5$}r6F';HH B +Gr6P(;HH C ,Qr6Z(;HH C ,[r62rj -#-">*Z  NNN ,XXv #;#;L5 +; T eK,-- /k'/=0 1k\ < <@3Wgo.@3F   S"S"lw ~w G>s( f0