
    ^j                      U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZ d dlmZmZmZmZ d dlmZmZmZmZmZ d dlmZ d dlmZ d dlZd dlZd dlZd dl Zd dl!m"c m#Z$ d dl%m&Z& d d	l'm(Z(m)Z) d d
l*m+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z? d dl@mAZAmBZB d dlCmDZDmEZEmFZFmGZG d dlHmIZI d dlJmKZKmLZLmMZMmNZNmOZOmPZPmQZQ ddlRmSZS ddlTmUZUmVZVmWZWmXZX ddlYmZZZm[Z[ ddlWm\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZk ddl"mlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZz ddl{m|Z|m}Z} erddlWm~Z~ ddlmZ  ed      Z ed      Z eId d!g      Z ej                  e      Zi Zd"ed#<   i Zd$ed%<   i Zd&ed'<    eIej                  j                            Zej                  j                  Zej                  j                  Zej                  j                   Z eIej                  j                            Z eIej                  j                     ej$                  j&                  g      Z eIej                  j                            Zi Zd(ed)<   ej                  j.                  Ze(e)fZdd*Z	 	 	 	 dd+Zdd,Z	 	 	 	 dd-Zdd.Zd/ Z	 	 	 	 	 	 dd0Z eej@                  ejB                  ejD                  ejF                  ejH                  ejJ                  ejL                  ejN                  ejP                  ejR                  ejT                  ejV                  ejX                  ejZ                  g       ej\                  ej^                  ej`                  ejb                  ejd                  ejf                  ejh                  ejj                  ejl                  ejn                  ejl                  ejp                  ejr                  d1Zdd2Zdd3Zdd4Zd5d6	 	 	 	 	 	 	 dd7Zd8 Z	 	 	 	 	 	 dd9Zdd:Z	 	 	 	 	 	 	 	 	 	 	 	 dd;Z	 	 	 	 	 	 dd<Z	 	 	 	 	 	 	 	 	 	 dd=Zd5e:j                  d5ef	 	 	 dd>Zd? Z	 	 d	 	 	 	 	 	 	 dd@ZdA ZɐddBZ	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddCZ edDe`E      Z	 	 d	 	 	 	 	 	 	 ddFZ	 	 	 	 	 	 	 	 	 	 ddGZ	 d	 	 	 	 	 	 	 ddIZ eej$                  j&                  dJ      dK        Z eej                  dJ      ddL       Zd5dMddNZ eej                  j                  dJ      ddO       Zd5d5dPddQZ eej                  dJ      dddR       ZddHe:j                  d5ddd5d5df	dSZ ewdTe:j                  dU        eej                  dHdV      ddW       ZdX Z eݫ        	 	 ddYZ eej                  d5dV      dZ        Z eej                  d5dV      d[        Z eej                  ej                  ej                  ej                  ej                  g      d\        Z eed]      r  eej                        e        eej                  dJ      dd^       Z eej                  dJ      dd_       Z eej                  g      dd`       Z eej                        da        Z eej                        db        Z eej                        dc        Z eej                        dd        Z eej                  j                        de        Z eej                        df        Z eej                  dJ      dg        Z eej                  dJ      dh        Z eej                  dJ      di        Z eej                        dj        Z eej                  dJ       eej                  dJ       eej                  dJ      ddk                     Z eej                  dJ      dl        Zdm Z eej                  dJ      d d e
j                  ddHfdn       Z eej@                  dJ      	 ddHdodp       Z eej                  dJ      ddq       Z eejB                  dJ      ddr       ZddsZ eej                   dJ      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddt       Zdu Z eej                  j                        dv        Z eej
                  j                        dw        Z eej                  dJ      ddx	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddy       Z eej                  j                  dJ      	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddz       Z
 eej                  j                  dJ      ddx	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd{       Z eej                  j                  dJ      	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd|       Z eej                  j                  dJ      ddx	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd}       Zdd~Z eej"                        dd       Z eej$                  dJ      dǐdd       Z eej&                  dJ      dǐdd       Z eej(                  dJ      dǐdd       Z eej*                  dJ      d        Z eej,                  dJ      dd       Z eej.                  dJ      dd       Z eej0                  dJ      dd       Z eej2                  dJ      d        Z eej4                  dJ      d        Z eej6                  dJ      d        ZddZ eej:                        dd       ZddZej>                  d        Z dddZ!dddZ"dʐddZ#	 	 	 	 ddZ$d Z% eej                  jL                  jN                  dJ      d        Z' eejP                  dJ      d        Z( eejR                  dJ      d        Z) eejT                  jV                  dJ      d        Z, eejZ                        d        Z- ej\                  d      d        Z/d Z0 eejb                  j                        Z2 eejb                  jf                        Z4 eejj                  j                        Z6 eejj                  jf                        Z7 e$ejp                          e$ejr                  dH        e$ejt                  dH        e$ejv                  dH        e$ejx                          e$ej                  jz                  j|                  j                          e$ej                  jz                  j~                  j                          e$ej                  jz                  j                  j                          e$ej                  jz                  j                  j                          eejb                        d        Z1 eejj                        d        Z5 eeVj                  dJ      d        ZC eeVj                  dJ      dd       ZE eeVj                  dJ      d        ZG eeVj                  dJ      d        ZIddZJ eeVj                  dJ      d ejh                  d	 	 	 	 	 	 	 	 	 dd       ZL e$eVj                          e$eVj                          eeVjp                  dJ      d d	 	 	 	 	 	 	 	 	 dd       ZOddZPddZQddZR	 	 	 	 	 	 	 	 ddZS eej                  j                  dJ      d5d5ddd	 	 	 	 	 	 	 	 	 	 	 	 	 dd       ZT eej                  j                  e:j                  J      d5d5d	 	 	 	 	 	 	 dd       ZVd ZXd ZYd ZZd Z[d Z\d Z]d Z^d Z_d Z` eejP                  j                  e`       d Za e$ej                          e$ej                  dH        e$ej                          e$ej                  d5        e$ej                  j                  d5        e$ej                  eZ        e$ej                  d5        e$ej                  d5       ej                  j                         r e$ej                  d5       ej                  j                         r e$ej                  d5        e$ej                          e$ej                          e$ej                          e$ej                  j                          e$ej                  j                          e$ej                          e$ej                  j                          e$ej                  j                          e$ej                  j                          e$ej                          e$ej                  d5        e$ej                  eY        e$ej                  eY        e$ej                  eZ        e$ej                  eY        e$ej                  eZ        e$ej                   eZ        e$ej                  eZ        e$ej                          e$ej                          e$ej                          e$ej                          e$ej
                          e$ej                          e$ej                  eY        e$ej                          e$ej                          e$ej                          e$ej                          e$ej                          e$ej                          e$ej                          e$ej                          e$ej                   eZ        e$ej"                          e$ej$                          e$ej&                  eZ        e$ej(                          e$ej*                  j                  d5        e$ej,                          e$ej.                          e$ej0                          e$ej2                          e$ej4                          e$ej6                          e$ej8                          e$ej:                          e$ej<                          e$ej>                          e$ej@                          e$ejB                          e$ejD                          e$ejF                          e$ejH                          e$ejJ                          e$ejL                          e$ejN                          e$ejP                          e$ejR                          e$ejT                  jV                          e$ejX                          e$ejZ                          e$ej\                          e$ej^                          e$ej`                          e$ejb                          e$ejd                          e$ejf                  j                          e$ejh                  j                  d5        e$ejj                  j                  d5        e$ejl                  j                  d5        e$ejn                  j                  d5        e$ejp                  j                  d5        e$ejr                  j                  d5        e$ejt                  eY        e$ejv                  jx                  jz                          e$ejv                  jx                  j|                          e$ejv                  jx                  j~                          e$ejv                  jx                  j                          e$ej                  j                  d5        e$ej                  j                  d5        e$ej                          e$ej                          e$ej                  eZ        e$ej                          e$ej                          e$ej                          e$ej                          e$ej                          e$ej                  j                  ead5        e$ej                  j                  ead5        e$ej                  j                  ead5        e$ej                  j                  d5        e$ej                  j                  ead5        e$ej                  j                  ead5        e$ej                  j                  ead5        e$ej                  j                  ead5        e$ej                  j                  ead5        e$ej                  j                  ead5        e$ej                  j                  ead5        e$ej                  j                  ea        e$ej                  j                          e$ej                  j                  ea        e$ej                  j                  ea        e$ej                  j                  ea        e$ej                          e$ej                  j                  dH        e$ej                  j                  eZ        eej                  dJ      dd       Z eej                        ddd       Zd Zd Zd Z eed      r  eej                        e        eej                        d        Z eej                  j                  dJ      	 dddddd5ddÄ       Z eej                  dJ      ddĄ       Z eej                  dJ      ddń       ZdƄ Z eej                  ej                  ej                  g      dddd5dǜdȄ       Z eej                        ddɄ       Z eej                        dʄ        Z eej                        d˄        Z eej                        d̄        Z eej                        	 dddd͜d΄       Zdτ ZdЄ Zdф Z eej                  ej                  g      ddddddҜdӄ       ZdԄ ZdՄ Z  eej                         ee            Z e ed            Z e ed             Zdք Z eej                        dddddלd؄       Z eej                        dddddלdل       Z eej                        dddddלdڄ       Z eej                  j                        dۄ        Z eej                  ej                  g      d܄        Z eej                  dJ      dd݄       Z eej                   dJ      ddބ       Z d߄ Z	 ddZd ZddZ eej
                  dJ      d        Z eej                  dJ      d        Z eej                  dJ      dd       Z eej                        dd       Zd Z	d Z
 eej                  dJ      dd       Z eeVj                  dJ      dd       ZddZ eej                  j                  d5      Z eej                   j                  d5      Z eej                  dJ      d        Z eej                   dJ      d        Zed        Z eej&                  dJ      dd       Z eej(                  dJ      dd       ZddHd	 	 	 	 	 	 	 ddZ eej,                  dJ      dddd       Z eej.                  dJ      dd       Z eej0                  dJ      dd       Z eej2                  dJ      dd       Z eej4                  dJ      dHddd       Z	 	 d	 	 	 	 	 ddZ eej8                  j                        ddd       Z eej:                  j                        ddd       Z eejV                  j                        	 d	 	 	 dd       Z eejX                  j                        	 d	 	 	 dd       Z eej<                  j                        	 	 	 d	 	 	 	 	 dd        Z eej>                  j                        	 	 	 d	 	 	 	 	 dd       Zd Z  eejB                  j                        d        Z!	 	 	 	 	 	 	 	 ddZ"	 	 	 	 	 	 	 	 ddZ# eejH                  dJ      dd       Z$ddZ%ddZ&dd	Z'	 dd
Z(dddZ)d Z*dddZ+d Z, eejZ                  dJ      	 dd       Z-	 	 	 	 	 	 	 	 	 	 ddZ. eej^                  dJ      d        Z/d Z0 eejN                  dJ      	 	 	 	 dd       Z eejP                  dJ      	 	 	 	 dd       Z eejR                  j                  d5      Z1 eejR                  dJ      d        ZddZ2d Z3d Z4d Z5 eejl                  j                  d5      Z7 eejl                        d        Z6 eejp                  j                  d5      Z9 eejp                        d        Z8d Z: eejv                        d        Z; eejx                        d         Z<d! Z= eej|                  j                        	 dd"       Z> eejD                  dJ      	 	 	 	 	 dd#       Z eej~                  dJ      	 	 	 	 	 dd$       Z? eej                  j                  d5       eejD                  j                  d5       eej~                  j                  d5      gZAd% ZB eejF                  j                  d5      ZC eejF                  dJ      	 dd&       Z eej                  j                  d5      ZE eej                  dJ      	 dd'       ZDd( ZFdd)d*ZG	 d	 	 	 dd+ZH	 	 	 	 dd,ZId- ZJ eej                        ddd.d/       ZKd0 ZLd1 ZMd2 ZNd3 ZO eej                  ej                  g      ddd5d4d5       ZQ eej                        ddd5d4d6       ZRd7 ZSeːd8        ZT eej                  j                  d5      ZW eej                  j                  d5      ZY eej                  j                  d5      Z[ eej                  dH9      d:        ZUdd;Z\ eej                        d<        Z] eej                  dJ      dd=       Z^eːd>        Z_eːd?        Z`eːd@        Za eej                  dH9      ddA       Zc eej                  gdH9      dB        ZdddCZe eej                  gdH9      dD        Zf eej                  ej                  j                  gdHe:j                  V      dE        Zb eej                  ej                  gdH9      dF        Zh eej                  ej                  g      ddd.dG       Zj eej                  j                        Zl eej                  j                        Zn eej                  j                        Zp eej                  j                        Zr eej                  j                        Zt eej                        ddH       Zk eej                        ddI       Zm eej                        dJ        Zo eej                  dJ      ddK       Zq eej                  dJ      ddL       Zs eej                        ddd.dM       ZuddNZv eej                        ddO       Zx eej                  dJ      ddP       Zz eej                  dJ      ddQ       Z|  eej                         eHdR               eej                         eHdS            Z  eej                          eHdT            Z  eej                         eHdUejd                  V            Z  eej                         eHdWejd                  V            Z eHdX      Z eHdY      Z eIdZ      Z eId[      Z eej                  dHdHd\]      Z eej                  j                  d5      Z eej                  j                  dJ      ddd5d^d_       Z eej                  j                  dJ      dd`       Z eej                  j                  d5      Z eej"                  j                  d5      Z eej&                  j                  d5      Z eej&                  j*                  d5      Z eej.                  j                  d5      Z eej&                  j                  dJ      da        Z eej&                  j*                  dJ      ddb       Z eej.                  j                  dJ      ddc       Z eej                  j                  dJ      ddd       Z eej"                  j                  dJ      dde       ZddfZddgZ eej<                        Z eej>                        Z eej@                        Z eejB                        Z eejD                        Z eejF                        Z eejH                        Z eejJ                        Z eejL                  dHh      Z eejN                  dH9      ddidj       Z eejP                  dH9      ddidk       Z eejR                  j                  edHdlm      Z eejV                  j                  edHdlm      Z eejZ                          eej\                          eej^                        Z eej`                        Z eejb                        Z eejd                  dno      Z eejf                        Z eejh                        Z eejj                        Z eejl                          eejn                        Z  eejp                  e:j                  J      e        eejr                          eejt                          eejv                          eej                          eejx                  ddHejp                  p      Z eejz                  ddHejp                  p      Z eej|                  ddHejp                  p      Z eej~                  ddHejp                  p      Z eej                        Z eej                        Z  eej                        e         eej                        e        eej                        Z eej^                        Z eej                        Z eej                          eej                  dqo      Z eej                          eej                        dr        Z  eej                        eī        eej                  ejp                  V        eej                  ejp                  V        eej                  ejp                  V        eej                  ejp                  V      Z eej                  ejp                  V        eej                  ejp                  V        eej                          eej                          eej                          eej                          eej                          eej                          eej                          eej                          eej                          eej                          eej                          eej                          eej                          eej                          eej                          eej                         ddslmZmZ dt ZeD ]S  Z eee      D ]  \  ZZZ eِeeeeu         eee      D ]  \  ZZZ eِeeeeu        U  eej                  j                  edHh      Z eej                  j                  edHh      Z eej                  j                  edHh        eej                  j                  ed      Z eej                  j                  ed        eej                  j                  ed      Z eej                  j                  edHh        eej                  j                  edHh        eej                  j                  eī        eej                  j                  e        eej                  j                  eU        eej                  j                  eU        eej                  j                  eU        eej                  j                  eb      Z eej                  j                  eb        eej                  j                  eb      Z eej                  e        eej                  e        eej                  j                  e        eej                  j                  e        eej                  j                  e        eej                  j                  e        eej                  j                  e        eej                  j                  e        eej                  j                  e        eej                  j                  e        eej                  eū        eej                  eǫ        eej                  eޫ        eej                   eݫ      Zdv Z eej                  j                  ej                  j                  e        eej                  j                  ej                  j                  e        eej                  j                  ej                  j                  e        eej                  j                  ej                  j                  e        eej
                  j                  ej                  j                  e        eej
                  j                  ej                  j                  e        eej                  j                  ej                   j                  e        eej                  j                  ejR                  j                  e        eej                  j                  ejV                  j                  e       dw Z	 e	ej                  e        e	ej                  e        e	ej                  e        e	ej                  e        e	ej                  e        e	ej                  e        e	ej                   e        e	ej"                  ed        e	ej$                  j                  eb        e	ej$                  j&                  ec        e	ej(                  e        e	ej*                  e        e	ej,                  e        e	ej.                  e        e	ej0                  e        e	ej2                  e        e	ej4                  e         eej6                        e         eej8                        e         eej:                        e         eej<                        e         eej>                        e        e	ej@                  ej6                          e	ejB                  ej8                          e	ejD                  ej:                          e	ejF                  ej<                          e	ejH                  ej>                          eejJ                        ddx       Z% eejL                  jN                        dy        Z& eejP                  jN                        dz        Z( eejR                        d{        Z)d| Z* eAjV                         D ]#  \  Z,Z- e eBe,            e-d}d~       Z.%  eej^                        d        Z/ eejZ                        d        Z0 eej                  jb                  jd                        d        Z3 eej                  jh                  jj                        d        Z5 eej                  j                  jl                  jn                        d        Z8 eej                  jr                  d      r; eej                  jr                  j                  j                        d        Z: eej                  j                  jv                        ddd       Z;d dl<m=Z=  e$e=        ee-      d        Z> eej                  j~                  j                  dJ      	 	 dd       Z@ eej                  j~                  j                  dJ      dd       ZA  eej                  j~                  j                  dJ       ej                  eAdH              eej                  j~                  j                  dJ      dd       ZD eE       ZFddZGd dlHmIZI  eƐeIdJ      d        ZJ eej$                  j                  dJ      dddd       ZL ee+dJ      dd       ZM eej                  j                   j                  j                        d        ZN eej                  j                   j                  j                        d        ZO eej                  j~                  j                  dJ      d        ZPddlQmRZRmSZS  eR         eS         eeVj                  dJ      d        ZT eeVj                  dJ      d        ZV eej$                  j                  dJ      dHddd       ZXddlTmYZY  eSeY       ddlTmZZZ  eZj                           eZj                          ddlTm]Z]m^Z^  e]j                          ej                  dd       Zay(      )annotationsN)defaultdict)CallableIterableMappingSequence)AnycastTYPE_CHECKING	TypeGuardTypeVar)	ParamSpec)patch)counters)SavedForBackwardsAOTOutput#SavedForBackwardsNoVcCheckAOTOutput)associative_scan_op)triton_kernel_wrapper_mutation)FakeScriptObject)is_opaque_value)get_layout_constraint_tag)canonicalize_dimcanonicalize_dimscheckdtype_to_typeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDget_computation_dtypeis_boolean_dtypeis_float_dtypeis_integer_dtypeNumber)magic_methodsmethod_to_operator)free_unbacked_symbolshas_free_unbacked_symbolsresolve_unbacked_bindingsSymTypes)
OrderedSet)CeilDivFloorDivIdentityMaxMinModModularIndexing   )import_submodule   )configinductor_primsirtest_operators)decompositionsget_decompositions)BaseView	DtypeView
ExpandViewIndexingConstantIRNode	is_triton
MutableBoxOnlineSoftmaxReductionops_wrapperPermuteView	Pointwise	ReductionSqueezeView	TensorBoxvalidate_irView)ceildivconvert_symint_to_exprdecode_device
is_dynamicis_gpuis_nvidia_sm100_or_lateris_pointwise_useis_triton_fp8_dtype_supportedis_view,needs_fallback_due_to_atomic_add_limitationspad_listlike#register_op_dtype_propagation_rules#register_op_requires_libdevice_fp64sympy_productuse_scatter_fallback)opsV)BaseConstant)ReductionType_T_Pztorchvision::roi_alignzaten::index_add2dict[Callable[..., Any] | str, Callable[..., Any]]	loweringsz/dict[torch._ops.OpOverload, Callable[..., Any]]user_loweringsz6dict[torch._ops.OpOverload, Callable[..., Any] | None]_maybe_layout_constraintsz2dict[torch._ops.OpOverload, torch._ops.OpOverload]inplaceable_foreach_opsc                     t         j                  j                  j                  D ]7  } | j                  D ]&  }|j                  dk(  r|j
                  t        v r%  y 9 y)Ncall_functionTF)rZ   graphcurrent_nodeusersoptargetforeach_ops)nodeusers     c/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/lowering.pycur_node_has_non_foreach_usersro      sS    $$** JJ 	DGG.DKK;4N	
     c                f   t        t              }d}t        |       D ]  \  }}t        |t              sd}|f}t        |  xs t        j                  }d }|D ].  }t        |t              s|j                  j                         } n |J d       |r|\  }|||f   j                  ||f        |S )NFTz.foreach op should have at least one tensor arg)r   list	enumerate
isinstancer   rM   r4   #combo_kernel_foreach_dynamic_shapesrG   data
get_deviceappend)	arg_pairsoutunpack_argsiargsuse_foreachdevicets           rn   group_foreach_argsr      s     d
CKY' 54$)K7DD!!OV%O%O 	  	A!Y'**,	 !S#SS!GTV[!"))1d)45  Jrp   c                    t        | t        j                  j                        syt	        | d      x}rt        |      S | t        v r	t        |    S y)zHGet layout constraints. Returns None if there are no layout constraints.NF)with_default)rt   torch_ops
OpOverloadr   tag_to_layout_constraintrb   )fnmaybe_layout_tags     rn   maybe_layout_constraintsr      sO    b%**//04ReLLL'(899	&&(,,rp   c                |   | t         j                  j                  j                  k(  rt        S | t         j                  j                  j
                  k(  rt        S | t         j                  j                  j                  k(  rt        S | t         j                  j                  j                  k(  ry t        d|        )NzUnknown layout constraint tag: )r   _CTagneeds_exact_stridesconstrain_to_fake_tensorsneeds_contiguous_stridesrequire_contiguous_stridesneeds_fixed_stride_orderconstrain_to_fx_stridesflexible_layoutAssertionError)tags    rn   r   r      s     ehhll...((
ehhll333))
ehhll333&&
ehhll***
:3%@
AArp   c                $    | st        d|       y )Nzinductor does not support NotImplementedErrorcondmsgs     rn   
assert_nyir      s    !$>se"DEE rp   c                :   t        | t        t        t        t        f      r| D cg c]  }t        |       c}S t        j                  |        t        | t        j                  j                        r$t        j                  | j                                y y c c}w N)rt   rr   settupler)   add_needs_realized_inputsneeds_realized_inputsaddr   r   OpOverloadPacketupdateop_overloads)r   xs     rn   r   r      sm    "tS%45689)!,99b!"ejj112$$R__%67 3 :s   Bc                    t        | t        j                  j                        r| j	                         D ]  }|t
        |<    y |t
        | <   y r   )rt   r   r   r   r   rb   )r   
constraintoverloads      rn   add_layout_constraintr      sF     "ejj112) 	=H2<%h/	= )3!"%rp   )r   r3   r1                     	   
         c                b    t        | t              s| S | t        v sJ d|  d       t        |    } | S )Nzid z missing from DTYPE_ID_LOOKUP)rt   intDTYPE_ID_LOOKUPdtypes    rn   decode_dtyper     s=    eS!O#Os5'1N%OO#E"ELrp   c                    t        | t              r4t        | j                               xs t	        | j                               S t        | t
        j                        r| j                  du S t        | t              S NT)	rt   rG   r!   	get_dtyper   sympyExpr
is_integerr   r   s    rn   is_integer_typer   "  sX    !Y.Q2B1;;=2QQ	Auzz	"||t##!S!!rp   c                t    t        | t              rt        | j                               S t        | t              S r   )rt   rG   r   r   boolr   s    rn   is_boolean_typer   +  s*    !Y..!T""rp   F)return_compute_dtypec                f    dd}|D cg c]
  } ||       }}t        |d| i\  }}|r|S |S c c}w )Nc                    t        | t        t        j                  f      r| S t	        | j                               }t        j                  dg|z  | j                               S )Nr3   r   )	rt   r"   r   Basiclenget_sizer   zerosr   )inpdims     rn   construct_inputz+get_promoted_dtype.<locals>.construct_input7  sG    cFEKK01Jclln%C;;sSy@@rp   type_promotion_kind)r   r	   returnr	   )r   )r   r   r}   r   arginpscompute_dtyperesult_dtypes           rn   get_promoted_dtyper   2  sP    
A -11SOC 1D1"4	##6#M< 1=BlB	 2s   .c                   t        | t        t        f      s| g} nt        |       } t        |       D ]V  }t        |t        j                  j
                        s(|j                         D ]  }|t        vs| j                  |        X | S r   )	rt   rr   r   r   r   r   r   r`   rx   )aten_fnr   other_fns      rn   get_overloadsr   F  sx    ge}-)w-7m -b%**556OO- -9,NN8,-- Nrp   c                    t        | t        j                  j                        r|| j                  v S t        | t        j                  j
                        r|| j                         v S yNF)rt   r   r   r   _qualified_op_namer   name)ri   	namespaces     rn   in_namespacer   U  sQ     "ejj112B1111	B

--	.BGGI%%rp   c           	         t        | j                  t        j                        rt	        | j                               r| S t        j                  j                  j                  | j                               }| j                         }|u|j                  dk(  rf||k7  rat        |      dk(  st        |      dk(  rE|d   dk(  r=t        t        j                  t        j                  j!                  | |d                  S | S )zB
    Copy cpu scalar if doesn't not match with given `device`
    cpur   r3   F)rt   rv   r6   ReinterpretViewr&   r   rZ   rf   sizevarsguarding_hints_or_throwrw   typer   rG   
StorageBox
DeviceCopycreate)r   r   size
cur_devices       rn   maybe_copy_cpu_scalarr   _  s     affb0015N	

6 7733AJJLADJOOu$& Y!^D	Q47a<r}}';';Az5'QRSSHrp   c           	         t               D cg c]  \  }}t        |t              s| }}}j                         D 	cg c]  \  }}	t        |	t              s| }
}}	|s|
s fS |s|r|rt        j
                  nl D cg c]0  }t        |t        t        j                  f      st        |d      r|2 }}|j                  d j                         D               t        |d|i|r |d      n|
d      j                         |D ]  }t         |          |<    |
D ]  }t        |         |<    d	fd} D cg c]
  } ||       c} j                         D 	ci c]  \  }}	| ||	       c}	}|r1t        t!        t#        j$                   fd|D        fd|
D                     }t!        |d   j'                               }t)        ||dt+        |             D ]
  \  }}| |<    t)        |
|t+        |      d       D ]
  \  }}||<    t-        t+                     D ]<  }t         |   t.        j0                        s!t3        j4                   |   |       |<   > D ]<  }t        |   t.        j0                        s!t3        j4                  |   |      |<   >  fS c c}}w c c}	}w c c}w c c}w c c}	}w )
zB
    Transforms arguments for broadcasting and type promotion
    r   c              3  :   K   | ]  }t        |d       s|  yw)r   N)hasattr).0as     rn   	<genexpr>z!transform_args.<locals>.<genexpr>  s     !T7@S!!Ts   r   r   c                    t        | t              rt        |       S t        | t        j                        r"t        j                  | j
                        S | S )Nvaluer   r   )rt   rG   to_dtyper6   Constantr   )r   r   r   s    rn   promoteztransform_args.<locals>.promote  sD    #y)U++C-{{%OO
rp   c              3  (   K   | ]	  }|     y wr    )r   r|   r}   s     rn   r   z!transform_args.<locals>.<genexpr>  s     3T!W3   c              3  (   K   | ]	  }|     y wr   r   )r   kkwargss     rn   r   z!transform_args.<locals>.<genexpr>  s     71VAY7r   N)r   r	   r   r	   )rs   rt   rG   itemsr   r   r"   r   r   r   extendvaluesr   rw   r   broadcast_tensorsrr   	itertoolschainr   zipr   ranger6   r   r<   r   )r}   r   	broadcastr   convert_input_to_boolr|   r   args_indicesr   vkwargs_indicesr   promoting_argsr   broadcastedr   r   r   s   ``              @@rn   transform_argsr  s  s    #,D/N$!QZ95MANLN$*LLNODAqjI6NaONOV|3 JJE
 a&%++!6771g;N N  !!!TV]]_!TT&$7E &2Da!vnQ>O7P
*, 	  	=A+DGV<DG	=   	AA-fQi@F1I	A	 %))q
),2LLN;DAq!WQZ-;'3l377
 KN++-.k2EC4E&FG 	DAqDG	C4E4G(HI 	DAqF1I	 s4y! 	;A$q'2;;/$++DGT:Q	;  	?A&)R[[1&--fQi>q		? <I OO> *;s(   KKK	K	
5KK4Kc                    t        j                        dfd       }t        |       }t        j	                  |       t
        j	                  t        j                  ||             |S )a  
    Add a foreach lowering to lowerings dict.

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c                 .     | i |}t        |       |S r   )rH   )r}   r   rz   	decomp_fns      rn   wrappedz+_register_foreach_lowering.<locals>.wrapped  s    ((C
rp   )r}   r	   r   r	   r   r	   )	functoolswrapsr   rk   r   r`   dictfromkeys)r   r  r  aten_fnss    `  rn   _register_foreach_loweringr    sX     __Y  
 W%Hx T]]8W56Nrp   c                     t        j                         fd       }t                |j                  t        j                   |             |S )a  
    Add a lowering to lowerings dict

    Arguments:
        aten_fn: torch.ops.aten.* fn we are lowering
        decomp_fn: alternate implementation on our IR
        broadcast: True to apply broadcasting to tensor inputs
        type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
        convert_input_to_bool: some logical ops require inputs are converted to bool
    c                 P   t        |       } t        |      }d}t        |       dk(  r)t        | d   t         t        f      rd}t        | d         } t        d D              st        d |D              rJ d       t        | |      \  } }|r| g}  | i |}t        |       |S )NFr3   r   Tc              3  H   K   | ]  }|t         v xs t        |d         yw)_c10d_functionalN)	fallbacksr   )r   r   s     rn   r   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>  s)      
JLR9_DR1C DD
s    "c              3  &   K   | ]	  }|d k(    yw)rz   Nr   r   r   s     rn   r   z6_register_lowering.<locals>.wrapped.<locals>.<genexpr>   s     6!1:6   zout= ops aren't yet supported)	rr   r  r   rt   r   allanyr  rH   )	r}   r   unpackedrz   r   r	  r
  r  r   s	       rn   r  z#_register_lowering.<locals>.wrapped  s    t*!%ft9>ja4-@HQ=D 
PW
 
 6v66W8WW6%&)%8:O
f 6D((C
rp   )r  r  r   r   r  r  )r   r  r	  r   r
  lowering_dictr  s   `````  rn   _register_loweringr'    sJ    & __Y  6 G$Gw89Nrp   c                @    t        j                  t        | ||||      S )z+
    Shim to support decorator syntax.
    )r	  r   r
  r&  )r  partialr'  )r   r	  r   r
  r&  s        rn   register_loweringr*    s)     /3# rp   c                   t        |      }| r| |k(  r|S g }t        j                  t        |       t        |      t        j
                  j                        D ]  \  }}t        j                  j                  j                  |      r|j                  |       Bt        j                  j                  j                  |      r|j                  |       }t        j                  j                  j                  ||       t        t	        j                  |      j                        t        t	        j                  |      j                        k  r|j                  |       |j                  |        t        t        |            S )z
    Broadcasting logic based on symbolic shapes.

    We give the shapes 0 and 1 concrete values, while all other shapes
    are symbolic sympy formulas.
    )	fillvalue)r   r  zip_longestreversedr   SOnerZ   rf   r   is_size_one_or_falserx   check_equalsr   expandfree_symbols)r   boutputr   ys        rn   broadcast_symbolic_shapesr8  )  s    	aAQF%%hqk8A;%''++V 
!177003MM!WW2215MM!GG))!Q/5<<?//03u||A7S7S3TTa a 
! &!""rp   c                  	 |	|J d       ||t         j                  }t        d | D              s| S t        d | D              r*|xs t	        | d|ifd}| D cg c]
  } ||       c}S t        d | D              }|j                         	|t        j                  k(  r(	t        j                  t        j                  fv r	fd}nd }g }| D ]  }t        |t        t        f      rg|j                  t        j                   t#        j$                   ||      	|j'                         	      t)        |j+                                            t        |t,        j.                        rW|j                  t        j                   t1        |	|j'                         
      t)        |j+                                            |j                  |        |S c c}w )NzEonly one of override_return_dtype or type_promotion_kind may be givenc              3  f   K   | ])  }t        |t        j                  t        t        f       + y wr   )rt   r   r   r   floatr!  s     rn   r   z$promote_constants.<locals>.<genexpr>O  s"     HAz!ekk367H   /1c              3  f   K   | ])  }t        |t        t        t        j                  f       + y wr   )rt   r   r;  r   r   r!  s     rn   r   z$promote_constants.<locals>.<genexpr>Q  s"     
D:a#uekk23
Dr<  r   c                    t        | t        j                        r!t        j                  | t        d             S t        j                  | t        d             S )Nindexr   r   r   )rt   r   r   r6   r=   rL   r   r   r   s    rn   
const_funcz%promote_constants.<locals>.const_funcX  sJ    !U[[)**5t1D  {{%d@STTrp   c              3  l   K   | ],  }t        |t        t        t        j                  f      s)| . y wr   )rt   rG   r<   r6   r   r!  s     rn   r   z$promote_constants.<locals>.<genexpr>a  s$     WA:a)Z1U#VaWs   *44c                N    t        j                  |       j                         S Nr   )r   tensoritem)r  tensor_dtypes    rn   <lambda>z#promote_constants.<locals>.<lambda>i  s    %,,q"E"J"J"L rp   c                    | S r   r   )r  s    rn   rI  z#promote_constants.<locals>.<lambda>k  s    ! rp   r   r?  )r   DEFAULTr$  r#  r   nextr   r   r   bfloat16float16rt   r   r;  rx   r<   r   r6   r   get_device_or_errorrr   r   r   r   r=   )
inputsoverride_return_dtyper   rB  r   ex_round_scalarrz   r   rH  s
           @@rn   promote_constantsrT  C  s   
 !(,?,G OG $)<)D=EEHHH

DV
DD% 
);*
 !4*
	U (..!
1..	WW	WB<<>L 

*|@ 0 M#
C a#u&JJ!!KK+A.*!557
 '	 5;;'JJ!!$|B<R<R<T '	 JJqM-0 JM /s    Gc                    t        | |t        j                        | j                         |j                         fd}t	        j
                  | j                         || j                               S )zBCompute a + alpha * b using FMA for CUDA floating-point precision.r   c                     |       } |       }t        t        j                        rt        j                        }nt        j
                        }t        j                  |||      S r   )rt   r   r   rY   
index_exprconstantfma)idxa_valb_val
alpha_expra_loaderalphab_loaderr   s       rn   inner_fnz%_add_with_alpha_fma.<locals>.inner_fn  sV    eU[[)u5JeU3Jwwuj%00rp   r   r   rb  ranges)r   r   rK  make_loaderrD   r   rw   r   )r   r5  r`  rb  r_  ra  r   s     ` @@@rn   _add_with_alpha_fmarf    sf    		;CCE
 }}H}}H1 ||~zz|	 rp   c                v   | yd}| j                   D ]  }|j                  dk7  sd} n |sy| j                   D ]~  }|j                  dk7  r|j                  j                  d      }|1t	        j
                  |j                  d         }t        ||      D ]  \  }}|| u st        |t              s  y  y)NFr6  Tdescr   )
rh   ri   metagetpytreetree_leavesr}   r  rt   &_SAVED_FOR_BACKWARDS_OUTPUT_DESC_TYPES)rl   has_non_output_userrm   descsoutput_valuesoutput_valuerh  s          rn   _has_saved_lowp_output_userr    s    |

 77h"&
 

 77h		f%=**499Q<8"%mU"; 	L$t#
<) 		 rp   c           	     2     ddd fd}|S )zcWraps a pointwise fn and returns a function representing the pointwise in
    the define-by-run IR.N)r`  c           	       
 t        d D              r	rJ   S t              r| | dk7  rrt        d   t              rnd   j	                         }d   j                         j                  r>t        j                  j                  s$|"|j                  dk(  rt        d   d   |       S t              t        d   |       d<   n| J D cg c]  }|j                          c}d   j                         xs d   j                         
dd  D ]`  }t        |t         j"                        rt%              t%        |j                               k(  rDJ d d d|j                                  t        j&                  t        j(                  ft*        j,                  t/        t*        j,                  dd       nd }|d uxr, |j0                  d uxr |j0                  j3                  d	d
      t4        j6                  xr  xr 
v xr t9        |      }xs |xr 
v 
f	d}sSd }D ]7  }	t;        |	j	                         j                        s'|	j	                         } n |sd   j	                         }xs }t=        j>                  |
|      S c c}w )Nc              3  V   K   | ]!  }t        |t              xr t        |       # y wr   rt   r>   r?   r   r   s     rn   r   z0make_pointwise.<locals>.inner.<locals>.<genexpr>  s(      /
;>JsF#6	#6/
   ')r3   r   cudazndim mismatch  rg   low_precision_pointwise_barrierFc                
  	 t        |       t              k(  sJ d|  d        t        j                  k(  r D cg c]
  } ||        c} S g }t              D ]e  \  }} ||       }|   j	                         }	r2|v r.t        j                  ||d      }t        j                  ||      }|j                  |       g  
| }r.t        j                  |d      }t        j                  |      S |S c c}w )Nzwrong ndim r{  F)use_compute_types)r   r   r   rs   r   rY   r   rx   )r@  loadinputs_loaded	inp_indexrz   	inp_dtypedowncastr   emulate_output_castemulate_precision_castsr   rP  loaders	low_pr_fpoverride_fn_when_input_boolrd  s          rn   rb  z/make_pointwise.<locals>.inner.<locals>.inner_fn  s
   u:V,LE7!F8.LL,

"'B'N2W4UTT%[4UVV "'0'9 .OItu+C &y 1 ; ; =I.9	3I#&<<YRW#X!ll8Y?!((-. -(&  #||C%PH<<%88
# 5Vs   D rc  ) r$  rT  rt   r>   rw   r   is_floating_pointr   versionhipr   rf  rr   mulre  r   r6   r[   r   rM  rN  rZ   rf   getattrri  rj  r4   (emulate_precision_casts_on_saved_tensorsrr  rN   rD   r   )r`  rP  
inp_devicer   otherrg   truncate_saved_outputrb  r   r|   r   r  r  r  r  rd  allow_alphar   override_devicer  rQ  triton_fallbackuse_fma_for_alphas    `        @@@@@@rn   innerzmake_pointwise.<locals>.inner  s   &3 /
BH/
 ,
 #"?"F++ #6+@A UaZ %F1Iv)F!'!5!5!7Jq	++-?? % 1 1&2&OOv526!9fQiOO f U3r
= =,23q1==?3##%%>)<)<)>ABZ 	AEeR__5V I : At1VHAenn.>-?@A 	A ^^U]]3	67gg6IGAGG^T2t 	 $ P!!-P!!%%&GO 	  ;; 9++9"9 +<8	 	 $<'<!y  		 	, F !,,.--.\\^F	
 --/ !*F	
 	
E 4s   $K)rP  r	   r   zTensorBox | _Tr   )r   rQ  r  r  r  r  r  r  s   ``````` rn   make_pointwiser    s     #' d
 d
L Lrp   foreach_node_t)boundc                $     dddd fd}|S )Nr3   )r`  r   c                  	 t        |      }dk(  r,|r*t        |d   t         t        f      s|j                         	n
dk(  r|	n| 	t	        t
        j                  j                  j                        dk(  xs6 t
        j                  j                  j                  t        v xs
 t               }d }|D ]  }t        |t         t        f      s|} n |J d       g }|D ]H  }t        |t         t        f      s|j                  |gt	        |      z         8|j                  |       J t        t        |       }d
	fd}t        |t	        |      ||      S )Nr   rz  r   z1at least one input must be a list to a foreach opc                &    r
 | i iS  |  S r   r   )r}   r  pw_fnscalar_kwarg
scalar_vals    rn   apply_fnz7make_foreach_pointwise.<locals>.inner.<locals>.apply_fnl  s%    dA|Z&@AAd|#rp   )r   r  )rr   rt   r   popr   rZ   rf   rg   rh   rj   inplace_foreach_opsro   rx   r   r  foreach_group_loop)r`  r   rP  realize_outputsa_list_inputinputbroadcast_inputsgroupsr  r  r  r  r  s            @rn   r  z%make_foreach_pointwise.<locals>.inner@  s^   
 fG#vbzD%=9  JW$JJ $$**+q0 0ww##**.AA0-/ 	  	E%$/$	 ' 	
?	
'
  	/EedE]3 ''#l2C(CD !''.	/ $C)9$:;	$ 	$ "&#l*;XWWrp   )rP  zlist[list[TensorBox]]r   list[foreach_node_t]r   )r  r  r  r  s   ``` rn   make_foreach_pointwiser  ;  s    
 56Q 2X 2Xh Lrp   c                   dg|z  }| j                         D ]  \  \  }}}g }|D ]v  \  }	}
 ||
      }|||	<   t        j                  j                  |t        j
                        sB|sE|sH|j                          |j                  |j                                x |st        j                  j                  |        t        d |D              sJ t        t        t           |      S )aa  
    Common loop over grouped foreach arguments.

    Args:
        groups: Result of group_foreach_args - dict mapping (device, use_foreach) to groups
        num_outputs: Number of outputs to produce
        apply_fn: Function to apply to each set of args, returns the output
        realize_outputs: Whether to realize outputs for foreach fusion
    Nc              3  $   K   | ]  }|d u 
 y wr   r   r!  s     rn   r   z%foreach_group_loop.<locals>.<genexpr>       .q}.   )r  rZ   rf   has_featureBackendFeatureFOREACHrealizerx   get_operation_nameregister_operation_listr#  r
   rr   r  )r  num_outputsr  r  outputsr   r~   groupoperation_list
output_indr}   r6  s               rn   r  r  w  s     -16K+?G(. <$u$& % 
	CJd^F"(GJ ##FN,B,BC# %%f&?&?&AB
	C GG++N;<" .g....^$g..rp   Tc                    | j                         k(  r|rt        |       S | S fd} t        |      |       S )Nc                    t        j                  |       }t        j                  t        j                  f}s|v rt        j                  |      }|S )N)	src_dtyper~  )rY   r   r   rM  rN  )r   resultr  r   r  r~  s      rn   	_to_dtypezto_dtype.<locals>._to_dtype  sR    /	
 ^^U]]3	 Ui%7 \\&%0Frp   rQ  )r   cloner  )r   r   copyr~  r  r  s    ` ` @rn   r   r     sE     IEuQx&Q& B>)5A!DDrp   rV  c                   ddl m} |}| j                  } ||t        j                        }t        j
                  |      5   |j                  |  ddd       |j                  }|sJ t        |      }dgt        |      z  }	|j                         D ]  \  \  }
}}g }|D ]k  \  }}||	|<   t        j                  j                  |
t        j                        s:|s=|j                          |j                  |j!                                m |s~t        j                  j#                  |        t%        d |	D              sJ |	S # 1 sw Y   xY w)aI  
    This lowers an invocation of foreach_map
    The way this works is that an arbitrary N-arg func is provided by the user, looped over by the
    polyfill with the same semantics as a foreach op (a loop applying an n-ary function to n args)
    and then traced into a subgraph by dynamo.
    This code allows us to inline the subgraph into the main graph lowering using the PontwiseSubgraphLowering.
    The graph outputs represent the vertically fused sequence of ops, and then register_operation_list
    below registers the buffers as horizontally fuseable in the scheduler.
    r3   )PointwiseSubgraphLowering)root_graph_loweringNc              3  $   K   | ]  }|d u 
 y wr   r   r!  s     rn   r   z_foreach_map.<locals>.<genexpr>  r  r  )subgraph_loweringr  graph_modulerZ   rf   set_graph_handlerrungraph_outputsr   r   r  r  r  r  r  rx   r  r  r#  )subgraphr}   r   r  rP  gmpw_subgraphsub_outputsr  r  r   r~   r  r  r  r6  s                   rn   _foreach_mapr    sA    =F			B+BAGGLK	
		[	) ! ! ++K;,Ffs;''G(. <$u$& 	C 
"(GJww""6>+A+AB{ %%f&?&?&AB	C GG++N;< .g....N3! !s   E		Ec                   |j                   s| j                         j                   rk| j                         r/t        | |      }t        j
                  j                  ||        |S  t        t        j                  j                  d      | |      S | j                         }t        j                  t        j                  f}t        j                  xr
 ||v xs ||v  }t!        | |d|      S )Nr   Fadd_to_fallback_setT)r  r~  )
is_complexr   r   
empty_liker6   InplaceCopyFallbackr   fallback_handlerprimsconvert_element_typedefaultr   rM  rN  r4   r  r   )r   r   dstr  r  r~  s         rn   _convert_element_typer    s    1;;=33::< Qe,C""))#q1J#**22  I/I&& 	;)#9u	'9 Au4;LMMrp   r  c               J   | j                         }||k(  r|rt        |       S | S d } ||      } ||      }||k7  rIt        j                  j	                  |       } t        t        j                  j                        ||      S t        t        j                  | |            S )Nc                    | j                   rt        j                  |       j                  S | t        j                  k(  ryt        j
                  |       j                  S )Nr   )r  r   finfobitsr   iinfor   s    rn   _get_primitive_bitwidthz1to_dtype_bitcast.<locals>._get_primitive_bitwidth  sD    "";;u%***ejj ;;u%***rp   )r   r  r6   ExternKernelr   r  atenviewr   rG   r;   r   )r   r   r  x_dtyper  src_bitsdst_bitsx_conts           rn   to_dtype_bitcastr    s    kkmG%uQx&Q&+ 'w/H&u-H8;;A>0		0??))!U344rp   c                &   |j                   s| j                         j                   r`t        j                  t        j
                  j                  t        j                  j                  j                  j                  | |            S t        | |      S r   )r  r   rG   r   r6   ComplexViewr   rY   r  r  r   r  rA  s     rn   _view_dtyper    se    1;;=33NN!!%)).."5"5";";QF
 	
 Au%%rp   r  non_blockingc                   t        |      }| j                         |k(  r|rt        |       S | S t        j                  t
        j                  j	                  | ||            S r   )rL   rw   r  rG   r   r6   r   )r   r   r  r  s       rn   	to_devicer    sO    6"F||~uQx&Q&BMM00FLIJJrp   c                     t        | |d|      S )NTr  )r  )r   r   r  s      rn   _device_putr  !  s    QTEErp   c
                    |xs | j                   }t        |      }
t        |||       |t        |      }t        |
|||||	      }
 t	        | |||      |
      }
t        t        |      r" t	        t        t        |      d|      |
       |
S )z3A pointwise function that maps ops.{name} to inputsN)rQ  r  r  r  r  )r	  r   r
  )r   r
  )__name__rB   rU   r  r*  r   r  r  )r   r   r	  r   r
  rQ  r  r  r  r  r   s              rn   register_pointwiser  &  s     #7##D	T	B'!#8 #.&12M&N#	
3$?+'
B
	/3	

 	

B ud	
E4  $"7	
 		
 Irp   ldexp)r   rQ  )r	  r   c                Z  	 t        d      | j                         }|j                         }|j                  }|j                   xr |t        j                  k7  }|r|rfd} t        |      | |      S t        |       rt        j                  n|		fd} t        |	      | |      S )Nr  c                     | |      S r   r   )r   nldexp_fns     rn   compute_ldexpz%ldexp_lowering.<locals>.compute_ldexph  s    Aq>!rp   c                    t        j                  |      }t        j                  d      }t        j                  ||      }t        j                  | |      S )Ng       @)rY   r   rY  powr  )r   r  
n_out_typetwo
pow_result	out_dtypes        rn   compute_fallbackz(ldexp_lowering.<locals>.compute_fallbackp  sD    a3J,,sI.Cj1J771j))rp   r  )rB   r   r  r   r   r  r   float32)
r   r  r  n_dtype
x_is_floatn_is_intr  r  r  r  s
           @@rn   ldexp_loweringr  \  s    7#HkkmGkkmG**J,,,FEJJ1FHh	" -~m,Q22 &5Q%7EMMW		*
~"+
 Q 	rp   c                 .   d} t        d      fd}fd}t        |      t        |t        j                        gfd} t	        t
        j                        |      }t        t        |       r! t	        t        t        |       d      |       |S )z2A pointwise function that maps ops.frexp to inputsfrexpc                      | i |d   S Nr   r   r}   r   r
  s     rn   frexp0zregister_frexp.<locals>.frexp0      d%f%a((rp   c                      | i |d   S Nr3   r   r  s     rn   frexp1zregister_frexp.<locals>.frexp1  r  rp   r  c                 0     d   | i | d   | i |fS Nr   r3   r   )r}   r   pw_fnss     rn   r   zregister_frexp.<locals>.fn  s.    vay$)&)96!9d+Ef+EEErp   NrV  )
rB   r  r   int32r*  r  r
  r   r  r  )r   r  r  r   r
  r  s       @@rn   register_frexpr  |  s    D E)) 	vvU[[AF
F
	




B ud	
E4  $	
 	 Irp   c                :    t        |||      }t        | |      }|S )Nr  r  )r  r  )r   pointwise_lowering_fnr  r  r   s        rn   register_foreach_pointwiser    s(     
 ;\
B 
$GR	0BIrp   c           
        d }t        |t        t        f      r t        |      |      }t        |t        t        f      r t        |      |      }| ||g}t	        |d   |d   t
        j                        }t        |      D cg c]  \  }}t        |t              s| }}}t        |t        |D cg c]  }||   	 c}       D ]
  \  }}|||<    t        t        |            D ]Y  }t        ||   t        j                        s!t        j                   ||   t#        ||d      j%                                     ||<   [  t'        ||      |d   t)        |d   |      t)        |d   |            S c c}}w c c}w )Nc                 &    t        j                  |  S r   )rY   wherer}   s    rn   r   zwhere.<locals>.fn  s    yy$rp   r3   r1   rV  r   r  )rt   r;  r   constant_liker   r   rK  rs   rG   r  r  r  r   r6   r   r<   r   rr   r   r  r   )	r   r   r5  r   r}   r   r|   r   indicess	            rn   r  r    so     !eS\"M!Q!eS\"M!Q!Q<DQa.M.U.UE 't_ITQ
1i0HqIGIG.'0JQa0JKL 1Q3t9 Td1gr{{+ ''Qd71:6F6O6O6Q1RSDGT ;>"E:Q$q'5)8DGU+C  J0Js   E9 E96E?c                    t        |       dk(  r&t        | d   t        t        f      rt	        | d    S | S t        j                  t        d | D        d      }g }| D ]q  }t        |j                               x}|k(  rn?t        |      t        |      k7  st        d t        ||      D              rt        ||      }|j                  |       s |S )Nr3   r   c              3  <   K   | ]  }|j                           y wr   )r   r!  s     rn   r   z$broadcast_tensors.<locals>.<genexpr>  s     #AQAJJL#A   r   c              3     K   | ]Z  \  }}t         j                  j                  j                  |      t         j                  j                  j                  |      k7   \ y wr   )rZ   rf   r   r1  r   r   r5  s      rn   r   z$broadcast_tensors.<locals>.<genexpr>  sP      .
 1 GG11!4ww44Q78.
s   A A")r   rt   rr   r   r  r  reducer8  r   r$  r  r3  rx   )rP  rj   r  r   sizess        rn   r  r    s    
6{afQi$/$fQi00(//!#A&#A2 F G 
1::<((EV3Z3v;&# .
 E6*.
 +

 q&!Aq
 Nrp   c                    | S r   r   r   s    rn   nopr*    s    Hrp   
lift_freshc                   t        | t              sJ |(t        t        j                  | j                              S t        |t
        t        j                  f      r)t        j                  j                  j                  |      nt        d |D              }t        t        | j                               |      }t!        t        |t              s|fn|      }g }t#        | j                               D ]X  \  }}||v r>t        j                  j                  j%                  t        j&                  |d            rH|j)                  |       Z || j                         k7  rt+        | |      S | S )Nc              3  n   K   | ]-  }t         j                  j                  j                  |       / y wr   )rZ   rf   r   	guard_intr   ds     rn   r   zsqueeze.<locals>.<genexpr>  s%     >Q177##--a0>s   35r3   )rt   rG   rF   r   rv   r   r   r   rZ   rf   r   r.  r   r   r   r   r)   rs   guard_or_falseEqrx   r  )r   r   dims	new_shaper0  ss         rn   squeezer6    s   a###
{++AFF344 cC,- 	
""3'>#>> 
 C

-s
3CJsE$:sfDDI!**,'  1T	agg..==ehhq!nMQ 
 "+ajjl!:49AArp   c                ,    t        t        | |            S r   )r  r6  )r   r   s     rn   squeeze_copyr8    s    C!!rp   c                    t        | |      }t        | t              sJ t        |t              sJ |j                  | _        | S r   )r6  rt   rG   rv   r   r   vals      rn   squeeze_r<    s=    
!S/Ca###c9%%%XXAFHrp   c                    t        |       rt        | dt        j                        S t	        d      } t        |t        j                        |       S )NFr   isinfr  r   	full_liker   r   rB   r  r   r   s     rn   r>  r>  
  ?    qE44	W	B?>"EJJ?BBrp   c                    t        |       rt        | dt        j                        S t	        d      } t        |t        j                        |       S )NFr   isnanr  r?  rA  s     rn   rD  rD    rB  rp   c                f    t        |       rt        |       S t        d      } t        |      |       S )Nceilr   r  rB   r  rA  s     rn   rF  rF    s/    qQx	V	B>"a  rp   c                f    t        |       rt        |       S t        d      } t        |      |       S )NfloorrG  rA  s     rn   rI  rI  "  /    qQx	W	B>"a  rp   c                f    t        |       rt        |       S t        d      } t        |      |       S )NroundrG  rA  s     rn   rL  rL  *  s/    qQx!!~b!!$$rp   c                f    t        |       rt        |       S t        d      } t        |      |       S )NtruncrG  rA  s     rn   rN  rN  3  rJ  rp   c                <   t        | g      \  } t        | t        j                        rt	        j
                  | t        |            S t        | t              sJ t        |t        t        f      sJ t        | j                               t        |      k(  r| S t        | j                               st        j                  j                  j                  t        | j                                     }|dkD  rut        |      sj| j!                  t        j                  j                  j                  t        |            |z  t"        j$                  xs t'        j(                                t        t	        j
                  | j*                  t        |                  S )Nr   )graph_reuse)rT  rt   r6   r[   r<   r   r   rG   rr   r   r%   rZ   rf   r   guarding_hint_or_throwrW   
mark_reuser4   deterministicr   $are_deterministic_algorithms_enabledrv   )r   r(  x_size_products      rn   r3  r3  ;  s2   aS!DQ!R__%  E%L11a###edE]+++QZZ\eEl* .))@@!**,'
 A&;E&B
 LL  77e8LM!""00 @==?	   Z&&qvvuU|<==rp   c                    t        |      }|D ]  }d||<   	 | }t        |      D ]  \  }}|dk7  st        ||      } t        ||      S Nrz  )rr   rs   	unsqueezer3  )r   shapebroadcast_dimensionsr5  broadcast_dimensionr  r[  r   s           rn   broadcast_in_dimr\  [  sf    UA3 $!#
$ 	
AA, "Q7!S!A" !Urp   c                6    t        | |j                               S r   )r3  r   )r   r7  s     rn   	expand_asr^  i  s    !QZZ\""rp   c                j   t        | j                               t              t              kD  rKt        j                  j
                  gt              t              z
  z  z   t        | t                    } t              t        | j                               k(  sJ t        | j                               }d}t        t                    D ]  }|   dk(  rd}||   |   z  ||<    |r*t        || j                         | j                               S t        d t              D              rt        t        | |            S fd}t              st        |      s{t         j"                  j$                  j'                  t)                    }|dkD  rD| j+                  t         j"                  j$                  j'                  t)        |            |z         | j-                         t/        j0                  | j                         | j                         |t        |            S )NFr   Tr   r   c              3  :   K   | ]  \  }}|d k(  xs |d k(    ywr3   Nr   r&  s      rn   r   zrepeat.<locals>.<genexpr>  s$     
A$!QAFa1f
A   c                   t        |       t              k(  sJ t        |       } t        t                    D ]G  }|   dk7  s|   dk(  rt        j                  j
                  | |<   2t        | |   d|         | |<   I  |       S r  )r   rr   r  r   r/  Zeror0   )r@  r|   old_sizerepeatsx_loaders     rn   rb  zrepeat.<locals>.inner_fn  s    5zS\)))Us7|$ 	IAqzQA;!#$ww||E!H.uQxHQKHE!H	I rp   rc  )rr   r   r   r   r/  r0  r  r  emptyr   rw   r#  r  r  r3  r%   rZ   rf   r   rQ  rW   rR  re  rD   r   )	r   rg  new_sizezero_tensorr|   rb  old_size_productrf  rh  s	    `     @@rn   repeatrm  n  s   AJJL!H
7|c(m#GGKK=CL3x=$@AHLDN#w<3qzz|,,,,AJJL!HK3w<  /1:?KqkGAJ./
 XQ[[]1<<>JJ

A#gx*@
AAVAx())	 !*3H3R77++BB(#
 a LL  77h8OP#$
 }}H||~kkmH~	 rp   c                T    t        t        j                  | j                  |            S r   )rG   rI   r   rv   )r   r(  s     rn   r  r    s     T[[/00rp   c                    t        | t              sJ t        |t        t        f      sJ t        t	        j
                  | j                  t        |                  S r   )rt   rG   rr   r   rC   r   rv   )r   r3  s     rn   permuterp    sF    a###dT5M***[''d<==rp   c                h   ddl m}m} t        j                  j
                  } |t        j                  j                  j                  ||j                  j                  di       ni       }d\  }	}
|rt        |      dk  sJ |       |j                         D ]7  \  }}| |d      t        j                  |       fk(  r|}	)| |d      fk(  s6|}
9 |	\t        j                  |	||||      }t        j                  j!                  |      |_        t        j                  j%                  |       |	|
fS )ad  Register DynamicSliceSize for any unbacked size bindings on the current FX node.

    Unbacked symbols may have been allocated at trace time of the slice
    that inductor must define so the assertion
    new_unbacked_defs >= renamed_unbacked_bindings passes in run_node.

    Returns (sym_size, sym_storage) parsed from the bindings, or (None, None).
    r   )CallMethodKeyr'   unbacked_bindingsNNr1   r   storage_offset)%torch.fx.experimental.symbolic_shapesrr  r'   rZ   rf   rg   r   	shape_envri  rj  r   r  rk  SequenceKeyr6   DynamicSliceSizeregister_bufferr   register_operation)r   startendstepr   rr  r'   rg   node_unbacked_bindingssym_sizesym_storagesymkeypathb_sizes                 rn   &_register_unbacked_slice_size_bindingsr    s*    77''L6	""# 	126	 'Hk)*a/G1GG/288: 	"LC=0&2D2DS2IJJ]+;<>>!		"
 ((5#tTJF''11&9FKGG&&v. [  rp   c           
        t        | t              sJ t        | |d      }| j                         |   }t	        j
                  |      }t        |t        j                        s|dkD  sJ |       	 |dk(  rLt        j                  j                  j                  ||      r"|dk(  rt        |||||      \  }}|J d       | S dd}	d\  }
}|}|r[ |	||d      }
|;t        j                  j                  j                  |t        j                        r|}n
 |	|||      }|
||
|}}d}|sSt        |||||      \  }}|J d       t        t         j"                  j%                  | j&                  |||||            S |sJ |
|J t        ||||| j                         |         \  }}|J | j)                         | j+                          |
6|J | j-                         j.                  |
| j1                         |   z  z   }nt!        j2                  ||| j-                         j.                  | j1                         |   | j                         |   d	      }t        j                  j5                  |      |_        t        j                  j9                  |       |}t;        | j                               }t;        | j1                               }|||<   ||xx   |z  cc<   t=        | |||d
      S # t        $ r Y @w xY w)a	  
    Lowers a slice call, creating ExternKernels for the output size & storage offset symbols,
    if the indices are unbacked and appropriate semantics aren't known.
    If they are known (indices are static/backed/unbacked with info), a SliceView is created.
    r   r3   z:Unexpected storage_offset unbacked binding for no-op slicec           	        | |S d }t        j                  |       } t        j                  |      } |t        j                  t        j                  | d      t        j                  | |                  r| S  |t        j                  t        j
                  | d      t        j                  | |                   r| |z   S  |t        j                  | |            r|S  |t        j
                  | |             ry |t        j                  | d            rt        | |      S  |t        j
                  | d            rt        | |z   d      S y )Nc                T    t         j                  j                  j                  |       S r   )rZ   rf   r   r1  r   s    rn   rI  z5slice_.<locals>.compute_slice_index.<locals>.<lambda>  s    qww''66q9 rp   r   )	r   r3  AndGeLeLtGtr.   r-   )r@  r   r  r   s       rn   compute_slice_indexz#slice_.<locals>.compute_slice_index  s   =N9U#||D!eii*EHHUD,ABCL		%((5!,ehhute.DEF4<%&K&'"#ud##"#ut|Q''rp   rt  FzWUnexpected storage_offset unbacked binding when both start and end indices are resolvedclampT(storage_offset_relative_to_input_storager   )rt   rG   _validate_dimr   r   r3  r   rZ   rf   r   statically_known_leqr  	TypeErrorstatically_known_equalssysmaxsizer6   	SliceViewr   rv   maybe_get_layoutr  
get_layoutoffset
get_strideDynamicSelectStorageOffsetrz  r   r{  rr   
as_strided)r   r   r|  r}  r~  r  r   _r  r  start_index	end_indexambiguous_slicer  new_storage_offset	b_storage	new_sizesnew_stridess                     rn   slice_r    s'    a###
3
"C::<D<<DdEJJ'4!89T93QJ  55dC@	CUCtNA{ & L& H
. (K
 O)%q9 ?qww//GG 
 I+Ct<I"y'<$i3E#O?T4
; " 	
1	
"
 LLUCUK
 	

 L5)"333BUCqzz|C0Hk #			"""\\^22[1<<>RUCV5VV11LLN!!LLN3JJL
	 00;		""9-(QZZ\"Iq||~&KIcN	16 {  s   /AK4 4	L Lr  c                  |d uxr |}d }d }t        | t              rt        | j                  t        j                        rb| j                         }| j                  }|*| j                         | j                         j                  }| j                  j                         } | j                          t        j                  |       st        d|  d      t        j                  |       \  }}	|t        |      nd}t        |t        j                         r|j                  n|}
|rht        |
t        j"                        rNt%        j&                  |t(        j*                  j,                  j/                  |
j1                         d      z
        }t        j2                  |r|n|	j4                  |r|n|	j                  |D cg c]  }t%        j&                  |       c}|D cg c]  }t%        j&                  |       c}t%        j&                  |            }t        t        j6                  ||            S c c}w c c}w )Nzunrealized as_strided(z, ...)r   rv   layout)rt   rG   rv   r6   r:   rw   r   r  r  r  unwrap_viewr  is_storage_and_layoutr   as_storage_and_layoutrK   r   InputBufferr   r3  rZ   rf   graph_input_storage_offsetsrj  get_nameFixedLayoutr   r   )r   r   strideru  r  explicit_storage_offset
new_device	new_dtypestorage
old_layoutstorage_datar5  
new_layouts                rn   r  r  a  s    	d"O'O  JI!YJqvvr{{$C \\^
GG	!a&8&8&:&F\\^22NFF IIK##A&!$:1#V"DEE2215GZ2@2L~.RS  $.gr}}#E7<<7L:lBNN#K gg1155l6K6K6MqQR
  
j&7&7	J$4$4"&'Qa'"()Qa)^$J R''WZHII	 	()s   H:#H?c                d    t        | t              sJ t        | |||      j                  | _        | S r   )rt   rG   r  rv   )r   r   r  ru  s       rn   as_strided_r    s/    a###48==AFHrp   c                4    t        | |||      }t        |      S r   )r  r  )r   r   r  ru  r  s        rn   as_strided_copyr    s    48F=rp   c                    g d} D ]1  }j                  |||j                            z   f       d   d   }3  D cg c]  }|j                          c} fd}t         d   j                               }d   d   |<   t	        j
                   d   j                          d   j                         ||      S c c}w )Nr   rz  c           	       
 t        j                  |    t        j                        }g }g }t	        t                    D ]?  

dk(  r$t        j                  dt        j                        n)t        j                  
   d   t        j                        }t        j                  
   d   t        j                        }t        j                  ||      }t        j                  ||      }
dk(  r|}n*
t              dz
  k(  r|}nt        j                  ||      }|j                  |       t        |       t           
   d   z
        <   |j                  t        j                  |
fdd             B |d   }	t	        t              dz
  dd      D ]  
t        j                  |
   |
   |	      }	! |	S )Nr   r3   c                                S r   r   )r|   idx_loadinputs_loaderss   rn   rI  z1pointwise_cat.<locals>.inner_fn.<locals>.<lambda>  s    -N1-h7 rp           rz  r1   )rY   rX  r   int64r  r   rY  geltand_rx   rr   r,   maskedr  )r[  idx_dimmasksmasked_loadsr|  r}  
start_condend_condmasknext_valr|   r  r   rP  r  inputs_rangess             @@rn   rb  zpointwise_cat.<locals>.inner_fn  s   ..S5;;7s6{#  	A 6 Q,^^M!$4Q$7E 
 ..q!1!!4ekkBC/Jvvgs+HAvc&kAo%!xx
H5LLCyH %Xc]]15Ea5H%HIHSM

75 	D  #Fq("b1 	AyyaQH	 rp   rc  )rx   r   re  rr   rD   r   rw   r   )rP  r   prev_endr   rb  rj  r  r  s   ``    @@rn   pointwise_catr    s    9;MH )h3<<>#3F(FGH $R() 4::Ccoo':N.` F1I&&()H!"%b)HSMay##%Qi!!#	 k ;s   Cc           	       	
 t        j                               dk(  sJ d       t        j                               dk(  sJ d       | j                         t        j                  k(  rt        | t        j                        } | j                         t        j                  k(  sJ d| j                                 t        | j                               k  s!J dt        | j                                       | j                         j                         	j                         
	
f	d}t        j                  | j                         || j                               S )Nr3   expect scales 1 dimexpect zero_points 1 dim<Expecting input to have dtype torch.float32, but got dtype: Expecting axis to be < c                b  	 | 
   f} |       } |      } |      }t        t        j                        \  }}j                  t        j                  k7  r$t	        j
                  |t        j                        }j                  t        j                  k7  r$t	        j
                  |t        j                        }t	        j                  |      }t	        j                  ||z        |z   }t	        j                  |t	        j                  ||            }	t	        j
                  |	      S rE  )_create_constantsr   r  r   rY   r   r  
reciprocalrL  maximumminimum)r[  channel_idxr  scale
zero_pointqminqmax	inv_scaler;  clampedaxisr   input_loader	quant_max	quant_minscalesscales_loaderzero_pointszero_points_loaders             rn   rb  z;quantized_decomposed_quantize_per_channel.<locals>.inner_fn  s    4ylS!k*'4
&y)5==Q
d<<5==(LL6E+j%++>JNN5)	ii	)*Z7++dCKKc$:;||GU++rp   rc  )r   r   r   r   rM  r   r  re  rD   r   rw   )r  r  r  r  r  r  r   rb  r  r  r  s    `````` @@@rn   )quantized_decomposed_quantize_per_channelr    s?    v !Q&=(==&{##%&!+G-GG+ENN*.??- 
FuGXFYZ- #enn&'' 
!#enn&6"7!89' $$&L&&(M$002, ," !~~	 rp   c           	     "     j                          t         t        j                          fd}t	        j
                   j                          j                         |t         j                                     }|j                          |S )Nc                    t         j                  j                         5  t        j                   j                         |             cd d d        S # 1 sw Y   y xY wr   )r6   ComputedBufferforce_realizerY   device_assert_asyncre  )r@  r   r   s    rn   rb  z_assert_async.<locals>.inner_fn  sO    ,,. 	K**+=4+;+;+=e+DcJ	K 	K 	Ks   *AArc  )
r  r   r   r   rD   r   rw   r   rr   r   )r   r   rb  assertion_ops   ``  rn   _assert_asyncr    sm    LLND%**%DK ## nnDMMO$	L rp   c                    t        | |      S r   r  r   s     rn   lower_assert_asyncr  ,      s##rp   c                    t        | |      S r   r  r   s     rn   lower_assert_functional_asyncr  1  r  rp   )r  c               v  	
 t        j                               dk(  sJ d       t        j                               dk(  sJ d       | j                         |k(  sJ d| d| j                                 t        | j                               k  s!J dt        | j                                       t        j                  | j                         	j                         
j                         	
fd}t        j                  | j                         || j                               S )	Nr3   r  r  Expecting input to have dtype , but got dtype: r  c                   |    f} |       } 
|      } |      }	j                   t        j                  k7  r$t        j                  |t        j                        }j                   t        j                  k7  r$t        j                  |t        j                        }t        j
                  t        j                  |t        j                        |      |z  }t        j                  |      }|S r   )r   r   r  rY   r   sub)r[  r  r  r  r  r;  r  r  r  r  r  r  r  s         rn   rb  z=quantized_decomposed_dequantize_per_channel.<locals>.inner_fnT  s    4ylS!k*'4
<<5==(LL6E-j%--@Jggcll5%--8*EMll3	*
rp   rc  	r   r   r   r   r  re  rD   r   rw   )r  r  r  r  r  r  r   r  rb  r  r  r  s    ```   ` @@@rn   +quantized_decomposed_dequantize_per_channelr  6  s-    v !Q&=(==&{##%&!+G-GG+??% 
(/@AR@ST% #enn&'' 
!#enn&6"7!89' MM	$$&L&&(M$002  !~~	 rp   c                   | j                         t        j                  k(  rt        | t        j                        } | j                         t        j                  k(  sJ d| j                                 | j                         fd}t        j                  | j                         t        j                  |t        |      t        |            | j                               S )Nr  c                L    
|       }t        d|z  |t        j                        \  }}t        j                  ||z        |z   }t        t        j                        \  }}t        j
                  t        j                  ||      |      }t        j                  |	      S )N      ?r   )r  r   r  rY   rL  r  r  r   )r[  r  r  r  r  r;  r  r  r  r   r  r  r  s            rn   rb  zBquantized_decomposed_quantize_per_tensor_default.<locals>.inner_fn~  s    S! 1%K5==!
	: ii	)*Z7&y)5==Q
d++ckk#t4d;||GU++rp   r  r  rc  )r   r   rM  r   r  re  rD   r   rw   r  r)  r;  r   r   )r  r  r  r  r  r   rb  r  s      ``` @rn   0quantized_decomposed_quantize_per_tensor_defaultr  k  s     ENN*.??- 
FuGXFYZ- $$&L, !""E%LS_
 ~~ rp   c               l   | j                         |k(  sJ d| d| j                                 t        j                  | j                         fd}t	        j
                  | j                         t        j                  |t        |      t        |            | j                               S )Nr  r  c                     |       }t        ||t        j                        \  }}t        j                  t        j
                  |t        j                        |      |z  }t        j
                  |      }|S rE  )r  r   r  rY   r  r   )r[  r  r  r  r;  r  r  s        rn   rb  zDquantized_decomposed_dequantize_per_tensor_default.<locals>.inner_fn  s]    S!-eZu}}Uzggcll5%--8*EMll3	*
rp   r  rc  )r   r   r  re  rD   r   rw   r  r)  r;  r   r   )	r  r  r  r  r  r   r  rb  r  s	         ` @rn   2quantized_decomposed_dequantize_per_tensor_defaultr	    s     ??% 
(/@AR@ST% MM	$$&L !""E%LS_
 ~~ rp   c           	     T  	
 | j                         t        j                  k(  rt        | t        j                        } | j                         t        j                  k(  sJ d| j                                 t        j                               dk(  s9t        j                               dk(  rj                         d   dk(  sJ d       t        j                               dk(  s9t        j                               dk(  rj                         d   dk(  sJ d       | j                         j                         	j                         
| j                         	
f	d}t        j                  | j                         || j                               S )Nr  r   r3   expect scale as scalar tensor"expect zero_point as scalar tensorc                l  	  
|       } t        j                               dk(  rdnd      } t        j                               dk(  rdnd      }j                  t        j                  k7  r$t        j                  |t        j                        }j                  t        j                  k7  r$t        j                  |t        j                        }rOj                  dk(  r@t        j                  |t        j                  |      |      }t        j                  |	      S t        j                  |t        j                  |      z        |z   }t        t        j                        \  }}t        j                  t        j                  ||      |      }t        j                  |	      S )Nr3   r   r   r   r   )r   r   r   r   r  rY   r   r   rZ  r  round_to_intrL  r  r  r  )r[  r  _scale_zero_pointr;  r  r  r  r   r   r  r  r  r  scale_loaderr  zero_point_loaders           rn   rb  zAquantized_decomposed_quantize_per_tensor_tensor.<locals>.inner_fn  s5   S!c%..*:&;q&@dbI'ENN4D0E0JPRS;;%--'\\&%--8Fu}},,,{EMMBKfkkU*''%!7EC##C//iiv 667+E&y)5==Q
d++ckk#t4d;||GU++rp   rc  )r   r   rM  r   r  r   r   re  rw   rD   r   )r  r  r  r  r  r   rb  r   r  r  r  s    ````` @@@@rn   /quantized_decomposed_quantize_per_tensor_tensorr    su    ENN*.??- 
FuGXFYZ- u~~ A%ENN"u~~'7':a'?'&'  z""$%*J!"a'J,?,?,A!,D,I,+,  $$&L$$&L"..0F, ,  !~~	 rp   c                 	
 t        j                               dk(  s9t        j                               dk(  rj                         d   dk(  sJ d       t        j                               dk(  s9t        j                               dk(  rj                         d   dk(  sJ d       | j                         |k(  sJ d| d| j                                 t        j                  | j                         j                         	j                         
	
fd}t        j                  | j                         || j                               S )	Nr   r3   r  r  r  r  c                V    |       } t        j                               dk(  rdnd      } 
t        j                               dk(  rdnd      }j                  t        j                  k7  r$t        j                  |t        j                        }	j                  t        j                  k7  r$t        j                  |t        j                        }t        j                  t        j                  |t        j                        |      |z  }t        j                  |      }|S )Nr3   r  r   )r   r   r   r   r  rY   r   r  )r[  r  r  r  r;  r  r  r  r  r  r  s        rn   rb  zCquantized_decomposed_dequantize_per_tensor_tensor.<locals>.inner_fn  s    S!c%..*:&;q&@dbI'ENN4D0E0JPRS;;%--'\\&%--8Fu}},,,{EMMBKggcll5%--8+FOll3	*
rp   rc  r   )r  r  r  r  r  r   r  rb  r  r  r  s    ``   ` @@@rn   1quantized_decomposed_dequantize_per_tensor_tensorr    sU    u~~ A%ENN"u~~'7':a'?'&'  z""$%*J!"a'J,?,?,A!,D,I,+,  ??% 
(/@AR@ST% MM	$$&L$$&L"..0
 
 !~~	 rp   c                   t        |       dk  ry| d   j                         }| dd D ]  }||j                         z  } |syt        t        j                  j
                  j                  d   j                               }d}d}|D ]  }t        j                  j                  |      }|%t        |t        j                        s@t        |j                  t        j                        set        |j                  j                               t        |j                  j!                               z  }	t        j                  j"                  j%                  ||	      s|}|} n |yt'        |dd      }
|
syt)               | D ]#  }t'        |dd      }|sj+                  |       % |
D ]  }|j,                  dk(  st        |j.                  t0        j2                  j4                        sBt0        j6                  j8                  |j.                  j:                  v sst=        fd|j>                  D              r y |S )	aj  If all cat inputs share a common upstream reduction buffer whose
    only consumers feed into this cat, return its name so it can be
    excluded from the can_fuse_reduction check.

    Checks common reads for an IR reduction whose numel matches the cat
    output, then verifies via FX origins that all of the reduction's
    consumers feed into the cat inputs.r1   Nr   r3   r;  originsre   c              3  &   K   | ]  }|v  
 y wr   r   )r   ucat_input_originss     rn   r   z2_cat_inputs_recombine_reduction.<locals>.<genexpr>h  s     E1..E   ) r   get_read_namesrK   rZ   rf   rg   ri  numeltry_get_bufferrt   r6   r  rv   rE   rW   r   get_reduction_sizer   r  r  r)   r   ri   rj   r   r   r   r   	reductiontagsr#  rh   )rP  r   common_readsr   cat_out_numelreduction_namereduction_bufr   bufreduction_numelr  inp_originsoriginr  s                @rn   _cat_inputs_recombine_reductionr,     s    6{Q!9++-Labz ;#c&8&8&::; +177+?+?+D+DU+K+Q+Q+STMNM gg$$T*O3 1 12388R\\2+CHH,=,=,?@=++-D O ww77W!% #   mY5G3=< 2c9d3$$[12  II(6==%***?*?@		##v}}'9'99EEE rp   c                b     d   j                         j                  dk(  }|rt        d  D              rp D ]  }|j                           t        d  D              rt	        t
        j                  g  \   } t        t
        j                  j                         |      S t               dk(  rt         d         S d } D cg c]
  } ||       c}t              rOt               D cg c]
  \  }}|r	| }}}|st         d         S t        |      dk(  rt        |d         S | t         d   |d      }t         dt        j                   i}	 D 
cg c]  }
t#        |
|	       c}
 ddd	 t%               fdfd
t'         |      }|rt%        |g      n	t%               t        fd D              }dfdt(        j*                  rt-         |      S |r)t/        t0        j2                  j5                   |            S fddddt               t(        j6                  k  s1t               t(        j8                  k  rt        fd D              r~t        fdt:        j<                  j>                  D              }t        fd D              xr |}d fd} |       }t        fd D              xr | }|s|s|rt-         |      S t/        t0        j2                  j5                   |            S c c}w c c}}w c c}
w )z@Lower aten.cat, choosing between pointwise_cat and ConcatKernel.r   r   c              3  |   K   | ]4  }|j                         t        j                  t        j                  fv  6 y wr   )r   r   int8uint8r   r  s     rn   r   zcat.<locals>.<genexpr>s  s.      ;@ejj%++66s   :<c              3  T   K   | ]   }t        |j                               d k(   " yw)r   N)r   r   r1  s     rn   r   zcat.<locals>.<genexpr>z  s!     >es5>>#$)>s   &(r3   c                    | j                         }t        |      dk(  xr- t        j                  j                  j                  |d   d      S Nr3   r   )r   r   rZ   rf   r   r  )r   r   s     rn   _cat_should_skipzcat.<locals>._cat_should_skip  s=    zz|4yA~V!''"2"2"J"J4PQ7TU"VVrp   r   c                   t        | t              rJt        | j                  t        j                        r| j                  j                         S | j                  S t        | t        j                        r| j                  S | S r   )rt   rG   rv   r6   r:   r  r   r   s    rn   unwrap_tensorzcat.<locals>.unwrap_tensor  sV    a#!&&"++.vv))++vva'66Mrp   c                    t        | t        j                        xr$ t        | j                  t        j                        S r   )rt   r6   r  rv   rE   )r   s    rn   is_reductionzcat.<locals>.is_reduction  s)    !R../TJqvvr||4TTrp   c                    t        | t        t        j                  f      r  |             S  |       xs? t        | t        j                        xr# t        fd| j                         D              S )Nc              3  v   K   | ]0  }|vxr&  t         j                  j                  |             2 y wr   )rZ   rf   
get_buffer)r   readcan_fuse_reductionexcludes     rn   r   z2cat.<locals>.can_fuse_reduction.<locals>.<genexpr>  sE        G# J&qww'9'9$'?IJs   69)rt   rG   r6   r   rD   r$  r  )r   r?  r>  r9  r7  s    `rn   r>  zcat.<locals>.can_fuse_reduction  sl    a)R]]34%mA&6@@O !R\\*   ,,. 	
rp   c              3  0   K   | ]  } |        y wr   r   )r   r   r>  r?  s     rn   r   zcat.<locals>.<genexpr>  s     Kq.q':Ks   c                6   t        j                  |       r:t        j                  | d      \  }}t         j                  j	                  |       S t        | t        t         j                  f      r  |             S t        | t         j                        ryy)NF)freezeT)	r6   r  r  ConcatKernelcan_realize_into_without_copyrt   rG   r   rD   )r   r  r  should_lower_cat_inputr7  s      rn   rE  z#cat.<locals>.should_lower_cat_input  sx     ##A&11!EBJGQDDWMMMa)R]]34)-*:;;a&rp   c                H   t        | t        t        j                  f      r  |             S t        | t        j                        sy| j                         j                  }| j                         D ]*  }| t        j                  j                  |            z  }, |S r  )rt   rG   r6   r   rD   inner_fn_opcountnum_opsr  rZ   rf   r<  )r   countr=  op_countr7  s      rn   rJ  zcat.<locals>.op_count  s    a)R]]34M!,-- !R\\*""$,,$$& 	8DXagg00677E	8 rp   r1   c                n    | t         j                  j                  t         j                  j                  fv S r   )r  catr  constant_pad_ndri   s    rn   additional_pointwise_opsz%cat.<locals>.additional_pointwise_ops  s(    dhh&&(<(<(D(DEEErp   c              3  4   K   | ]  } |      k    y wr   r   )r   r   MAX_SIMPLE_OP_COUNTrJ  s     rn   r   zcat.<locals>.<genexpr>  s     Cq22Cs   c              3  6   K   | ]  }t        |        y wr   rP   )r   userO  s     rn   r   zcat.<locals>.<genexpr>  s!      
 S":;
   c              3  .   K   | ]  } |        y wr   r   r   r   rE  s     rn   r   zcat.<locals>.<genexpr>       >&s+>   c                    t         j                  yj                  d   } t        | t        t
        f      r| }n)t        | t        j                  j                        r| g}nyt        	      r!t        |	      D cg c]
  \  }}|r	| }}}
fdt        |      D ]T  \  }}t        |d      rt        |j                        dk  r+t        fd|j                  D              r y |      sT y yc c}}w )NFr   c                    t        | t        t        j                  f      r  |             S t        | t        j                        S r   )rt   rG   r6   r   rD   )r   is_unrealized_pointwiser7  s    rn   r\  zKcat.<locals>.any_input_has_multi_consumers.<locals>.is_unrealized_pointwise
	  s7    a)R]]!;<2=3CDD!!R\\22rp   rh   r3   c              3  >   K   | ]  }|ust        |        y wr   rS  )r   r  rg   s     rn   r   z=cat.<locals>.any_input_has_multi_consumers.<locals>.<genexpr>	  s     Wq,AV'*Ws   	T)rZ   rg   r}   rt   rr   r   r   fxNoder$  r  r   r   rh   )fx_argsinput_nodesr  skipr   ir_inputrg   r\  rP  	skip_maskr7  s         @@rn   any_input_has_multi_consumersz*cat.<locals>.any_input_has_multi_consumers  s    >>L#"''*G'D%=1%GUXX]]3&i9~03K0KXWQSWqXX3
 "%[&!9  XsG,CII!0C WCIIWW +84  ' Ys   
C>C>c              3  .   K   | ]  } |        y wr   r   rW  s     rn   r   zcat.<locals>.<genexpr> 	  rX  rY  )r   zTensorBox | ir.StorageBoxr   	ir.IRNode)r?  zOrderedSet[str]r   r   ri   torch._ops.OpOverload) rw   r   r#  r  require_channels_lastr  rL  r  r  r   r  r$  r  r  r   r   rK  r   r)   r,  r4   force_pointwise_catr  rG   r6   rC  r    max_complex_pointwise_cat_inputsmax_pointwise_cat_inputsrZ   rg   rh   )rP  r   
cpu_devicer  r  r5  r   rb  valid_inputsr   r   
recombinedfusable_reductionpointwise_usesfuse_pointwise_usere  has_multi_consumershorizontal_fuse_catrQ  rO  r>  r?  r9  rJ  rE  rd  r7  s   `                 @@@@@@@@@rn   rL  rL  o  s    %%',,5Jc DJ 
  	EMMO	>v>>-dhh@@IFA1 0 01&#>>
6{aVAYW /55!!$5I
9~),VY)?LgatLL##|!a))
q	3
*C	%D%L%LE /55shsE"5F
U :D 
 
" 1=J;Ez:,7:<GKFKK !!VS)) //<==  F 6{f===	V77	7CFCC 
~~++
 
 >v>>Q> 	 	D <= >v>>XGXCX 	 #(:>Q --R__++FC899 6L 6s   L!>
L&	L&+L,c                  
 | j                         t              t              t              t        k7  fd       t        j
                  j                  j                  t        j                  |d            }|r\t        j
                  j                  j                  t        j
                  j                  j                     |z            d      }n[t        j
                  j                  j                  t        j
                  j                  j                        |z
        d      }d
|r| df
nd|f
t              D cg c]  \  }}|fvs| }}}|j                  |       
fd}	t        t        j                   j#                  | ||	            S c c}}w )N)r[  rankc                     d  d S )Nz(diagonal dimensions cannot be identical z, r   )dim1dim2s   rn   rI  zdiagonal.<locals>.<lambda>1	  s     HbQUPVW rp   r   )r   r   c                    | d   }dgt        	      z  }d}t              D ]1  }|k(  r|d   z   ||<   |k(  r|d   z   ||<   %| |   ||<   |dz  }3 |t        	      dz
  k(  sJ |S )Nrz  r   r3   r1   )r   r  )
r[  diag_idxoriginal_idxcur_dimr0  base_idxrz  r{  num_dimsoriginal_shapes
        rn   	reindexerzdiagonal.<locals>.reindexerO	  s    r7sS00x 	ADy"*Xa["8Qd"*Xa["8Q"%g,Q1	 #n-1111rp   )r   r   r   r   rZ   rf   r   evaluate_exprr   r  evaluate_maxevaluate_minrs   rx   rG   r6   GenericViewr   )r  r  rz  r{  offset_negative	diag_sizer|   r5  r(  r  r  r  r  s     ``      @@@rn   diagonalr  )	  s   ^^%N>"H84D84D	W gg&&44UXXfa5HIOGG$$11GG))t$v-t$ 
	 GG$$11GG))t$t$v- 
	 HGQ<v;$^4N41a$8MQNEN	LL   R^^**5%CDD' Os    F:/F:c                0    t        t        | |||            S r   )r  r  )r  r  rz  r{  s       rn   diagonal_copyr  b	  s    %t455rp   c                P    t        |       }t        ||||      }t        ||       |S r   )r  r  	mutate_to)r  srcr  rz  r{  r6  rj   s          rn   diagonal_scatterr  g	  s*    5\FffdD1FfcMrp   c                p   t        j                  |      }t        j                  | j                         |         }d }t        j                  j
                  j                  t        j                  |d            r||z   }n?t        j                  j
                  j                  t        j                  |d            r|}|t        |      ri| j                          | j                         }| j                         }| j                         j                  ||   |z  z   }||= ||= t        | |||d      S t        | |||dz   d      }t!        ||      S t#        t        j                  j
                  j$                  t        j                  j&                  j(                  d         }	|	J t+        |	      dk(  sJ |	       t-        t/        |	j1                                     \  }
}| j                          | j                         }| j                         }|
}t3        j4                  |
|| j                         j                  ||   | j                         |   d      }t        j                  j7                  |      |_        t        j                  j;                  |       ||= ||= t        | |||d      S )Nr   Fr  r3   r  rs  )r   r3  r   rZ   rf   r   r1  r  r  r&   r  r  r  r  r  r  r6  r'   rw  rg   ri  r   rL  iterr  r6   r  rz  r   r{  )r   r   r[  r   actual_indexrj  
new_strider  slice_resultrs  unbacked_offset_symr  buffers                rn   selectr  o	  sY   
,,s
C<<

S)*DLww&&uxxQ'78Tz	
			(	(#q)9	:$S) IIKzz|HJ!"!6!6C<9W!W3"9>  "!S,q8HPUVL<-- 2	""AGG$8$8$=$=>Q$R ((( !Q&9(99&!$'8'>'>'@"AB IIKzz|HJ,**	3	

SF ''))&1FKGGv&3	16 rp   c           
     <   t        | |d      }|}t        |t        t        f      st        j
                  j                  }|j                  |d      rt        d|       | j                         |   }|j                  |d      rt        | |ddd      gS |j                  |d      rt        d|       |j                  t        ||z   dz
  |            }|g|z  }||dz
  |z  z
  |d<   g }d}|D ](  }	||	z   }
|j                  t        | |||
d             |
}* |S )Nr   z=split expects split_size be non-negative, but got split_size=Fr  zKsplit_size can only be 0 if dimension size is 0, but got dimension size of r3   rz  )r  rt   rr   r   rZ   rf   r   statically_known_ltRuntimeErrorr   r  r  r.  r+   rx   )r   r(  r   sizes_r   x_sizechunksr  r|  r   r}  s              rn   splitr  	  sP   
3
"CF edE]+77##''q1OPUwW  c"++FA61c1au566++E15--3H6 
 ##HVe^a-?$GH6!vzU22r
 FE dl 	fQUCu=> Mrp   c                    t        | ||      S r   )r  )r   r(  r   s      rn   split_with_sizesr  	  s    E3rp   c                    t        | |d      }t        j                  j                  j	                  | j                         |         }t        |      D cg c]  }t        | ||       }}|S c c}w r  )r  rZ   rf   r   r.  r   r  r  )r   r   r  r|   r  s        rn   unbindr  	  s_    
3
"CWW''

S(9:F).v7AfQQ7F7M 8s   A-c                @   | j                         }t        |      }t        ||      |dk(  rt        t	        | d      |d      S |   }t
        j                  j                  }|j                  ||       |j                  d       t        ||z
        dz   }|j                  |      dkD  r-| j                  |j                  t        ||z  |                   g |d  ||dz   d  |}	fd}
t        t        j                   j#                  | |	|
            S )Nr   F)r}  r  r3   c                D    | d   |    z  z   }g | d  || dz   d S )Nrz  r3   r   )r[  dim_idxr   r~  s     rn   r  zunfold.<locals>.reindexer	
  s?    b'CHtO+8Tc8G8c#'B&788rp   )r   r   r   r  rX  rZ   rf   r   	check_leqcheck_ltr+   rQ  rR  r*   rG   r6   r  r   )r   	dimensionr   r~  r(  ndimdim_sizer   new_dim_sizeout_sizer  r   s      `       @rn   unfoldr  	  s   JJLEu:D
4
+Cqyi1o4u==SzHwwHtX&aHtOT2Q6L&&x014	++GL44G,RS	
 EtDlDU379-=DtDH9 R^^**1h	BCCrp   c                    t        | |d      }t        | j                               }|j                  |t        j
                  j                         t        | |      S r  )r  rr   r   insertr   r/  r0  r  )r   r   r4  s      rn   rX  rX  
  sE    
3
"CQZZ\"IS%''++&9rp   c                    t        | |      }t        | t              sJ t        |t              sJ |j                  | _        | S r   )rX  rt   rG   rv   r:  s      rn   
unsqueeze_r  
  s>    
As
Ca###c9%%%XXAFHrp   c                   t         j                  j                  j                  j	                  t        j                  |            }t        | j                               }|dk  r|||z   z  }d|cxk  r	||z   k  sJ  J |S r  )	rZ   rf   r   rw  r  r   sympifyr   r   )r   r   r  r  s       rn   r  r  !
  sr    
''


$
$
2
25==3E
FCqzz|D
Qwtf}#dVm#####Jrp   rz  c                   t        | |d      }t        j                  j                  j	                  | j                         |         dz  }t        | |d|d      }t        | |||dz  d      }t        |t        |            S )Nr   r1   Fr  )	r  rZ   rf   r   r.  r   r  r  sigmoid)r   r   new_lenr   r5  s        rn   glur  *
  ss    
3
"Cgg((c):;q@Gq#q'/Aq#w!59Aq'!*rp   c                N     |rt         j                           fd}d|_        |S )Nc                 x    d }t        j                  |t        j                  j                  g| i |      S )Nc                Z    t        | t        j                        r| j                         S | S r   rt   r6   r>   wrap_for_loweringr   s    rn   wrap_tensorsz7fallback_handler.<locals>.handler.<locals>.wrap_tensors:
  #    ,6q")),D1&&(K!Krp   )rk  tree_mapr6   FallbackKernelr   )r}   r   r  kernels      rn   handlerz!fallback_handler.<locals>.handler9
  s<    	L "++226KDKFK
 	
rp   T)r  r   _is_fallback_handler)r  r  r  s   `  rn   r  r  5
  s&    f
 $(G Nrp   c                 .    t        j                  d       y )NzjTorchinductor does not support code generation for complex operators. Performance may be worse than eager.)warningswarnr   rp   rn   _warn_complex_not_supportedr  G
  s    MMtrp   c                   | j                         rt                y| j                  ry| j                  ryt	        | j
                  | j                        sy| j
                  t        j                  k(  r|syt        |j                  t        j                  j                        xr |j                  t        j                  j
                  t        j                  j                   t        j"                  j                   t        j$                  j                   t        j&                  j                   fv xsE t        |j                  t        j                  j                        xr t)        |j                         S y)z0Do not support reading or writing to this tensorTF)r  r  is_meta	is_sparserQ   r   r   r   float8_e8m0fnurt   rj   r   r   r  r  rL  r  r  
_scaled_mm_scaled_mm_v2rR   )r   rl   s     rn   unsupported_input_tensorr  P
  s   ||~#%yy{{(!((;ww%&&&
 t{{EJJ$9$9: 		  

""''""**	Y 4;;

(=(=>W74;;CW
 	
 rp   c                *   t         j                  j                  t        j                  j
                  j                  j                  f}||j                  |v r| j                         ryt        | |      ry| j                  xr t        j                  S )z2Do not support writing tensor but can read from itFT)r  r  r   r   rY   r  r  r  rj   r  r  is_cpur4   disable_cpp_codegen)r   rl   supported_complex_viewss      rn   unsupported_output_tensorr  v
  so     					,,44 DKK+BBq||~4(882222rp   c                R     j                   t        j                  j                  u ry j                  dk(  ry j                   t        j
                  j                  u ry fd}t        j                   j                  i  j                  D ]  } ||d      s y  | d      S )NFplaceholderc                D   t        | t        j                  j                        syd| j                  vryt        j                  | j                  d         D ]F  }t        |t        j                  j                        s(|rt        |      s7 yt        |      sF y y)NFr;  T)rt   r   r^  r_  ri  rk  rl  _subclasses
FakeTensorr  r  )inp_out_node	is_outputri  rl   s      rn   check_skip_conditionzCfallback_node_due_to_unsupported_type.<locals>.check_skip_condition
  s    ,6)))&&|'8'8'?@ 		 DdE$5$5$@$@A,T48+D$7		  rp   )r  T)
rj   r  view_as_complexr  ri   lift_fresh_copyrk  arg_tree_leavesr}   r   )rl   allow_cpu_inputsr  r   s   `   rn   %fallback_node_due_to_unsupported_typer  
  s    {{d**222ww- {{d**222* %%tyy@DKK@ u5  55rp   c                (   | |       nt         }| |vs|s
J d|         |rt        t        j                  d            rt	        | g      rt
        j                  r&| t        j                  j                  j                  v si|sgt        j                  j
                  j                  r4dt        j                  j
                  _        t        j                  d       t        d|  d      fd}t!        | t        j"                  j$                        r| j'                         D ]
  } ||        y t!        | t        j"                  j(                  t        j"                  j*                  f      r	 ||        y t-        d|  d	t/        |              )
Nz*both a fallback and a decomp for same op: CIFznA make_fallback error occurred in suppress_errors config, and suppress_errors is being disabled to surface it.zmake_fallback(a.  ): a decomposition exists, we should switch to it. To fix this error, either add a decomposition to core_aten_decompositions (preferred) or inductor_decompositions, and delete the corresponding `make_fallback` line. Get help from the inductor team if unsure, don't pick arbitrarily to unblock yourself.c                n    t        |        t        |         t        | d       t        |             S NrV  )r   r   r*  r  )op_overloadlayout_constraints    rn   register_fallbackz(make_fallback.<locals>.register_fallback
  s;    !+.(!+/@AG $G[)
 	
rp   zUnsupported fallback z with type )r8   r   osgetenvr9   r4   fallback_randomr   _decompdecompositions_for_rngextra_random_decomps_dynamosuppress_errorslogwarningr   rt   r   r   r   r   HigherOrderOperatorr  r   )ri   r  r  override_decompget_decomp_fncheck_decompsr  r  s    `      rn   make_fallbackr  
  sU    (5'@MOnM]"o 
4RD95 	4!t$ ""emm::OOO ==//38EMM  0KKH RD !f f
 	

 "ejj112??, 	+Kk*	+	B..

0N0NO	P"22$k$r(LMMrp   c                T    d}| D ]  }||z  }	 t        |t        j                        S )z
    TorchInductor offset calculation differs from PyTorch eager offset
    calculation for random ops (tl.rand vs torch.rand). In future, we should
    strive for same impl for tl.rand and torch.rand.
    r3   r   rF  r   r  )rY  r  r5  s      rn   philox_rand_offsetr  
  s2     E 	%u{{++rp   c                B  	
 t        j                  || t         j                  j                  |             j	                         
|j                         |j                         		
fd}t        j                  ||t        |             }t        |       }||fS )Nc                   t        j                   g       t        j                        }t        j                   g       t        j                        }t        j                  t        j
                   |       t        j                        |      }t        j                  ||      }t        j                  |      S r   )rY   r   r   r  r   rX  rand)	r@  seed_index_exproffset_index_exprrand_index_exprr  r   offset_loader
random_posseed_loaders	        rn   rb  zphilox_rand.<locals>.inner_fn
  s     ,,{2DLLr):EKKH''NN:e,ekk:<M
 
 ||FE**rp   rc  )
r6   r  FlexibleLayoutcontiguous_stridesmake_indexerre  rD   r   rr   r  )r   seedr  r  r   r   rb  random_values_nodeoffset_noder  r  r  s        `   @@@rn   philox_randr
  
  s     
,,T2	
 ln  ""$K&&(M+ #))Dz	 %T*K{**rp   c           	         t         j                  r\t        j                  t        j
                  t        j                  j                  t        j                  j                  | ||            S t        d      )Nz&should be handled in replace_random.py)r4   r  rk  r  rG   r   r6   r  r  native_dropoutr  r   )r   ptrains      rn   r  r    sY    $$T%8%8%@%@!QN
 	

 EFFrp   c                x   t         j                  s-| j                         t        j                  d      k(  sJ d       | j                          t        |      dk(  st        |d   t              rt        j                  j                  nt        j                  j                  }t        j                  || g|  | S )Nr   Tthis should be handled in decomps unless config.fallback_random or the device is CPUr   )r4   r  rw   r   r   r  r   rt   r;  r  
bernoulli_Tensorr6   InplaceBernoulliFallback)r   r}   r  s      rn   r  r  !  s    !!Q\\^u||E7J%J ^J IIK t9>ZQ7 	__## 
 Q66Hrp   c                    t         j                  s-| j                         t        j                  d      k(  sJ d       t        t        |       g| S )Nr   r  )r4   r  rw   r   r   r  r  )r   r}   s     rn   bernoulli_pr  0  sG    !!Q\\^u||E7J%J ^J eAh&&&rp   c                    t         r   r   )r  s    rn   _foobarr  9  s    
rp   c                .    t         j                  d       y )Nz1using triton random, expect difference from eager)r  info)salts    rn   _warn_triton_randomr  >  s    HH@Arp   c                 J    t        t        j                  j                         y r   )r  rZ   rf   creation_timer   rp   rn   warn_triton_randomr  C  s    --.rp   )r  c                     |j                  d      t        | i |S t        j                  s|j                  d      r|j	                  dd        t        | i |S t        d      N	generator
pin_memoryz-should have been handled in replace_random.py)rj  fallback_rand_generatorr4   r  r  fallback_rand_defaultr   r}   r   s     rn   r  r  Y  s_    zz+*&777			6::l#;

;%$d5f55
H
IIrp   c                     |j                  d      t        | i |S t        j                  s|j                  d      r|j	                  dd        t        | i |S t        d      r!  )rj  fallback_randn_generatorr4   r  r  fallback_randn_defaultr   r&  s     rn   randnr*  c  s_    zz+*'888			6::l#;

;%%t6v66
H
IIrp   c                    t        j                  |t        j                  j                  j
                        }t         j                  j                  | |      S r   )r6   get_stride_orderrZ   rf   r   rw  r  require_stride_order)input_tensorr  stride_orders      rn   inductor_force_stride_orderr0  m  s;    &&vqww/?/?/I/IJL??//lKKrp   c                    t        d      )Nz.should be handled in fuse_seed_creation_pass()r  r   s    rn   inductor_seedr3  s  s    
I
JJrp   c                z    t                t        j                  t        j                  | t        |                  S r   )r  rG   r   r6   RandomSeedsrL   )rI  r   s     rn   inductor_seedsr6  x  s)    BNN5-2GHIIrp   c                |      fd}t        j                   j                          j                         |g       S )Nc                L    t        j                  j                               S r   )rY   	load_seedr  )r  r@  seedss    rn   rb  z&inductor_lookup_seed.<locals>.inner_fn  s    }}U^^-u55rp   rc  )rD   r   rw   r   )r:  r@  rb  s   `` rn   inductor_lookup_seedr;  ~  s9    6 !oo	 rp   c                V   t        | t        j                        st        j                  |       } | j                  dk(  rf| j                  }|t        j
                  j                         }t        j
                  j                  |      }|j                  |j                  z  }|S d}|}|S )Nry  i   )
rt   r   r   r   r@  ry  current_deviceget_device_propertiesmulti_processor_countmax_threads_per_multi_processor)r   r[  propthreads_per_round_CPU_GRAIN_SIZEs        rn   get_threads_per_roundrD    s    fell+f%{{fll;**++-Czz//4&&)M)MM 	   +rp   )r  align_dtypec                 
 t         j                  rJ dv sJ g | } t        j                  }|j	                         }t        j                  ||| t
        j                  j                  |       |      j                         
|j                         t         j                  r/|j                  dk(  r t        |      dd} ||      
fd}n
fd}t        j                  |||g |       }	|	j!                          |	S )	N)r  r*  r  ry  c                J    | t         j                  t         j                  fv ryy)Nr   r   )r   rN  rM  )dts    rn   _vec_from_dtypez(inductor_random.<locals>._vec_from_dtype  s    emmU^^44rp   c           
          dg      } dg      }t        j                  ||t        j                   |       t        j                        t                    S )Nr   r3   )vec)rY   
rand_eagerrX  r   r  r   )r@  rng_seedbase_offsetr  r  rB  rL  s      rn   rb  z!inductor_random.<locals>.inner_fn  sS    "A3'H%qc*K>>!z%0%++>H rp   c                     t        t               g       t        j                   |       t        j                              S r   )r  rY   rX  r   r  )r@  moder  r  s    rn   rb  z!inductor_random.<locals>.inner_fn  s7    %73%Bz%0%++> rp   rc  )rI  torch.dtyper   r   )r4   r  r   r  rO  r6   r  r  r  r  re  align_random_eagerr   rD  rD   r   r  )r   r  rQ  r  rE  r   r   rJ  rb  r  r  r  rB  rL  s     `       @@@@rn   inductor_randomrT    s     %%%%$$$$T7DMME%%'FtR..AA$GPVln  ""$K  V[[F%:1&9	
 k*		 			 w	F NNMrp   rG  c               f   	 t         j                  rJ g |}t        j                  }|j	                         }t        j                  |||t
        j                  j                  |      |      j                         |j                         	 	fd}t        j                  |||g |      S )NrG  c           	        t        j                   g       t        j                   |       t        j                        t        j
                  t        j                        t        j
                  t        j                              S r   )rY   	randint64rX  r   r  
value_exprr  )r@  highlowr  r  s    rn   rb  z"inductor_randint.<locals>.inner_fn  sV    }}ONN:e,ekk:NN3,NN4-	
 	
rp   rc  )r4   r  r   r  rO  r6   r  r  r  r  re  rD   r   )
rZ  rY  r   r  r  r   r   rb  r  r  s
   ``      @@rn   inductor_randintr[    s     %%%%T7DKKE%%'FtR..AA$GPVln  ""$K
 w	 rp   c                    | j                         }| j                         }| j                         |d   | j                         j	                         |d   fS rW  )r   r  r  r  storage_size)tbr   r  s      rn   _boundaries_helperr_    sK    ;;=D]]_F
R
$$&r
	 rp   c                R    t        t        j                  j                  |             S r   )rG   r6   r  realize_inputr^  s    rn    _realize_bucketize_lookup_tensorrc    s     R__222677rp   c                H    | j                         | j                         d   fS rW  )r  r  rb  s    rn   _sorter_helperre  
  s    ;;="--/"---rp   c           
     H   | j                         }| j                         j                  t        d t	        |d d |d d       D        t
        j                  j                        z   }|s%t        |t
        j                        rt        |      S t        j                  ||      S )Nc              3  ,   K   | ]  \  }}||z    y wr   r   )r   r5  r|   s      rn   r   z%_bucketize_indices.<locals>.<genexpr>  s     941aQ9   rz  )r  r  r  sumr  r   r/  re  rt   Integerr   rY   rX  )r^  r@  index_dtypestridesflattened_indexs        rn   _bucketize_indicesrn    s     mmoGmmo,,s93ws|U3BZ89577<<0 O Z??##>>/;77rp   	out_int32rightsidesorterc               6   
 d } |       r ||      r
9 |      s1 t        t        j                  j                  d       |||      S ||dk(  rd|rt        j
                  nt        j                  
|j                         t                t              t         j                               dk(  r

 fd}n	
 fd	}|j                         }t        j                  |
||j                  
      }	|	j                          |	S )Nc                ^    t         j                  j                  | t        j                        S r   )rZ   rf   r  r  	BUCKETIZErb  s    rn   rI  zsearchsorted.<locals>.<lambda>$  s     AGG$7$7
N$$% rp   Fr  ro  rq  Tr3   c                     |       }t        j                  |t              t        d      d n
t	              d       S t        d            S )Nr   rs  sorter_indices)rY   	bucketizer_  rn  re  )r[  r;  rk  rq  sorted_sequencers  values_loaders     rn   rb  zsearchsorted.<locals>.inner_fnG  sm    $C=="?3"?BD%~t>&3I ~   ,FBD rp   c                             }d fd}t        j                  |t               |      d n
t              d       S  |            S )Nc                    t        |       S r   )rn  )r^  r[  rk  s    rn   get_flattened_indexz;searchsorted.<locals>.inner_fn.<locals>.get_flattened_index^  s    )"c;??rp   rx  )r^  rG   )rY   rz  r_  re  )r[  r;  r  rk  rq  r{  rs  r|  s   `  rn   rb  zsearchsorted.<locals>.inner_fnY  si    $C@ =="?3#O4%~t>&3I'-~t  <Ov;V rp   rc  )r  r  searchsortedr  r   r  r  re  rc  r   r   rw   rD   r   rY  r  )r{  selfrp  rq  rr  rs  validate_bucketizerb  r   r  rk  r|  s   `  ` `    @@rn   r  r    s    /!$''9&'AT 1 1 8 8eT
 	
 DGO!*%++K$$&M 7GO1&9
?##%&!+	 	$	 	$ __Fzz	F NNMrp   rp  rq  c               x   t        j                               dk(  sJ t        j                  j	                  | t
        j                        r.t        j                  j	                  t
        j                        s/ t        t        j                  j                  d      | |      S t              | j                         }| j                         |rt        j                  nt        j                   fd}t#        j$                  ||| j                               }|j'                          |S )Nr3   Fr  r  c           	     r     |       }t        j                  |t              t        d            }|S )Nr   )rY   rz  r_  rn  )r@  r;  r!  
boundariesrk  r  rq  s      rn   rb  zbucketize.<locals>.inner_fn  s?    5!--z*z2{;
 rp   rc  )r   r   rZ   rf   r  r  rv  r  r  rz  r  rc  rw   re  r   r  r  rD   r   r  )	r  r  rp  rq  r   rb  r  rk  r  s	    ` `   @@rn   rz  rz  x  s     z""$%*** 	
E>#;#;<GG
N,D,DEQ 5 55Q:%
 	
 2*=JF$$&L!*%++K
 ~~	F NNMrp   c                    t        | t        j                        xr+ t        | t        j                  t        j                  f       S r   )rt   r6   r>   NonTensorObjOpaqueMultiOutputr   s    rn   _is_tensor_irnoder    s:    a# J	BOOR112- ) rp   c                z    t        j                  t        t        j                  j
                  ||f      \  }}||fS r   )rk  tree_map_onlyr  r6   r  require_stride1r  r}   r   s      rn   require_denser    s7    ''2??::T6NLD& <rp   c                z    t        j                  t        t        j                  j
                  ||f      \  }}||fS r   )rk  r  r  r6   r  require_contiguousr  s      rn   r  r    s9    ''
**	vLD&
 <rp   c                z    t        j                  t        t        j                  j
                  ||f      \  }}||fS r   )rk  r  r  r6   r  r   r  s      rn   r   r     s;     ''
22	vLD&
 <rp   c                z    t        j                  t        t        j                  j
                  ||f      \  }}||fS r   )rk  r  r  r6   r  rk  r  s      rn   rk  rk    s9    ''
--	vLD&
 <rp   c           	        || S t        |t              st        |      r| S t        | t        j                        r.t        j
                  j                  | |j                               S t        | t              r!| D ci c]  }|t        | |   ||          c}S t        | t        t        f      r" t        |       d t        | |      D              S | S c c}w )Nc              3  :   K   | ]  \  }}t        ||        y wr   constrain_to_fake_tensor)r   r   f_as      rn   r   z+constrain_to_fake_tensor.<locals>.<genexpr>  s!      
19!S$Q,
rc  )rt   r   r   r6   r>   r  require_exact_stridesr  r  r  r   rr   r   r  )r   fake_argkeys      rn   r  r    s    
(,-1J
#ryy!44S(//:KLL#tRUV3-c#hFFVV	C%	'tCy 
=@h=O
 
 	
 J Ws   >Cc           
         t        d t        | |      D              } |j                         D ci c]  \  }}|t        |||          }}}| |fS c c}}w )Nc              3  :   K   | ]  \  }}t        ||        y wr   r  )r   r   r  s      rn   r   z,constrain_to_fake_tensors.<locals>.<genexpr>  s#      C 	!h/rc  )r   r  r  r  )r}   r   	fake_argsfake_kwargsr   r  s         rn   r   r     sb      y1 D JPXAa)![^<<XFX< Ys   Ac           
         d fdt        fdt        || j                        D              }|j                         D ci c]  \  }}| || j                  |          }}}||fS c c}}w )Nc                    t        | t        j                  j                        r| j                  j                  d      S | S Nr;  )rt   r   r^  r_  ri  rj  )fx_args    rn   get_fake_valz-constrain_to_fx_strides.<locals>.get_fake_val  s,    fehhmm,;;??5))rp   c           	     `    |      }t        |       rt        |t        j                        s| S t	        j
                  |j                         t        j                  j                  j                        }t        j                  j                  | |      S t        | t              r0t        |t              s| S | D ci c]  }| | |   ||          c}S t        | t        t        f      r<t        |t        t        f      s| S  t!        |       fdt#        | |      D              S | S c c}w )Nc              3  6   K   | ]  \  }} ||        y wr   r   )r   r   fx_aapply_constraints      rn   r   zDconstrain_to_fx_strides.<locals>.apply_constraint.<locals>.<genexpr>  s"      .5a D)rU  )r  rt   r   r  r6   r,  r  rZ   rf   r   rw  r  r-  r  r   rr   r   r  )r   r  fake_valr/  r  r  r  s        rn   r  z1constrain_to_fx_strides.<locals>.apply_constraint  s   'S!h5
..!177#3#3#=#=L ??77\JJc4 h-
NQRsC)#c(HSMBBRRcE4=)h6
49 9<S(9K   
 Ss   =D+c              3  6   K   | ]  \  }} ||        y wr   r   )r   r   r  r  s      rn   r   z*constrain_to_fx_strides.<locals>.<genexpr>  s"      *5#vf%rU  )r   r  r}   r  r   )fx_noder}   r   r   r  r  r  s        @@rn   r   r     sx    
*  9<T7<<9P D EKLLNSDAqa!!W^^A%677SFS< Ts   !A/c                H    t         j                  s||fS t        | g|i |S r   )r4   r  r   )r  r}   r   s      rn   *constrain_to_fx_strides_if_fallback_randomr    s*    !!V|"7<T<V<<rp   c                     fd fdt        fdt        t        | j                              D              }|j	                         D ci c]  \  }}| d| j
                  |          }}}||fS c c}}w )zGApply stride constraints to SDPA inputs, ensuring dense last dimension.c           
        t        |      s|S |j                  d   }|j                         D cg c]4  }t        |t        j
                        r|j                  j                  n|6 }}t        j                  j                  j                  }t        j                  |j                         |      }|r<|d   dk7  r4t        t        t!        t#        |j%                                                 }j&                  t(        j*                  j,                  k(  r| dv rt#        |      dk(  sJ d}d }t.        j0                  rd|j3                         }	t5        |      |	|rt7        |      nd f}|t        j                  j8                  v rt        j                  j8                  |   S  | ||||      }
||
t        j                  j8                  |<   |
S c c}w )Nr;  rz  r   )r   r   r   )r   r3   r1   r   )r  ri  r  rt   r   SymIntrl   exprrZ   rf   r   rw  r6   r,  rr   r.  r  r   r   rj   r  0_scaled_dot_product_efficient_attention_backwardr  r4   cache_sdpa_constraintmaybe_get_nameidr   sdpa_constraint_cache)r[  r   r  meta_valr5  meta_stride_exprrw  r/  	cache_keyarg_namer  _apply_constraint_innerr  s              rn   r  z)sdpa_constraint.<locals>.apply_constraint)  s    %J;;u%GOGX
BC:a6AFFKKA=
 
 GG$$..	**8??+<iHL,1s3<<>/B)C DEL NNDDLLMv|$)))
 (L 	''))+H3'3l#I
 AGG999ww44Y??(h 0,
  7=AGG)))4Y
s   9Gc                ^   |j                   s,|j                  s t        j                  j	                  ||      S dj
                  t        j                  j                  j                  j                  k(  xr | dk(  }t        |t              sJ t        |j                               dvr|S t        j                  |      }|r3t        j                   t        j                  j#                  |      |      S t        |t$              rE|j'                         5|r3t        j                   t        j                  j#                  |      |      S |rt)        |j                               }g }|j'                         }	t+        t        |j                               dz
        D ]q  }
t,        j.                  j0                  j3                  ||
   d      s1|	3t,        j.                  j0                  j3                  |	|
   d      sa|j5                  |
       s dgt        |      z  }d|d<   d}t+        t        |      dz
  dd      D ]  }
||
dz      dk7  r|||
dz      z  }|
|v r@t,        j.                  j0                  j3                  t7        ||
dz            d      rd||
<   ]t,        j.                  j0                  j3                  t7        |      d      st9        |      z  }|||
<    t        j                  j;                  ||      S |r3t        j                   t        j                  j#                  |      |      S t        |t$              rE|j'                         5|r3t        j                   t        j                  j#                  |      |      S fd}t        |j<                  t        j>                        rQ ||      sI ||jA                               r3t        j                   t        j                  j#                  |      |      S t        j                  j	                  ||      S )	Nr   r   r   r   r3   r   rz  r1   c                    | j                         }t        |      dk(  ryt        j                  j                  j                  t        j                  t        |d         d            S )Nr   Frz  )	r   r   rZ   rf   r   r1  r   r2  r/   )r   r   	ALIGNMENTs     rn   
is_alignedzDsdpa_constraint.<locals>._apply_constraint_inner.<locals>.is_aligned  sR    ::<D4yA~77##22T"Xy115 rp   )!is_cudais_xpur6   r  r-  rj   r   rY   r  '_scaled_dot_product_efficient_attentionr  rt   rG   r   r   is_aligned_realized_tensortry_match_insignificant_stridesra  r>   maybe_get_striderr   r  rZ   rf   r   r  rx   r/   rJ   r  rv   r:   r  )r[  r   r  r  r/  effn_attn_fwd_biasis_aligned_tensorr  expanded_dimsmaybe_strider|   out_stridesr  r  r  r  s                 @rn   r  z0sdpa_constraint.<locals>._apply_constraint_inner\  s     HOO??77\JJ 	 NNyy~~EEMMN q 	 #y)))s||~f,J99#yI55--c24D 
 sF#$$&2!55--c24D  CLLN+HM //1L3s||~.23 ,77##;;<LQ<OQRS ,((@@aRST!((+, $X.KKOF3x=1,b"5 (q1u%*#hq1uo5F %ww''??KA.	:A *+A ww''??	*A %VY7)CF!'A%(( ??88kJJ55--c24D 
 sF#$$&2!55--c24D 	 chh,c?coo/0==55c:<L  33CFFrp   c              3  >   K   | ]  \  }\  }} |||        y wr   r   )r   r[  r   r  r  s       rn   r   z"sdpa_constraint.<locals>.<genexpr>  s*      C#v 	c6*s   rz  )r   rs   r  r}   r  r   )r  r}   r   r   r  r  r  s   `    @@rn   sdpa_constraintr  &  s    1fgGR  "+Cgll,C"D D IOW1a!"a):;;WFW< Xs   "A<)r  c                t   t        |t        j                        s*t        || j	                         | j                               }|}| j                         |j                         k7  rt        || j                               }| j	                         |j	                         k7  rt        || j	                               }| j                         |j                         k7  r&t        || j                               }t        |      }nt        |      }| j                         }||j                  rt        |       |S Nr`  )rt   r6   r>   rF  r   rw   r  r   r   r3  r  r  	is_pinned_realize_as_pinned)r  r  r  r   rz   r  self_layouts          rn   r  r    s    c299%S 09JKACNN,,a*+~~3==?*Q()}}#,,.(Q(sq'')K;#8#86"Mrp   )memory_formatc          	         t        j                  | j                         | j                         | j	                         t        | j                                     S Nrc  )rD   r   rw   r   re  rr   r   )r   r  s     rn   r  r    s@    
 ||~kkmAJJL!	 rp   c                v    | j                          d| j                  j                  j                         _        y r   )r  rv   r  r  )r  s    rn   r  r    s'    
NN.2FKK!+rp   c                   g }t        | t              rt        | j                  t        j                        rw| j                  } t        | t        j                        rF|j                  | j                                | j                  } t        | t        j                        rFt        |       } t        |       } |r;| j                  } |d d d   D ]  }t        j                  | |      }  t        |       } | S )Nrz  r  )rt   rG   rv   r6   r   rx   r  r  )r   reinterpret_view_layoutsr  s      rn   clone_preserve_reinterpret_viewr    s    !!YJqvvr7I7I$JFFB../$++ALLN;A B../ aLaAFF.tt4 	:F""&9A	:aLHrp   c                n    t        |       }| j                         }||j                  rt        |       |S r   )r  r  r  r  )r   r  input_layouts      rn   r  r    s6    1XF%%'LL$:$: 	6"Mrp   r  c               \    fd}t        j                  t        |      || g      S )Nc                D    t        j                  | d   z  z         S )Nr   r   rY   rX  )r@  r   r|  r~  s    rn   r   ziota.<locals>.fn	  s!    ~~dU1Xo5UCCrp   rc  )rD   r   rL   )lengthr|  r~  r   r   requires_gradr   s    ```   rn   iotar    s1    D V$x	 rp   )r   r   r  r#  r  c               ^    |J t        || z
  |      }t        || |||||      S d|      S )Nr   r|  r~  r   r   r  )rJ   r  )	r|  r}  r~  r   r   r  r#  r  r  s	            rn   arange_start_stepr    sV     S5[$'F+v# 
 27# rp   c           	        t        || j                               }| j                         t        | d      t        j
                  j                  j                  t        j                  d            r| j                            z   njt        j
                  j                  j                  t        j                  d            rn, t        t        j                  j                        | |      S t        j
                  j                  j!                  d       t        j
                  j                  j#                  | j                                   t%        t'        |      | j                               }|j                         fd}t)        j*                  | j-                         | j                         |t/        | j                                     S )Nr   c           	         t        j                  t        j                  t        j                  |    t        j
                        t        j                  t        j
                               |        |             S r   )rY   r  eqrX  r   r  )r[  r   r@  
src_loaderrh  s    rn   rb  z select_scatter.<locals>.inner_fn>  sX    yyFFs3x5uekk2 sOSM
 	
rp   rc  )r   r   re  r  rZ   rf   r   r1  r   r  r   r  r  r  select_scatterr  r  r  r3  rX  rD   r   rw   rr   )r   r  r   r@  rb  r  rh  s     `` @@rn   r  r  ,  sT   
3
&C}}H
3
"Cww&&uxxq'9:

S))	
			(	(%);	< = 3 3 ; ;<QS%PPGGq%(GGeQZZ\#%67
3$ajjl
3C"J
 ||~kkmAJJL!	 rp   c           	     :   	
 t        | j                               } j                         
t         d       j	                            t
        j                  j                         \  t         j	                               }t        z
  dz
  z         |<   t        ||      }|j                         		 
fd}t        j                   j                          j                         |t         j	                                     S )Nr   r3   c                   	dk(  rk(  r
dk(  r |       S t        j                  |    t        j                        }t	        |       t        |    	z
  
      <   g }	dk7  rZ|j                  t        j                  |t        j                  t        j                  	      t        j                                     k7  rZ|j                  t        j                  |t        j                  t        j                        t        j                                     
dk7  rz|j                  t        j                  t        j                  t        |    	z
  d
      t        j                        t        j                  dt        j                                     |sJ t        j                  t         j                   |      }t        j"                  |fdt%              rdnd      }t        j&                  || |             S )Nr   r3   c                             S r   r   )src_idxr  s   rn   rI  z1slice_scatter.<locals>.inner_fn.<locals>.<lambda>  s    Jw' rp   r  )rY   rX  r   r  rr   r+   rx   r  r   r3  r  r  r0   rY  r  r'  r  r  r   r  )r[  r  r  src_valr  r   r  r}  r  r|  r~  r   rh  s       @rn   rb  zslice_scatter.<locals>.inner_fn_  s   A:#/daic?"..S5;;7s)C5 0$7A:KKNN5<<#6D (?KKNN5<<#4ekkB 19KKNN'C5(8!TBEKK LLEKK0	 t$/**' #A

 yySM
 	
rp   rc  )r   r   re  r  r   r6   r  normalize_start_endrr   r+   r3  rD   r   rw   )r   r  r   r|  r}  r~  src_sizerb  r  r  rh  s   ` ````  @@@rn   slice_scatterr  P  s    
3
&C}}H
3
"Czz|C H 11!S%EJE3AJJL!HS5[D1H5t<HSM
h
C"J,
 ,
\ ||~kkmAJJL!	 rp   c                j    t        | t        t        f      rt        |       dkD  rt	        | d         S | S r  )rt   rr   r   r   _unwrapr   s    rn   r  r    s-    !dE]#A
qt}Hrp   r   r   r  r#  c          	     B    |t         j                  k(  rt         j                  }t        |d t         j                  fv d|        t	        t               t              rxs t         j                  nxs t        j                         g }t	         t        j                        r fd}nt	         t        t        f      r fd}nt               dk(  s't	         d   t        t        f      rBt               dk  r4|j                  t        j                  t                             fd}n:t        j                   j#                  t        j$                   ||xs d            S t'        j(                  t+        |      ||	      }|rt-        |       |S )
Nlayout=c                0    t        j                        S r   r  r@  rv   r   s    rn   rb  ztensor.<locals>.inner_fn  s    >>$..rp   c                0    t        j                        S r   rY   rY  r  s    rn   rb  ztensor.<locals>.inner_fn  s    <<e,,rp   r   r   c                      fdt              dk(  rt        j                  d      S  dt                    S )Nc           	     j   | |k  sJ || z
  dk(  rt        j                  |          S || z
  dz  | z   }t        j                  t        j                  t        j                  d   t
        j                        t        j                  |t
        j                               | |       ||            S )Nr3   r1   r   )rY   rY  r  r  rX  r   r  )r|  r}  midbinary_searchrv   r   r@  s      rn   r  z/tensor.<locals>.inner_fn.<locals>.binary_search  s    s{"{;!#<<UU;;U{q(50yyFFuQx=S%++6 "%-!#s+ rp   r   )r   rY   rY  )r@  r  rv   r   s   `@rn   rb  ztensor.<locals>.inner_fn  s7     4yA~||Au-- CI..rp   F)r   r   r#  rc  )r   jaggedstridedr   rt   r  r   r  get_default_dtyper   r   r;  r   rx   rj  rZ   rf   add_tensor_constantrF  rD   r   rL   r  )rv   r   r   r  r#  rd  rb  r  s   ``      rn   rF  rF    sC    v$..'&0BC'$-%$2002!F$$	/ 
D5#,	'	- 
Ta:d1gs|<TaemmCI./	/( ww**LLE&Z=P5
 	
 V$	F 6"Mrp   c                z    t        | t              r|t        | |      } |t        | |      } | S t	        | ||      S r  )rt   rG   r   r  rF  )rv   r   r   s      rn   	as_tensorr    sD    $	"D%(DT6*D$eF33rp   c                8    t        | t        j                        S rE  r  )rv   s    rn   long_tensorr    s    $ekk**rp   c                   t        t        j                  j                  j                  t        j                  j
                  j                  d         }|J t        |      dk(  sJ |       t        t        |j                                     \  }}t        j                  |||       }t        j                  j                  |      |_        t        j                  j                  |       t        j                  j
                  j                  d   }t!        |t"        j$                  t"        j&                  t"        j(                  f      r|j*                  j,                  S t/        j0                  |      S )Nrs  r3   r;  )r'   rZ   rf   r   rw  rg   ri  r   rL  r  r  r6   DynamicScalarrz  r   r{  rt   r   r  SymFloatSymBoolrl   r  r   r  )rv   rs  binding_symr  r  r;  s         rn   _local_scalar_denser    s    2	""AGG$8$8$=$=>Q$R ((( !Q&9(99&   %6%<%<%> ?@Kk7D9F''))&1FKGGv& ''


#
#E
*C#ennemmDExx}}}}S!!rp   c                     y r   r   )rv   r   s     rn   _assert_scalarr    s     rp   )r   r  c                    y r   r   )r   r   r  r   r   r  s         rn   _assert_tensor_metadatar  "  s     rp   c                   | t        | t        t        f      st        d      rj                  t        t        t        f      rfd}nTt        t
        j                        rfd}n3t        j                               dk(  sJ j                         fd}t        j                  ||t        |            S )Nr   c                0    t        j                        S r   r  r@  r   r   s    rn   rb  z_full.<locals>.inner_fn0  s    <<u--rp   c                0    t        j                        S r   r  r  s    rn   rb  z_full.<locals>.inner_fn5  s    >>%//rp   r   c                     g       S r   r   )r@  value_loaders    rn   rb  z_full.<locals>.inner_fn<  s    ##rp   rc  )rt   r   r;  r   r   r   r   r   r   re  rD   r   rr   )
fill_valuer   r   r   rb  r   r  s     `  @@rn   _fullr  )  s    Ej3,/GE74K%#u&	. 
E5;;	'	0 5>>#$)))((*	$ Dz	 rp   c                8     t        t        |            | fi |S r   create_tensor_liketensor_constructor)r   r  r   s      rn   r@  r@  G  s     =0<=aJ6JJrp   c                      d d d dd d fd
}|S )NFr   r   r  r#  r  c                   t        |d t        j                  fv d|        t        | d       t        |      }| xs t        j                         } t        |      dk(  r6t        |d   t        t        t        j                  f      rt        |d         }|D ]  }t        |t        j                        sJ  |D cg c]  }t        j                  |       }}t        t        |      | |      }|rt        |       |S c c}w )Nr  r  r3   r   )r   r   r   rL   r  r   rt   rr   r   Sizer  r   r3  r  r  )	r   r   r  r#  r  r   r5  full_pointwiser  s	           rn   r  z!tensor_constructor.<locals>.innerM  s     	6dEMM22gfX4FG}$o6v&2002t9>ja4

2KLa>D  	3A!!U\\222	3)-.AQ..z=+@%N~. /s   >Dr   )r  r  s   ` rn   r  r  K  s      4 Lrp   )r   r  r   r#  r  c                    t        |      }t        |      dk(  r6t        |d   t        t        t
        j                  f      rt	        |d         }t        |d | |||      S )Nr3   r   r   r  r   r#  )rL   r   rt   rr   r   r   r   empty_strided)r   r  r   r#  r  r   s         rn   ri  ri  j  sX     6"F
4yA~*T!WtUEJJ.GHT!W~d%v* rp   c                      dddddd fd
}|S )zZ
    Shim to convert X_like(...) into X(...).  For example zeros_like() into zeros().
    NFr  c                  t        | d       t        |d t        j                  fv d|        || j                         }nt	        |      }|xs | j                         }t        | j                               } |||||      S )Nr#  r  r  )r   r   r   r   r   rw   rr   r   )r   r   r   r  r#  r  r   creation_fns          rn   _constant_likez*create_tensor_like.<locals>._constant_like  s     	z><06dEMM22gfX4FG=KKME 'E)1<<>AJJL!fV

 	
rp   r   )r'  r(  s   ` rn   r  r  {  s     duTX
 rp   c                *    t        t        |             S r   r  )r  s    rn   r   r     s    0<==rp   c                     d d d d d fd
}|S )Nr#  c               n   t        |t        t        f      sJ t        | d       t        |d t        j
                  fv d|        t        |      xs | j                         }|xs | j                         }|D cg c]  }t        j                  |       }}t        t        |      ||      S c c}w )Nr#  r  )rt   rr   r   r   r   r   r   r   rw   r   rj  r  rL   )r   r   r   r  r   r#  r5  r  s          rn   _new_constantz#new_constant.<locals>._new_constant  s     $u...z><06dEMM22gfX4FGU#4q{{})1<<>*./Qa //Zv!6tDD 0s   =B2r   )r  r,  s   ` rn   new_constantr-    s    tDT
E rp   r#  c          	     ~    || j                         }|| j                         }t        |d ||t        |      |      S Nr#  r   rw   r$  rL   )r   r   r   r  r   r#  s         rn   	new_emptyr1    sF    }~V$ rp   c                  t        | t        t        f      sJ t        |t        t        t        d       f      sJ t	        |d t
        j                  fv d|        t        |      xs t        j                         }|xs t        j                  d      j                  }t        |      }t        d|||       }|j                          |j                  j                  }t        j                   |j                  dgt#        |       z        |_        t        |t$        j&                        sJ | D cg c]  }t)        j*                  |       } }|r#|D cg c]  }t)        j*                  |       c}nt$        j,                  j/                  |       }t%        j0                  ||| ||xs d      |_        |S c c}w c c}w )Nr  r  r   )r  r   r   r   )rd  F)r   r   r   r  r  )rt   rr   r   r   r   r   r   r   r  rF  r   rL   r  r  rv   dataclassesreplacer   r6   r  r   r3  r  r  r  r  )	r   r  r   r  r   r#  	pointwiser  r5  s	            rn   r$  r$    sw    dT5M***ftUDJ7888v$..'&0BC<5#:#:#<E/u||C(//F6"F6TJI^^  F%%fkk1#D	/JFKfb//000%)*ELLO*D*  #))Qa)11$7 
 NN%FM  +)s   3F:F?c          	     ~    || j                         }|| j                         }t        ||||t        |      |      S r/  r0  )r   r   r  r   r  r   r#  s          rn   new_empty_stridedr7    sH     }~V$ rp   c                    t         j                  j                  j                  |      }t	        t        t        |            |j                        }t        j                  j                  | |      S )N)r  )rZ   rf   r   r   sortedr  r   __getitem__r6   r  r-  )r   r  r/  s      rn   copy_stridedr;    sN    WW55f=F%F,&2D2DEL??//<@@rp   c                V    |j                  d      J d        t        |      | fi |S )Nr   z(dtype should be handled by decomposition)rj  r  )r   r  r   s      rn   fullr=    s5    ::g*V,VV*)j)$9&99rp   c                   t        | t              sJ |j                         dk(  rt        | |j	                               S | j	                         t              dk(  }t        | |      |rt        | dg      } dg| j                         |j                         fd}t        j                  | j                         | j                         ||j	                               S )Nr   r3   c                    t        |       } t        j                   |          d      }t        |       dk(  r|g} n|| <    |       S )NFwrap_negr   )rr   rY   indirect_indexingr   )r[  
gather_idxr   index_loaderr   rh  s     rn   r   zgather.<locals>.fn  sO    3i**<+<d3iRWX
s8q=,C!CH}rp   rc  )rt   rG   	get_numelr1  r   r   r  r3  re  rD   r   rw   r   )	r   r   r@  sparse_gradr  r   rD  r   rh  s	    `    @@@rn   gatherrG    s     a###AENN,--::<DY!^F
3
'C1qcNs}}H$$&L ||~kkm~~	 rp   c                0  	
 |r- t        t        j                  j                        | ||||      S |rJ t	        | t
              sJ t	        |t
              sJ dt        |j                               v sJ | j                         	|j                         t        |j                               | j                         
g |j                         
dd  	
fd}t        j                  | j                         | j                         |      S )Nr   r3   c                    t        |       t              k(  sJ |  d         | d        }t        j                  |d   d      gg | d  z   } |      S )Nz != r   Fr@  )r   rY   rB  )r[  	var_index
weight_idxindices_loaderindices_ndimrj  weight_loaderweight_sizes      rn   r   zembedding.<locals>.fn2  sx    3x3x=(@SEhZ*@@("3}#56	!!)[^eL
!c,- !"
 Z((rp   rc  )r  r  	embeddingr  rt   rG   strr   re  r   r   rD   r   rw   )weightr!  padding_idxscale_grad_by_freqsparser   rL  rM  rj  rN  rO  s         @@@@@rn   rP  rP     s   7 6 67G[*<f
 	
 :fi(((gy)))C))+,,,,&&(M((*Nw'')*L//#K6!!#6k!"o6H) )   " 	 rp   c           
        t        d | D              s)J d| D cg c]  }||j                          c}        t        d | D              rt        d      t	        |       D cg c]  \  }}t        |t              s| }}}t        |      dkD  sJ d       d gt        |       z  }t        |t        |D cg c]  }| |   	 c}       D ](  \  }}|j                         |k7  rt        d      |||<   * ||fS c c}w c c}}w c c}w )Nc              3     K   | ]T  }|P|j                         t        j                  t        j                  t        j                  t        j
                  fv  V y wr   )r   r   r  r  r   r0  r   r|   s     rn   r   z.check_and_broadcast_indices.<locals>.<genexpr>C  sA      = 	
%++u{{EJJLLs   AAz)indices must be int64, byte or bool. Got c              3     K   | ]7  }||j                         t        j                  t        j                  fv  9 y wr   )r   r   r   r0  rX  s     rn   r   z.check_and_broadcast_indices.<locals>.<genexpr>J  s1      78q}%**ekk22s   ?5?zFallback for bool indicesr   z"requires at least 1 non-None indexz.Fallback when indices is on a different device)r#  r   r$  r   rs   rt   rG   r   r  r  rw   )r!  r   r|   r   
valid_idxsnew_indicess         rn   check_and_broadcast_indicesr\  B  s/      
 4G4eqWXWdQ[[]4e3fg   <C  ""=>> )' 2O1jI6N!OJOz?QD DD&3w<'KJ 1
3S1GAJ3S TU 1 <<>V#%&VWWA 
""# 5f P 4Ts   C=C='D D:Dc	           
     T   dt        j                        D ]  \  }	}
|
|	z
  dk7  sd t              D cg c]  \  }}|	| |    c}}g | t              t              z   d  d   }rz   nd | z   |d  z   f
d}|fS c c}}w )NFr3   Tr   c                  
 t        |       t              k(  sJ t              t        
      k(  sJ t              }g }d   }rdn|}d}t        d   dz         D ]}  }||k(  r||z  }|   *|t        |       k  sJ |j                  | |          |dz  }<|   }|J 
|   }|j                  t        j                   || |||z          |	              g || |d  }|S  |      S )Nr   rz  r3   r   rA  )r   r  rx   rY   rB  )r[  rx  	new_indexfirst_tensor_indexstart_offsetnext_idxr|   loaderr   r   indexed_sizer!  indices_loadersnon_consecutive_tensorsoutput_sizetensor_indicestensor_sizerA  rh  s            rn   r   z*index_output_size_and_inner_fn.<locals>.fn  sN   3x3{++++?#s<'8888;	+A.3q9K~b)A-. 	AL D qz!#c(***  X/A(+)))#A  ))s<,2EFG#!)		&

^
	 %,yE(92EErp   )r  pairwisers   r   )r  r!  ri  rj  rf  re  rh  r   rA  previouscurrentr|   r;  ra  r   rg  rh  s    ````````      @@rn   index_output_size_and_inner_fnrn  \  s    $ $&//? +'X"&*#+ ,5W+=MC6!9MKSKS&[)9C<O)O)Q"RSK'*!K/ ++,,-./ 	 F  FD ?_ Ns   
B$B$c                    t        | ||      \  }}}t        j                  | j                         | j	                         ||      S r  )index_impl_helperrD   r   rw   r   )r   r!  r   rh  rb  r  s         rn   
index_implrq    sB    0GUCK1||~kkm	 rp   c                   t        |t        t        f      sJ | j                         t	        || j                               \  }}t        |      dkD  sJ d       |D cg c]  }||j                         nd  }}t        ||d      j                               }| j                         }t        t        |            D cg c]  }||   	||    }	}|rd|	v rd|vrt        d      t        t        |            D cg c]  }||   	 }	}t        ||||||	d ||	      \  }
fd}|
|fS c c}w c c}w c c}w )Nr   z Must have at least one valid idxz0index is out of bounds for dimension with size 0r_  c                       |             S r   r   )r[  index_inner_fnrh  s    rn   rb  z#index_impl_helper.<locals>.inner_fn  s    s+,,rp   )rt   rr   r   re  r\  rw   r   r   r  
IndexErrorrn  )r   r!  r   rA  ri  r|   rf  rj  r  re  rh  rb  rt  rh  s               @@rn   rp  rp    sS   ge}---}}H9'1<<>RG^~"F$FF"KRSa!-q}}TASOS w~a01::<=KZZ\F',S\':U!gaj>TF1IULUl"q';KLL',S\':;!F1I;L;"@
#K- .009 T V <s   !D2
D7D7D<c                    	 t        | |d      S # t        $ r? | j                           t        t        j
                  j                  d      | |      cY S w xY w)NTr   Fr  )rq  r   r  r  r  r@  r  r   r!  s     rn   r@  r@    sT    
!WD11 
			M

 1 1uMw
 	

s    AAAc                    t        | |d      S )NFrw  )rq  rx  s     rn   _unsafe_indexrz    s    a..rp   c                6    t        t        |       |||dd      S )NTFr   may_realizeindex_put_impl_r  r   r!  r  
accumulates       rn   	index_putr    s     a'6:Tu rp   c                6    t        t        |       |||dd      S )NFr|  r~  r  s       rn   _unsafe_index_putr    s     a'6:U rp   c                    |j                         | j                         k7  rt        || j                               }|rt        | |      }t        | t	        |d   ||             S r  )rw   r  r   r  r  )r  r!  r   r  s       rn   index_put_as_masked_fillr    sV    T__..%!23D% T5UD9::rp   c                   ddl m} t        t        j                  t
        j                  j                  j                  j                        }t
        j                  j                  }t        j                  sF|D ||      r<d}|j                  j                  dd       x}r| d| }|t
        j                  _        t        j                   || |||       | S )Nr3   ),_fx_node_is_input_dependent_cudagraph_unsafezLindex_put_ fallback with boolean indexing is not compatible with CUDA graphsstack_trace Found from : 
 )utilsr  r  r  
index_put_rZ   rf   rg   rj   _overloadnamer4   graph_partitionri  rj  disable_cudagraphs_reasonr6   IndexPutFallback)	r  r!  r  r  r  r  r  r   r  s	            rn   index_put_fallbackr    s    C$//177+?+?+F+F+T+TUK
 gg""G""8A\!,,**=$??;?E*;-8C,/)T7FJGKrp   c                $    t        | |||dd      S )NTr|  r  r  r!  r  r  s       rn   r  r    s    gvz4 rp   c                $    t        | |||dd      S )NFTr|  r  r  s       rn   _unsafe_index_put_r  "  s    gvzD rp   c           
        |rLd t        j                  |       |j                         v r$t        fd|D              s|j	                          |j                         dk(  rt        |      dk(  r|d   j                         t        j                  t        j                  fv r_|d   }t        t        |j                               t        | j                                     D ]  }t        |d      } t        | |g||      S t        j                         rt!        | |||      S |D ]E  }||j                         t        j                  t        j                  fv s7t!        | |||      c S  | j                         }	t        |	      }
|rLt#        | j                               r3|
dk(  rt%        | dg      } t!        | |||      } |
dk(  rt%        | g       } | S t'        || j                               }	 t)        || j+                               \  }}|D cg c]  }||j/                         nd  }}t1        | t2              sJ | j	                          |
dk(  rt%        | dg      } t5        ||d      j                               }t        t        |            D cg c]  }|	|   	 }}t7        |	|||||d |      \  }}t9        ||      }| j+                         }|J t        j:                  || j                         |j/                         |||rdnd       }t        j<                  d t        j>                  |       |	      }t@        jB                  jE                  |      |_#        t@        jB                  jI                  |       |
dk(  rt%        | g       } | S # t,        $ r t!        | |||      cY S w xY wc c}w c c}w )
Nc                   t        | t              rt        | j                  t        j                        r| j                  j                         } t        | t        j                        xr t        | j                  t        j                        xrg t        | j                  dd       xrN | j                  j                  j                  t        j                  j                  j                  j                  u S y)Nr  F)rt   rG   rv   r6   r:   r  r   r  r  r  rj   r   rY   r  randpermr  )indices    rn   indice_slice_from_randpermz3index_put_impl_.<locals>.indice_slice_from_randperm,  s     &),FKK1U002vr}}5 V"6;;@VY=V ++22eiinn6M6M6U6UU	 rp   c              3  .   K   | ]  } |        y wr   r   )r   r  r  s     rn   r   z"index_put_impl_.<locals>.<genexpr>:  s      H
39&v.H
rY  r3   r   rz  rw  
atomic_addr   r   rb  rd  output_indexerscatter_moder   r  rv   )%r6   try_get_namer  r#  r  rE  r   r   r   r   r0  r  r   rX  r  rT  r  rS   r  r   r\  rw   r   re  rt   rG   rr   rn  r3  Scatterr  MutationLayoutSHOULDREMOVErZ   rf   rz  r   r{  )r  r!  r  r  r   r}  r  r  r@  r  x_ndimri  r|   rf  rj  re  expected_vals_sizerb  r   scatterr  r  s                        @rn   r  r  )  s   	 ??4 F$9$9$;;C H
=DH
 E
 NN 	aLAAJ  "uzz5;;&??qzs4==?+S-AB 	'AT2&D	''tffjII 113!$DD  I!2uzz5;;6O!O%dGVZHHI ]]_F[FB4>>CSTQ;qc?D!$DQ;b>Dfdnn./FE"=T__&#
 LSSa!-q}}TASOSdI&&&LLN {D1# w~a01::<=K',S\':;!F1I;L;#A	$  F./F __Fjjnn##%!%/\TG ,,T2F
 ''))&1FKGGv&{D"~Kc  E!$DDE T <s   3N N7N<N43N4r  c                   	 t        | |dd      \  }}j                         | j                         		fd}t        j                  | j	                         | j                         ||      S )NFr_  c                     j                   t        j                  k7  r+t        j                          t        j                        }n        }t        j
                  | fd      S )Nc                                     S r   r   )_unsafe_index_fnr[  self_loaders   rn   rI  z8_unsafe_masked_index.<locals>.inner_fn.<locals>.<lambda>  s    K8H8M,N rp   )r   r   r   rY   r   r  )r[  mask_valr  fillr  mask_loaderr  s   ` rn   rb  z&_unsafe_masked_index.<locals>.inner_fn  sK    ::#||K$4ejjAH"3'Hzz($NPTUUrp   rc  )rp  re  rD   r   rw   r   )
r  r  r!  r  rd  r  rb  r  r  r  s
    ` `   @@@rn   _unsafe_masked_indexr    sw    "3gUU#FA ""$K""$KV V  nn	 rp   c           	         t        ||d      }| j                         }t        t        |            D cg c]#  }||   rt	        ||   ||    ||   dz
        nd % }}t        | ||d      S c c}w )Nr   r3   T)r  )r  r   r  r   r  r  )r   r  r!  r  masked_valuerY  r|   clamped_indicess           rn   #_unsafe_masked_index_put_accumulater    s    vq)LJJLE s7|$ 7>ajgaj58)U1X\2dJO  Q$OOs   (A-c                V    t        j                  |t        j                  ||             S r   )rY   r  r  r   minmaxs      rn   r  r    s    ;;sCKKQ/00rp   c                P    t        |       }t        ||||      }t        ||       |S r   )r  r  copy_)r  r  r   r  ru  r6  output_views          rn   as_strided_scatterr    s*    4[FVT6>BK	+sMrp   c                2    t        t        |       |||fi |S r   )scatter_r  )r   r   r@  r  r   s        rn   r  r    s    E!Hc5#888rp   r'  include_selfc               :   t        |t              }t        | ||j                         t	        t
        j                  |r|j                         n
t        |            |r|j                         j                  nd|      rt        j                  | ||||||       |S y )Nznot implr  )rt   rG   rX   r   r
   r   r   r   rw   r6   ScatterFallback)r  r  r   r@  r  r'  r  src_is_tensors           rn   scatter_fallbackr    s     sI.MU[[]#--/S	J!.J 	%	
 rp   r'  c                  |dv sJ |[t        t        j                  t        j                  j
                  j                  j                        }t        || ||||      }||S |dk(  rd}n|dk(  rd}t        | ||||      S )N)Nr   multiplyr  r   ri  r  prod)
r  r  r  rZ   rf   rg   rj   r  r  scatter_reduce_)r  r   r@  r  r'  r  fallback_results          rn   r  r     s    ....~dmmQWW-A-A-H-H-V-VW*sE3v
 &""	:	4eS&99rp   c                0    t        t        |       |||      S r   )scatter_add_r  r   r   r@  r  s       rn   scatter_addr    s    a#uc22rp   c                     t        | |||d      S )Nri  )r  r  s       rn   r  r    s    1c5#u55rp   c                4    t        t        |       ||||fi |S r   )r  r  )r   r   r@  r  reduction_typer   s         rn   scatter_reducer    s    58S%nOOOrp   )r  c          
         |dv sJ t        t        j                  j                               dk(  r dt        j                  j                         v sJ d       t	        t
              rt               t        t        j                  j                   |||      }|r|S t	         t              sJ dt        |j                               v sJ t         j                               }|dk(  rt         dg       t	        t              r)t        j                               dk(  rt        dg      t	        |t              r)t        |j                               dk(  rt        |dg      }|j                         dk(  r S t                j!                          |j#                         t	        t              rj#                         nd  fd} fd	}	d
 }
 j%                         }|J |st'        j(                  | j                          fd|j                         |d       }t'        j*                  d t'        j,                         |      }t.        j0                  j3                  |      |_        t.        j0                  j7                  |       t'        j(                  | j                         |	|j                         | |
|            }t'        j*                  d t'        j,                         |      }t.        j0                  j3                  |      |_        t.        j0                  j7                  |       |dk(  rt         g         S )N)Nri  r  meanamaxaminr3   r   zKaten.scatter_reduce_.two is not the unique overload of aten.scatter_reduce_r  r   r   c                    j                         }t        |      }t        |       }t        j                   |       |dk(  rdn|   d      |<   |S )Nr   r3   Fr@  )r   r   rr   rY   rB  )r[  rY  r  indirect_idxr   rD  r  s       rn   r  z'scatter_reduce_.<locals>.output_indexerO  sU    5zCy11DAIq5:
S rp   c                `    r |       S t        j                  j                               S r   rY   rY  r   )r[  r  r  r  s    rn   r   zscatter_reduce_.<locals>.fnY  s*    c?" <<T^^%566rp   c                    | dk(  ry| J y )Nri  r  r   r  s    rn   backend_reduce_strz+scatter_reduce_.<locals>.backend_reduce_stra  s    U? >!>rp   c                L    t        j                  dj                               S r  r  )r@  r  s    rn   rI  z!scatter_reduce_.<locals>.<lambda>q  s    3<<4>>3C#D rp   r  r  )r   r  r  	overloadsrt   r"   r@  r  r   rG   rQ  r   r   r  rE  r  r  re  rw   r6   r  r  r  rZ   rf   rz  r   r{  )r  r   r@  r  r'  r  r  r  r  r   r  r   zero_outr  r  rD  r  s   `` `           @@rn   r  r  !  s   BBBBD  **,-2T))3355U UU	6 #vc"&  !O dI&&&C)****t}}DqyD1##y!c#,,.&9Q&>3n%#ENN,<(=(BUQC A
c
"CLLN$$&L&0i&@"dJ7 __F::.."D>>#)
 ""006

 gg--f5	""6*
 jjnn~~%'/G ,,T2F
 ''))&1FKGGv&qyD"~Krp   c                T   | j                          | j                         | j                          d  | j                         d   }D cg c]+  }t        j                  j
                  j                  |      - c}t        |      k(  sJ |}t        |      D cg c]
  \  }}||z   c}}t        |      D ]  \  }}	|		d|	z  |<    fdfd}
t        j                  | j                         | j                         |
g ||      S c c}w c c}}w )Nr  c                   t        j                  | t        j                        } r8t        j                  | t        j
                  dt        j                              } t        j                  | t        j
                  |t        j                              } t        j                  | t        j                        } t        j                  | |d      S )N      ?Frw  )
rY   rX  r   r  r   rY  r  r   r  rB  )r   r  r   exacts      rn   scale_fnz$upsample_nearestnd.<locals>.scale_fn  s     NN1emm,3<<U]];<AGGAs||E5==9:LLEKK($$QE::rp   c                    |  d  }| d   } 
g |t        |      D cg c]  \  }}} 	|||       c}}}      S c c}}}w r   )r  )r[  r   r5  r|   r5  r   i_sizes
inv_scalesr  r  rh  s         rn   r   zupsample_nearestnd.<locals>.fn  s`    H!HWaW#aW:UVVJAq$8Aq$'VW
 	
Vs   Arc  )realize_hintre  r   rZ   rf   r   r.  r   r  rs   rD   r   rw   r   )r   rh  scales_xr  r  batchr|   o_sizesor  r   r  r  r  rh  s      ``      @@@@rn   upsample_nearestndr    s    NN}}HjjlA23GJJL1"E6=>qww))!,>Gx=AG$'$9:DAq!a%:Jh' (5%KJqM(	;
 
 ||~kkm!!!	 9 ?
 ;s   0D'D$c                "    t        | ||fd      S )Nr3   r  r  r   rh  r  s      rn   upsample_nearest1dr    s    avi1==rp   c                $    t        | ||fdd      S )Nr3   Tr  r  r  r  s      rn   _upsample_nearest_exact1dr    s    avi1DIIrp   c                $    t        | |||fd      S )Nr1   r  r  r   rh  scales_hscales_ws       rn   upsample_nearest2dr    s     ax.BaHHrp   c                &    t        | |||fdd      S )Nr1   Tr  r  r  s       rn   _upsample_nearest_exact2dr    s     ax.BatTTrp   c                &    t        | ||||fd      S )Nr   r  r  r   rh  scales_dr  r  s        rn   upsample_nearest3dr     s     ax8.LPQRRrp   c                (    t        | ||||fdd      S )Nr   Tr  r  r  s        rn   _upsample_nearest_exact3dr    s#     	;8X6!4 rp   c                ,     t         fd|D              S )Nc              3  J   K   | ]  }t        j                  |        y wr   r  )r   r   r   s     rn   r   z$_create_constants.<locals>.<genexpr>  s     6Aa'6s    #)r   )r   r}   s   ` rn   r  r    s    6666rp   c                    | j                         | j                         fd}t        j                  | j	                         | j                         |      S )Nc                    t        |       } t        |       t              k(  sJ D ]  }|   dz
  | |   z
  | |<     |       S r  )rr   r   )r[  r   r3  r(  rh  s     rn   rd  zrev.<locals>.loader  sW    3i3x3u:%%% 	3Cc
Q#c(2CH	3 }rp   rc  )re  r   rD   r   rw   r   )r   r3  rd  r(  rh  s    ` @@rn   revr    sM     }}HJJLE ||~kkm	 rp   c                   d } |       ryt        |      dk7  st        | j                               dk7  ry| j                          t        | t        j
                        rt        | j                  t        j                        rt        | j                  j                  t        j                        s>t        j                  rNt        | j                  j                  t        j                        r | j                  j                  j                  sy| j                          t	        j                  |       \  }}|j                  }|d   dk7  ry|d   dk7  s|d   dk7  s|d   dk7  ry|d   }|dk(  ry|d   }|j                   d   }	||	|z   k  ry| j                  j                  j                  }
|j                   d   |j                   d   |z   g}|t"        j$                  j&                  |
<   t)        | ||j                  |j*                  d	      }t-        |d|	|	|z   d
      }t/        ||       t0        d   dxx   dz  cc<   |S )z
    This optimization changes the semantics of padding from 'clone'
    style to 'view' style.

    Thanks to functionalization, this change can still maintain numerical
    correctness.
    c                    t         j                  j                  } | yt        | j                        }t        |      dk(  rD|d   j                  t        j                  j                  t        j                  j                  fv ryy)a  
        Conservatively check if padding can be fused with downstream op.
        1. if the downstream op is a sum, then there is little benefit to
           do inplace padding
        2. if the downstream op is a matmul, doing inplace padding can
           save membw.
        Tr3   r   F)rZ   rf   rg   r   rh   r   rj   r  mmr  addmm)rg   rh   s     rn   _padding_can_be_fusedz6inplace_constant_pad_nd.<locals>._padding_can_be_fused  sk     ww++l(()u:?uQxGGOOJJ3
  
 rp   Nr   r1   r3   r   r   Fr  )r   r|  r}  r  inductorinplace_padding)r   r   r  rt   r6   rG   rv   r   r  r4   can_inplace_pad_graph_inputr  r   freeze_layoutr  r  r   rZ   rf   buffer_to_padded_sizer  r  r  fill_r   )r   paddingr  r  r  r  rl  npadstride0rowsizebufnamepadded_size	resized_xsliced_xs                 rn   inplace_constant_pad_ndr    s   (  7|qC

-2
 IIK
 q",,'!&&"--0qvv{{B$5$5622qvv{{BNN; vv{{OO((+IAvmmGqzQqzQ'!*/WQZ1_1:DqyajGkk!nG4ffkkG;;q>6;;q>D#89K-8AGG!!'*	16I iQg7T>QVWH	(JZ*+q0+rp   c           	     0   t        d |D              sy| j                         }t        |      }t        t	        |ddd   |ddd               }d}d}t        |      D ]*  \  }\  }	}
|	dk7  r y|
dkD  r| y|dz
  |z
  }|
}$|
dk  s* y |y| j                         }||j                  dk(  ryt        |      }|||<   | j                         } t        |      |      } t        |      |||      }t        d   d	xx   dz  cc<   t        | |g|      S )
a7  Decompose right-pad into cat([x, fill], dim) and delegate to cat lowering.

    The cat lowering already has heuristics for choosing between pointwise_cat
    (fusion) and ConcatKernel (memory planning / zero-copy).  By routing through
    cat() we reuse those heuristics rather than duplicating them here.
    c              3  <   K   | ]  }t        |t                y wr   )rt   r   r   r  s     rn   r   z_pad_as_cat.<locals>.<genexpr>y  s     3az!S!3r$  Nr1   r3   r   r   r`  r  pad_rewritten_as_cat)r#  r   r   rr   r  rs   rw   r   r   r   r  r   rL  )r   r  r  r(  r  	pad_pairspad_dim
pad_amountr|   leftrq  r   	pad_shaper   fill_value_typed
pad_tensors                   rn   _pad_as_catr'  o  sK    3733JJLEu:DS1wqt!t}56I GJ%i0 	=D%1919"QhlGJQY	  \\^FfkkU2 UI#IgKKME+}U+J75#$45vJ Z/0A50:((rp   c                   t        |      dz  dk(  sJ t        d |D              rt        |       S t        j                  rt        | |      }|r|S t        | |      }||S | j                         }t        t        t        t        |d d d   |dd d                           t        |      t              z
  g D ]?  \  }}j                  t        j                  j                  j                  |      |f       A t        |d        }g t        |d        D ]C  \  \  }}	}
j                  |
       |j                  t!        j"                  |
|z   |	z                E t        |      t        |      k(  sJ  t%        | j'                                     fdfd}| j)                         t+        j,                  | j/                         | j'                         ||      S )Nr1   r   c              3  &   K   | ]	  }|d k(    ywr   Nr   r  s     rn   r   z"constant_pad_nd.<locals>.<genexpr>  s     
#a16
#r"  r3   c                B    g }t         	d        D ]J  \  }\  }}}|dk7  r|j                  t        |d             |dk7  s0|j                  t        ||             L t	        j
                  t        j                  |      }t        j                  | 
fd      S )Nr   c                             S r   r   )r@  rh  s   rn   rI  z/constant_pad_nd.<locals>.mask.<locals>.<lambda>  s     rp   )	r  rx   range_mask_lowrange_mask_highr  r'  rY   r  r  )r@  r  r[  rZ  rY  r  boundsr  
mask_sizesr  rh  s   `     rn   r  zconstant_pad_nd.<locals>.mask  s    (+E!"Ivz(J 	:$C#tfaxN323qyOC89		:
 $/zz$ 7DDrp   c                    t        | d        }t        | d        D ]  \  }\  }}|j                  ||z
          t        |      t        |       k(  sJ  |      S r   )rr   r  rx   r   )r@  r`  r[  rZ  _highbounds_precompr  r  s        rn   	offset_fnz"constant_pad_nd.<locals>.offset_fn  si    rO	!$U12Y!? 	(C#uS3Y'	(9~U+++Irp   rc  )r   r#  r  r4   r  r  r'  r   rr   r.  r  rx   rZ   rf   r   lookup_precomputed_sizer   r3  r   r   re  rD   r   rw   )r   r  r  rz   r(  lhrh  rZ  rY  r   r4  r/  r3  r  r0  r  rh  s     `         @@@@@@rn   rM  rM    s   L1"""

#7
##Qx%a*=J a*
-C

JJLE(4GCaCL'!$Q$- @ABCFE
S[ A 68N P1qww//GGJANOP uRay/KJ qr3 <dT$5<<s
T(9:;< {s5z)))-q{{}-j9JE E }}H||~kkm	 rp   c                    t        j                  t        j                  | t        j                        t        j                  t        j                  |      t        j                              S r   )rY   r  rX  r   r  r   rj  )r|   rZ  s     rn   r-  r-    s@    66q%++&u}}S)5;;7 rp   c                    t        j                  t        j                  | t        j                        t        j                  |t        j                              S r   )rY   r  rX  r   r  )r|   rY  s     rn   r.  r.    s7    66q%++&tU[[) rp   c                V    t        j                  t        | |      t        | |            S r   )rY   r  r-  r.  )r|   rY  rZ  s      rn   
range_maskr;    s'    88q#4  rp   c                      j                          d   j                         xs dgz   fd}|S )Nr   c                N   | d   |  d  t        j                  t        j                  t	              D cg c]  }t        |   |   
|   z   
|          ! c}      }	rt        j                  |fd      S t        j                  |fd      S c c}w )Nc                 4     t               g       S )Nr   )constant_boundary_condition)r   ihpad_fill_valueprefixr   s   rn   rI  z;constant_boundary_condition.<locals>.load.<locals>.<lambda>	  s$    O3A~3O"fNrN rp   c                      g        S r   r   )rA  rC  rh  s   rn   rI  z;constant_boundary_condition.<locals>.load.<locals>.<lambda>  s    (>V>b>*B rp   )r  r'  rY   r  r  r;  r  )r@  r|   r  rA  rC  r   r  r7  rB  r  	padding_hr   rh  s      @@rn   r  z)constant_boundary_condition.<locals>.load  s    uC45\HHLQRUJWqZ1qtil2Yq\MBW
  JJ 
	
 D"BJO
	
 Xs   $B"
)r   re  )	r   r  r  rB  r   r  r7  rE  rh  s	   ````` @@@rn   r@  r@    sE     	


cTUA}}H$A39I
 
* Krp   dilationc               (   |dgt        |      z  }t        | d||   z  z   ||   ||   dz
  z  z
  ||   dz
  z   ||         }|rt        | d||   z  z   ||   ||   dz
  z  z
  d||   dz
  z  z   ||         }t        j                  j                  j                  t        j                  |dz
  ||   z  | z
  ||   z
  d            r|dz  }t        j                  j                  j                  t        j                  ||            rd}||fS |}||fS )Nr3   r1   r   F)	r   r+   rZ   rf   r   r1  r   r  r2  )	r   r|   kernel_sizer  r  	ceil_moderG  x_outx_alts	            rn   pooling_sizerM    sN   3W%	A
NXa[KNQ,>??6!9q=Qq	E
 '!*nqk[^a/01 6!9q=!" 1I
 77**HHeai6!9,q071:=qA
 QJE77**588E5+ABI ) E)rp   c               l    t        | |      } t        j                  t        j                  |       }|dkD  S )N   )rT   r  r'  operatorr  )rI  n_dimwindow_sizes      rn   %should_fallback_max_pool_with_indicesrS  3  s/    {E2K""8<<=Krp   assert_fallbackc                  |dk(  rdg|z  }|dk(  rdg|z  }|s|}t        ||      }t        ||      }t        ||      }t        ||      }t        | t              sJ t        |      |k(  sJ t        |      |k(  sJ t        |      |k(  sJ t        |      |k(  sJ t        | j	                               |dz   |dz   fv sJ t        ||      }|||k(  sJ |||||fS )Nr   r3   r1   rQ  )rT   rt   rG   r   r   rS  )r   rI  r  r  rG  rQ  rU  use_fallbacks           rn   max_pool_checksrY  9  s    !|#+1}3;{E2K&%(F7E*GHe,Ha###{u$$$v;%w<5   x=E!!!qzz|EAI 66668ERL"...<??rp   c                  | j                          | j                  d   }| j                   d  }t        t              D 	cg c]  }	t	        ||	   |	||       c}	 \  }
}| j
                  }|t        j                  u rdn5|j                  rt        d      nt        j                  |      j                  }t        |      t        |
      z   }t              st        |      st        d D              rt        | |      n| j                         fd}t!        j"                  d| | j%                         |||||      }t!        j"                  d	| | j%                         t        j&                  ||||      }t)        |j*                  j*                  t               r|j-                          t)        |j*                  j*                  t               r|j-                          ||fS c c}	w )
NrF  F-infc              3  &   K   | ]	  }|d kD    ywrb  r   r/  s     rn   r   z)_max_pool_with_offsets.<locals>.<genexpr>u  s     ,EqQU,Er"  r?  c                    | d   }|  d  }t              D cg c]  }||   	|   z  ||   |   z  z   |   z
  ! }} 
g ||      S c c}w r   r  )r[  reduction_idxrC  bhr|   rA  rG  rQ  r  r  rh  s         rn   fn_innerz(_max_pool_with_offsets.<locals>.fn_innerz  s    Wuf%\ 5\
 UVAY=#3hqk#ABWQZO
 
 &2''	
s   $Ar  r  
input_noder   	dst_dtyper  rb  rd  reduction_rangesargmax)r  rY  r  r  rM  r   r   r   r  r;  r  r  rr   r$  r@  re  rE   r   rw   r  rt   rv   r  )r   rI  r  r  rG  rJ  rQ  r  dhwr0  dhw_outr   	min_valuerj  ra  r  offsetsrh  s     ``` `          @rn   _max_pool_with_offsetsrk  V  s    NNGGGeVE
''5&'
C
 5\	

  A;X

GY GGE EJJ 	$66eFmEKK<N<R<R  E{T']*H
7|s9~,EH,E)E.q)G==?( ( ||~$	F ||~++$	G &++""I.',,##Y/7?s

s   G8c           
        t        |      }t        | |||||d      \  }}}}}t        j                  d      5  t	        | ||||||      \  }}	|t        |	t        j                        fcd d d        S # 1 sw Y   y xY w)NFrT  rO  unroll_reductions_thresholdrW  )r   rY  r4   r   rk  r   r   r/  )
r   rI  r  r  rG  rJ  rQ  r  r  rj  s
             rn   !_low_memory_max_pool_with_offsetsro    s     E 1@	1-K(A 
"	5 
50
 x44
5 
5 
5s   1A55A>c                P   t              | j                         t        j                  t	        j
                  t        j                              fd}t        j                  | j                         t        j                  || j                               }|S )Nc                     	|       }t        j                  |
      }t        j                  |      } | |      }t        j                  t        j
                  | d        t        j                        S r   )rY   rB  r5   _flattened_index_to_ndrX  _flatten_indexr   r  )r[  r  offset_sympyr_  idhwincrements_to_index
input_sizerI  rQ  offsets_loaderrR  s        rn   offsets_to_indicesz4_pool_offsets_to_indices.<locals>.offsets_to_indices  sm    $,,V[A&==lKX"36~~))$
E670CDekk
 	
rp   rc  )r   re  r   r  r  r'  rP  r  rD   r   rw   r   r  r   )	rj  rI  rw  rv  ry  r!  rQ  rx  rR  s	    ```  @@@rn   _pool_offsets_to_indicesrz    s     E((*N--	 0 0{ KLK
 
 !!#kk#!	G Nrp   c                L    t        |      fd}t        | |||      S )Nc                    |  d  }t              D cg c]  }||   |   z  ||   |   z  z   |   z
  ! c}S c c}w r   r^  )r[  r_  r`  r|   rG  rQ  r  r  s       rn   rv  zD_low_memory_max_pool_offsets_to_indices.<locals>.increments_to_index  s_    %\ 5\
 UVAY=#3hqk#ABWQZO
 	
 
s   $<)r   rz  )rj  rI  rw  r  r  rG  rv  rQ  s      ``` @rn   '_low_memory_max_pool_offsets_to_indicesr}    s-     E
 $j*= rp   c           	         t        | |||||      \  }}}}}t        | ||||||      \  }}	t        |	|| j                  | d  |||      }
||
fS )NrW  )rY  rk  r}  rY  )r   rI  r  r  rG  rJ  rQ  r  rz   rj  r!  s              rn   _max_pool_with_indicesr    s}     1@	;1-K(A *	;9ELC 6	G <rp   c           	     &    t        | |||||d      S Nr1   rW  r  r   rI  r  r  rG  rJ  s         rn   max_pool2d_with_indicesr          "	;9A rp   c           	     &    t        | |||||d      S Nr   rW  r  r  s         rn   max_pool3d_with_indicesr  %  r  rp   c                   dk(  rddg|dk(  rddg}st        |t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        |      dk(  sJ t        |j                               dv sJ | j	                          | j                         }t        |t              rt        |j                  j                  t              r|j                  j                  }	|	j                         }
|
J t        j                  d t        j                  |
|	j                         |	j                               |	      }|j                          |j                         }n|j                         }|d uxr |d   dk(  xs |d uxr |d   dk(  }t        d |D              rt!        | ||||      S |j                         ^ }}| j                         ^ }|j#                         | j#                         t%        |j                               }t'        fdt)        d   dz        D              t'        fd	t)        d   dz        D              z  }|d
kD  rt!        | ||||      S |j                         fd}t        j*                  | j                         | j                         ||      }|rt        j,                  j/                  |      S |S )Nr   r3   r1   r  )r   r   r   r  c              3  &   K   | ]	  }|d k7    ywrb  r   r/  s     rn   r   z3max_pool2d_with_indices_backward.<locals>.<genexpr>d  s     
$a16
$r"  c              3     K   | ]=  }t        t        |d          t        d t        |d    z
  d                z
  d       ? ywr   r3   Nr  r+   r   r7  rI  r  s     rn   r   z3max_pool2d_with_indices_backward.<locals>.<genexpr>q  M       	HQq	"SHQQ5GPQ,S%TTVWX   AAc              3     K   | ]=  }t        t        |d          t        dt        |d    z
  d                z
  d        ? ywr3   r   Nr  r   wrI  r  s     rn   r   z3max_pool2d_with_indices_backward.<locals>.<genexpr>u  r  r  rO  c                   | ^ }}}t        j                  |z  |z   t        j                        }|d   z   }|d   z   }t        j                  t	        |d   z
  d   z   d         t        j                        }t        j                  t	        |d   z
  d   z   d         t        j                        }t        j                  t	        |d         dz   t        j                        }t        j                  t	        |d         dz   t        j                        }t        j
                  |t        j                  dt        j                              }t        j
                  |t        j                  dt        j                              }t        j                  |t        j                  t        j                              }t        j                  |t        j                  t        j                              }d }	t              D ]3  }
t              D ]!  }t        j                  |t        j                  |
t        j                              }t        j                  |t        j                  |t        j                              }g |t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d   d      t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d   d      } |      } |      }t        j                  ||      }|	;t        j                  ||t        j                  dt        j                              }	t        j                  t        j                  t        j                   ||      t        j                   ||            |      }t        j                  |t        j                  |	|      |	      }	$ 6 |	J |	S )Nr   r3   Frw  rz  r  )rY   rX  r   r  r+   r  rY  r  r  r   rB  r  r  r  r  r  r  )r[  rC  r7  r  
index_testphstartpwstartphendpwendgradientph_pw_phpw
grad_indexindex_actual	grad_partr   r  grad_loaderh_window_sizerL  indices_sizerI  r  pooled_heightpooled_widthr  w_window_sizewidths                      rn   r   z,max_pool2d_with_indices_backward.<locals>.fn  sA   A^^AIM5;;?

N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM' #	WC]+ "WWWWcll3&DEWWWcll3&DE))Bs||Au{{7S(TU$R(# ))Bs||Au{{7S(TU$R(#
  .j9'
3	|Z8#"yyy#,,sEMM*J H 88FF2u-FF2u- D  #yyswwx/KXVHE"W#	WH ###rp   rc  )rt   rG   r   r   r  r  rv   rD   rw   r6   r  r  r   decide_layoutr  r$  )fallback_max_pool2d_with_indices_backwardre  rr   r  r  r   r  rk  )grad_outputr   rI  r  r  rG  rJ  r!  	gO_striderv   r   x_bufferx_strideis_channels_last_batch_heightr  rj  rR  r   rz   r  r  rL  r  r  r  r  r  s     ```                @@@@@@@@rn    max_pool2d_with_indices_backwardr  9  s    !|a&1}q6a###{q   v;!w<1x=Aqzz|&&& ,,.I!YJqvv{{I$Fvv{{"!!!$$$$nn&]]_
 
 	 &&(%%' ,A!1A 3)A,!"3  
$8
$$8K(Iw
 	
  jjlVWe&1&:&:&<#Q|((*N))+KAJJL!H {1~)* M  {1~)* M
  -/KR8K(Iw
 	
 ##%L9 9v 

%%'##%	C 44S99
rp   c                6    | j                         fd}|S )Nc           
        	
 |\  	|\  
|\  }}t        j                  t        j                  t        j                  z   t        j
                        t        j                  |t        j
                              t        j                  t        j                  
	z   t        j
                        t        j                  |t        j
                                    }t        j                  |	 
fd      S )Nc                 ,     g  z   z         S r   r   )h_start_indexrA  iwrC  w_start_indexrh  s   rn   rI  z3pad_adaptive_loader.<locals>.load.<locals>.<lambda>  s&    HNvN}r'9N=2;MNO rp   )rY   r  r  rX  r   r  r  )rC  
incrementsstart_indicesend_indicesh_end_indexw_end_indexr  r  rA  r  r  pad_valrh  s   `      @@@@rn   r  z!pad_adaptive_loader.<locals>.load  s    B'4$}#. [xxFF}r15;;?{EKK8 FF}r15;;?{EKK8	
 zzOO
 	
rp   re  )r   r  r  rh  s    ` @rn   pad_adaptive_loaderr    s    }}H
, Krp   c                    t        j                  | ||      }t        j                  |||      }t        j                  | ||      }t        j                  |||      }	||||	fS )N)out_diminp_dim)r  r)  )
r  r  h_inw_inh_outw_outr  r  r  r  s
             rn    compute_indices_adaptive_poolingr    sa    %%k5$OM##IudKK%%k5$OM##IudKK+}kAArp   c                d    |\  }}|\  }}	t        | |||||	      \  fd}
|
S )Nc                   | ^ }}} |      } |      } |      } |      }d }	t        j                  t        d         t        d               D ]$  \  }
} |||
|g||g||g      }|	|}	 ||	      }	& |	S r  )r  productr  )r[  rd  rC  r`  bwr  r  r  r  r  rA  r  r;  h_end_index_fnh_start_index_fnkernel_maxes
pooling_fnw_end_index_fnw_start_index_fns                rn   r   z _adaptive_pooling_fn.<locals>.fn   s    R(,$R((,$R(''l1o(>lSTo@VW 
	1FBR.k*	C ~#C0
	1 rp   r  )r  r  r  in_sizes	out_sizesr  r  r  r  r  r   r  r  r  r  s     `  `     @@@@rn   _adaptive_pooling_fnr    sP     JD$LE5 	)YdE5	
 . Irp   c                h   
 |\  }|\  }}t        | ||||      \  

fd}	|	S )Nc                   | ^ }}} |      } |      } |      } |      }d }	d }
t        j                  t        d         t        d               D ]  \  }} ||||g||g||g      }t        j                  ||z   z  |z   |z   t
        j                        }|
|}
n+t        j                  t        j                  ||	      ||
      }
|	|}	| ||	      }	 |
S r  )	r  r  r  rY   rX  r   r  r  gt)r[  rd  rC  r`  r  r  r  r  r  maxvalmaxindexrA  r  r;  r@  r  r  r  r  r  r  r  s                  rn   r   z)_adaptive_pooling_fn_with_idx.<locals>.fn)  s   R(,$R((,$R(''l1o(>lSTo@VW 	1FBR.k*	C NN#t+m;b@%++E  99SVVC%8%J~#C0)	1, rp   r  )r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  s     `  `    @@@@@rn   _adaptive_pooling_fn_with_idxr    sQ     JD$LE5 	)YdE5	
! !F Irp   c                6     j                         t        j                  k(  rt        d      t	         t
              sJ t        |      dk(  sJ  j                           j                         ^ }}}t        j                  j                  j                  |      }t        j                  j                  j                  |      }|\  }}||k(  r||k(  rt               S |dk(  s|dk(  r2g |||}t        | j                          j                               S ||z  dk(  r,||z  dk(  r$t!        ||      t!        ||      g}t#         |      S t%        ||z   dz
  |      }	t%        ||z   dz
  |      }
t'        |      ||gz   } j                         }|	|
z  }|dkD  rt)         |      S d }d }t+        |||	|
g||g||gt,        j.                  	      t1        t3                      fd
}t5        j6                   j                         |||      }|S )Nz0'adaptive_avg_pool2d' not implemented for 'Long'r1   r   r`  r3   rO  c                     t        | |z  |      S r   r+   r@  r  r  s      rn   r  z)_adaptive_avg_pool2d.<locals>.start_indexz      733rp   c                2    t        | dz   |z  |z   dz
  |      S r  r  r  s      rn   r  z'_adaptive_avg_pool2d.<locals>.end_index}  "    g-7!;WEErp   r  r  r  r  r  r  c                ^    t        j                   | t                     |             S r   )rY   truedivr  )r[  fn_sumones_loaderr   s    rn   r   z _adaptive_avg_pool2d.<locals>.fn  s-    {{3+A./[1I
 	
rp   rc  )r   r   r  r  rt   rG   r   r  r   rZ   rf   r   r.  r  ri  rw   r+   
avg_pool2drJ   rr   fallback_adaptive_avg_pool2dr  rY   r   r  	ones_likerD   r   )r   rh  r  r  r  r  r  o_sizerI  h_kernel_maxw_kernel_maxrj  r   rR  r  r  r   rvr  r  s   `                 @@rn   _adaptive_avg_pool2dr  T  s   {{}#MNNa###{q   NNUD$77%%d+D77%%d+DLE5 u}QxzUaZ'5'%''V1;;=HHe|qTE\Q.e,htU.CD![))D5L1,u5LD5L1,u5LE{eU^+HKKME-KR+A{;;4F ""L1%.77F &il3K

 
		||~	
B Irp   c                     j                         t        j                  k(  rt        d      t	         t
              sJ t        |      dk(  sJ  j                           j                         ^ }}}t        j                  j                  j                  |      }t        j                  j                  j                  |      }|\  }}|dk(  s|dk(  r\g |||}t        | j                          j                               t        |t        j                   j                               fS ||z  dk(  r||z  dk(  rt        t!        ||z   dz
  |      }t!        ||z   dz
  |      }	t#        |      ||gz   }
 j                         }||	z  }|dkD  rt%         |      S d }d }t'        ||||	g||g||gt(        j*                  	      t-        ||||	g||g||gt(        j*                  	       fd
} fd}t/        j0                   j                         |||
      }t/        j0                   j                         t        j                  ||
      }||fS )Nz,adaptive_max_pool2d not implemented for Longr1   r   r`  r3   rO  c                     t        | |z  |      S r   r  r  s      rn   r  z(adaptive_max_pool2d.<locals>.start_index  r  rp   c                2    t        | dz   |z  |z   dz
  |      S r  r  r  s      rn   r  z&adaptive_max_pool2d.<locals>.end_index  r  rp   r  c           	     <     | t        t        d                  S Nr[  r  r;  )r[  inner_func_max_valr   s    rn   inner_fn_max_valz-adaptive_max_pool2d.<locals>.inner_fn_max_val      !#':1eFm'LMMrp   c           	     <     | t        t        d                  S r  r  )r[  inner_func_max_idxr   s    rn   inner_fn_max_idxz-adaptive_max_pool2d.<locals>.inner_fn_max_idx  r  rp   rc  )r   r   r  r  rt   rG   r   r  r   rZ   rf   r   r.  ri  rw   
ValueErrorrJ   rr   fallback_adaptive_max_pool2dr  rY   r  r  rD   r   )r   rh  r  r  r  r  r  r  r  r  rj  r   rR  r  r  r  r  r  rir  r  s   `                  @@rn   adaptive_max_pool2dr    sL   {{}#IJJa###{q   NNUD$77%%d+D77%%d+DLE5zUaZ'5'%''V1;;=H%%++allnK
 
 	
 e|qTE\Q.D5L1,u5LD5L1,u5LE{eU^+HKKME-KR+A{;;4F ."L1%.;; 7"L1%.;;NN 
		||~!	
B 
		||~kk!	
B r6Mrp   c                h               j                          fd}|S )Nc                `   j                         }t        |      dk(  r[t        |       dk(  r d| d   dz
  
z
  g      }nJt        |       dk\  r | d   | d   dz
  
z
  g      }n$ dddz
  
z
  g      }n g | dz
  
z
        }t        j                  |j	                               }t        j                  z
  t
        j                        }t        j                  dz
  t
        j                        }t        j                  t        j                  |t
        j                        t        j                  |t
        j                              }t        j                  t        j                  |d      d|      }t        j                  ||z   |z        t        j                  ||z        z
  }t        j                  |t
        j                        }t        j                  ||      }	t        j                  t        j                  |	||      t        j                               S )Nr   r3   r   r1   )r   r   rY   rX  r   r   r  r  r   float64r  r  rN  r  rB  r   r  )rC  r|   samples_shapesamplei_exprdiffout_sz_exprr`  seq_ir  r   in_sz	kernel_szndimsout_szsamplessamples_loaders             rn   r  z)_fractional_pooling_offsets.<locals>.load  s     ((*}"6{a (F1Iuqy3(GHV! (F1Iuqy3(OP (Auqy3(?@ $$>f$>eai#o$>?F7#4#4#67~~ei/=nnVaZ=LLu}}-s||K/W
 		#&&a0!U;		6F?e34syy%7PPUEKK0vvfk*$$SYYtUD%A5==QVCWXXrp   r  )r  r  r  r  r   r  r  r  s   `````` @rn   _fractional_pooling_offsetsr    s@    C[F#JE#I((*N!Y !YF Krp   c                "    t        | |||d      S r  _fractional_max_poolr   rI  rh  random_sampless       rn   fractional_max_pool2dr        ;^STUUrp   c                "    t        | |||d      S r  r  r  s       rn   fractional_max_pool3dr  "  r  rp   c                   | j                          | j                  d   | j                   d  }}t        j                  d      5  t	              D cg c]  }t        |||||       c}| j                         fd}fdt        |      t        |      z   }	| j                         }
t        j                  d| | j                         |
|
||	|      }t        j                  d| | j                         t        j                  |
||	|      }t        |t              sJ |       t        |j                   j                   t              r|j#                          t        |t              sJ |       t        |j                   j                   t              r|j#                          t%        ||| j                        }||fcd d d        S c c}w # 1 sw Y   y xY w)	NrO  rm  )r  r  r  r  r  r   c                6    | d   } g | | |            S r   r   )r[  r_  rC  rv  rQ  rh  s      rn   ra  z&_fractional_max_pool.<locals>.fn_inner:  s.    5&\FOfO':3'NOPPrp   c                ~    | d   }|  d  }t              D cg c]  } |   |||         ||   z    c}S c c}w r   r^  )r[  r_  rC  bdhwr0  dhw_index_fnrQ  s        rn   rv  z1_fractional_max_pool.<locals>.increments_to_index>  s^    5&\Fvw<D u  QQ0=3CC  s   :r  rb  rf  )r  rY  r4   r   r  r  re  rr   r   rE   r   rw   r   r  rt   rG   rv   r  rz  )r   rI  rh  r  rQ  r  inp_dhwr0  ra  rj  r   r  rj  r!  r   rv  rh  s       `         @@@rn   r  r  '  s   NNWWWuf%qwwvw'77E	"	5 = 5\

  (&"%

 ==?	Q	 ;k!22!! <<>(	
 ""#<<>kk(	
 &),4f4,fkk&&	2NN'9-6w6-gll''3OO*[!''+>
 w{= =

= =s   GG1EGGG%c                ~     j                           j                         ^ }}}t        j                  j                  j                  |      }t        j                  j                  j                  |      }|^ }}}	||z  dk(  r,||	z  dk(  r$t         t        ||      t        ||	      gd      S t        ||      }
t        ||	      }d fd}t        ||
|g||g||	gt        j                         fd}t        j                   j                          j                         |t!        |            }|S )	Nr   r3   )divisor_overridec                F    t        | |z  t        j                  |            S r   )r*   r   r  r  s      rn   r  z0upsample_nearest2d_backward.<locals>.start_index  s    uwg(>??rp   c                     | dz   ||      S r  r   )r@  r  r  r  s      rn   r  z.upsample_nearest2d_backward.<locals>.end_index  s    EAI99rp   r  c                (     | t                    S r   )r  )r[  r  r   s    rn   r   z'upsample_nearest2d_backward.<locals>.fn  s    c.q122rp   rc  )r  r   rZ   rf   r   r.  r  r+   rJ   r  rY   r   rD   r   rw   r   rr   )r   rh  rw  r  r  r  inp_hinp_wout_hout_wr  r  r  r   r  r  r  s   `              @@rn   upsample_nearest2d_backwardr+  k  s1    NNJJLVUEGG&&u-EGG&&u-E 'VUEu}eemq0&(>?RS
 	
 5%(L5%(L@: ""L1%.77F3 
		||~kkmJ
B Irp   c           
     (    t        | ||||||d      S )Nr1   r?  _avg_poolndr   rI  r  r  rJ  count_include_padr#  s          rn   r  r    )     		 	rp   c           
     (    t        | ||||||d      S )Nr   r?  r-  r/  s          rn   
avg_pool3dr3    r1  rp   c                &   ssdgz  t              t              t              t        | t              sJ t              k(  sJ t              k(  sJ t              k(  sJ t        | j	                               dz   dz   fv sJ | j                          | j	                         d   }| j	                          d  t        t              D 	cg c]  }	t        |	   |	|       c}	 \  }
}t              st        |      rt        | d      d}n| j                         d}t        |      t        |
      z   }| j                         }t        | t        j                   d      }fd	}t#        j$                  t&        j(                        }|d
kD  r6t        d t              D              rt*        dz
     } || ||      S |d
k  rt-        j.                  d
      nt1        j2                         }| j5                         }|J |5  t7        j8                  d| |||||      }d d d        t;        j<                  d      r4t        |j<                  j<                  t6              r|j?                          |r|r|r|n|}tA        ||      }n=fd}tC        j8                  | j5                         |||      }tA        ||      }tE        ||      S c c}	w # 1 sw Y   xY w)Nr   r3   r1   r  r?  TF)r   r   c                    | d   }|  d  }|}t              D cg c]  }||   |   z  ||   z   |   z
   }} 	g ||      S c c}w r   r^  )
r[  r_  rC  r`  rA  r|   r   r  r  rh  s
         rn   ra  z_avg_poolnd.<locals>.fn_inner  sq    Usd#Z>CCjIbefQi"Q%''!*4II&2'' Js   A	rO  c              3     K   | ]D  \  }}t         j                  j                  j                  t	        j
                  ||             F y wr   )rZ   rf   r   statically_known_truer   Ne)r   r   r5  s      rn   r   z_avg_poolnd.<locals>.<genexpr>  s;       Aq 	
..uxx1~> s   A
Arm  ri  rb  rv   c                ~   |  d  }g }t              D ]  }||   |   z  |   z
  }t        |
|   z   	|   |   z         }st        |d      }t        |	|         }t        j                  ||z
  t
        j                        }|j                  |        t        j                  t        j                  |      S r  )r  r.   r-   rY   rX  r   r  rx   r  r'  r  )r[  r`  divide_factorsr|   hstarthendfactorr0  r   r7  rI  r  r  s          rn   fn_countz_avg_poolnd.<locals>.fn_count;  s    cTUBN3Z .A*WQZ76KN2AaD71:4EF( ^FtQqT?Dvu{{C%%f-. ##CGG^<<rp   rc  )#rT   rt   rG   r   r   r  r  r  rM  r$  r@  re  rr   r   r   r   rK  r  r'  rP  r  fallbacks_avg_poolndr4   r   
contextlibnullcontextrw   rE   r   r   rv   r  div_primrD   r   )r   rI  r  r  rJ  r0  r#  r   r  r|   r  
ceil_modeshad_paddingrj  r   output_dtypera  rR  fallbackcontextr   r  divisorr  r>  divide_factorr7  rh  s    ``` ` `                  @@rn   r.  r.    s    #){C0K&#&F7C(Ga###{s"""v;#w<3qzz|q#' 2222NNJJL3$E	

cTUA 3Z

 1q+vw	J

E: 7|s:.q#3?==?E{T%[(HKKME%	;CC!L( ""8<<=KRC  V,   (a0
 	
 " 	4##%  \\^F	 

 "(	


 rwwJrww||Y$G


*&6"K"g&	= 	= "((<<>	
 "m,FE""I

t

 

s   )L2LLc                   dk7  sJ d       ssddgt        | t              sJ t        |t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        |j                               dv sJ | j	                          |j                         ^ }t        d|      \  }	}
t        d|      \  }}| j                         d   xs d   xs |
xs || j                         ^ }t        |j                               }|j                         }t        fdt        d   dz        D              t        fdt        d   dz        D              z  }|dkD  rt        | ||      S fd	fd
}t        j                  | j                         |||      }|S )Nr   divisor must be not zeror1   r  r3   c              3     K   | ]=  }t        t        |d          t        d t        |d    z
  d                z
  d       ? ywr  r  r  s     rn   r   z&avg_pool2d_backward.<locals>.<genexpr>  r  r  c              3     K   | ]=  }t        t        |d          t        dt        |d    z
  d                z
  d        ? ywr  r  r  s     rn   r   z&avg_pool2d_backward.<locals>.<genexpr>  r  r  rO  c           	     @   t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  d   t        j                        }t        j                  t        j
                  | |      |      }t        j                  t        j
                  ||      |      }	t        j                  t        j                  ||      t        j                  t        j                  t        j                        |            }
t        j                  t        j                  |	|      t        j                  t        j                  t        j                        |            }t        j                  |t        j                  dt        j                              }t        j                  |	t        j                  dt        j                              }	t        j                  |
t        j                  t        j                              }
t        j                  |t        j                  t        j                              }t        j
                  t        j                  |
|      t        j                  ||	            }|S )z{
        This computes the scaling factor that we will divide an element
        by when `count_include_pad=False`
        r   r3   )
rY   rY  r   r  r  r  r  r   rX  r  )r  r  stride_hstride_wpad_hpad_wkernel_hkernel_wr;  wstartr<  wendrI  heightrI  r  r  r  s                rn   !compute_pool_size_without_paddingz>avg_pool2d_backward.<locals>.compute_pool_size_without_padding  s   
 <<q	5;;7<<q	5;;7WQZ5WQZ5<<A<<<A<X.6X.6{{GGFH%GGCNN65;;7?
 {{GGFH%GGCNN5%++6>
 VS\\!U[[%ABVS\\!U[[%AB{{4!DE{{4u{{!CDf 5swwtV7LMrp   c                8   | ^ }}}|d   z   }|d   z   }t        j                  t        |d   z
  d   z   d         t        j                        }t        j                  t        |d   z
  d   z   d         t        j                        }t        j                  t        |d         dz   t        j                        }t        j                  t        |d         dz   t        j                        }t        j
                  |t        j                  dt        j                              }t        j
                  |t        j                  dt        j                              }t        j                  |t        j                  t        j                              }t        j                  |t        j                  t        j                              }d }t              D ]+  }	t              D ]  }
t        j                  |t        j                  |	t        j                              }t        j                  |t        j                  |
t        j                              }}nssd   d   z  }n	 ||      }t        j                   g |t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d      t        j                  t        j                  |t        j                  |t        j                  dt        j                                    d            |      }t        j                  t        j                  ||      t        j                  ||            }|;t        j                  ||t        j                  dt        j                               }t        j                  |t        j                  ||      |      } . |J |S )Nr   r3   Frw  r  )rY   rX  r+   r   r  r  rY  r  r  r   r  rB  r  r  r  r  r  )r[  rC  r7  r  r  r  r  r  r  r  r  r  r  r  partr  rX  r0  r#  r  r  rD  rI  r  r  r  r  r  s                   rn   r   zavg_pool2d_backward.<locals>.fn  s2   A
N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM' *	RC]+ )RWWWcll3&DEWWWcll3&DE#/,E&k'N[^;E=b"EE{{#11 #$&s||Au{{7S(T!" !.&+  11 #$&s||Au{{7S(T!" !-&+& ). xxFF2u%FF2u% #"yytS\\#u}}5UVH"yyswwx/FQHS)R*	RV ###rp   rc  )rt   rG   r   r   r  rM  re  rr   r   r  r  fallback_avg_pool2d_backwardrD   r   rw   )r  r   rI  r  r  rJ  r0  r#  r  _h_out
ceil_mode1_w_out
ceil_mode2rj  r   rR  r   r  rX  r  r  rD  rW  r  r  r  r  s     ``` ``          @@@@@@@@@rn   avg_pool2d_backwardr`  Y  s    #'71'<X>XX<a&k9---a###{q   v;!w<1qzz|&&&

Q%;FJ &eQVWiXFJ))+K!*F
FjFJK&1&:&:&<#Q|AJJL!HKKME {1~)* M  {1~)* M
  -/KR+	
 		
 8? ?B 
		%%'	
B Irp   c                    dk7  sJ d       ssg dt        | t              sJ t        |t              sJ t              dk(  sJ t              dk(  sJ t              dk(  sJ t        |j                               dv sJ | j	                          |j                         ^ } t        d|      \  }	}
t        d|      \  }}t         d|      \  }}| j                         t              xs
 |
xs |xs || j                         ^ }t        |j                               }|j                         }fdt        d      D        \  z  z  }|d	kD  rt        | ||      S  fd
fd}t        j                  | j                         |||      }|S )Nr   rK  )r   r   r   r   )r   r   r3   r1   c              3  j   K   | ])  t        fd t           dz        D               + yw)c           
   3  t   K   | ]/  }t        |   z  t        d |   z
     z        z
  d       1 ywr  )r  )r   r0  r|   rI  r  s     rn   r   z0avg_pool3d_backward.<locals>.<genexpr>.<genexpr>2  sF      
 VAYQ[^);q	(I!JJAN
s   58r1   N)r  r  )r   r|   rI  r  s    @rn   r   z&avg_pool3d_backward.<locals>.<genexpr>1  s;      3
 	 	 
;q>A-.
 	
3s   /3}   c           	        d D        \  }}}d D        \  }}}d D        \  }	}
}d t        | ||g|||g|||g      D        \  }}}d t        |||g|	|
|gg|||g      D        \  }}}d |||fD        \  }}}d t        |||gg      D        \  }}}t        j                  t        j                  t        j                  ||      t        j                  ||            t        j                  ||            }|S )Nc              3  d   K   | ](  }t        j                  |t        j                         * y wr   rY   rY  r   r  r   r5  s     rn   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>H  s     'UQ(D'U   .0c              3  d   K   | ](  }t        j                  |t        j                         * y wr   rg  r  s     rn   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>I  s     Ms||Au{{;Mri  c              3  d   K   | ](  }t        j                  |t        j                         * y wr   rg  )r   r   s     rn   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>J  s$      (
-.CLLEKK((
ri  c              3  x   K   | ]2  \  }}}t        j                  t        j                  ||      |       4 y wr   )rY   r  r  )r   r  r5  pads       rn   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>N  s3      "
1c GGCGGAqM3'"
s   8:c           
   3     K   | ]i  \  }}}}t        j                  t        j                  ||      t        j                  t        j                  |t        j
                        |             k y wr   )rY   r  r   rX  r   r  )r   r|  r   r   rm  s        rn   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>T  sU      

 #q#s KKq!3773>>#u{{+KS#Q

s   A/A1c              3     K   | ]<  }t        j                  |t        j                  d t        j                               > ywr*  rY   r  rY  r   r  )r   r|  s     rn   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>_  s2      "
 KKs||Au{{;<"
   AAc              3     K   | ]?  \  }}t        j                  |t        j                  |t        j                               A y wr   rY   r  rX  r   r  )r   r}  r   s      rn   r   zQavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>c  s6      
S KKS^^C=>
   AA)r  rY   r  r  )pdr  r  stride_drO  rP  pad_drQ  rR  kernel_drS  rT  dstartr;  rU  dendr<  rV  rI  depthrW  rI  r  r  r  s                      rn   rX  z>avg_pool3d_backward.<locals>.compute_pool_size_without_paddingG  sH   'Uf'U$(HMWMue(
2=(
$(H"
 Rx8<ueU>S"


 '*(8X.&u%	'	

dD"
 &&1"

tT 2UFE4JK
dD GGCGGD&)3774+@A3774QWCX
 rp   c                .   | ^ }}}}d t        |||g      D        \  }}}d t        |||g!      D        \  }}}d t        |||g!      D        \  }}	}
d |||fD        \  }}}d t        ||	|
g g      D        \  }}	}
d }t              D ]{  }t              D ]i  }t        "      D ]W  }d t        |||g|||g      D        \  }}}}n ssd   d   z  d	   z  }n
 |||      }t        j                   g |t        j                  t        j
                  |t        j                  |t        j                  dt        j                                    d
      t        j                  t        j
                  |t        j                  |	t        j                  dt        j                                    d
      t        j                  t        j
                  |t        j                  |
t        j                  dt        j                                     d
            |      }t        j                  t        j                  t        j                  ||      t        j                  ||	            t        j                  ||
            }|;t        j                  ||t        j                  dt        j                              }-t        j                  |t        j                  ||      |      }Z l ~ |J |S )Nc              3  ,   K   | ]  \  }}||z     y wr   r   )r   r  rm  s      rn   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>n  s     Avq#1s7Arh  c              3     K   | ]<  \  }}}t        j                  t        ||z
  |z   |      t        j                         > y wr   rY   rX  r+   r   r  )r   r  r   r5  s       rn   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>p  s<      %
1a NN8AEAIq15;;?%
rq  c              3     K   | ]8  \  }}t        j                  t        ||      d z   t        j                         : ywrb  r  )r   r  r5  s      rn   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>u  s6      
1 NN8Aq>A-u{{;
s   >A c              3     K   | ]<  }t        j                  |t        j                  d t        j                               > ywr*  rp  )r   pstarts     rn   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>z  s2      %
 KKQ <=%
rq  c              3     K   | ]?  \  }}t        j                  |t        j                  |t        j                               A y wr   rs  )r   pend
pooled_dims      rn   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>~  s6      
 j KKcnnZEF
rt  c              3     K   | ]?  \  }}t        j                  |t        j                  |t        j                               A y wr   )rY   r   rY  r   r  )r   r  p_s      rn   r   z2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>  s6      "&FB R(EF"rt  r   r3   r1   Frw  r  )r  r  rY   r  rB  r  r  rY  r   r  r  r  r  r  r   )#r[  rC  r0  r7  r  pdstartr  r  pdendr  r  r  pd_r  r  ru  r  r  r  rZ  r  rX  r0  d_window_sizer#  r  r  rD  rI  r  pooled_depthr  r  r  r  s#                        rn   r   zavg_pool3d_backward.<locals>.fnl  s   AqAaAY)@A1a%
1ay+v>%
!'

Q1Iv.
ue
%
"GW5%
!'
$'u%m\'R%
ue ' 8	VC]+ 7V / 6VC"*-$gw7#sC+"JBB (3 0*+ +AQ ?+a. P A"b" M;;#!' # 5 5$'KK(*CGGE3<<5;;;W,X%& %1*/!" !$ 5 5$'KK(*CGGE3<<5;;;W,X%& %2*/!"  !$ 5 5$'KK(*CGGE3<<5;;;W,X%& %1*/!"!4 7D< 88E!2CFF2u4EFr5)D  '#&99 $S%--(H$ $'99T3778T3JH#Um6V7V8	Vr ###rp   rc  )rt   rG   r   r   r  rM  re  r$  rr   r   r  fallback_avg_pool3d_backwardrD   r   rw   )!r  r   rI  r  r  rJ  r0  r#  r  _d_outceil_mode_dr\  ceil_mode_hr^  ceil_mode_wr  rj  r   rR  r   r  rX  r  r{  r  r  rD  rW  r  r  r  r  r  s!     ``` ``             @@@@@@@@@@@@rn   avg_pool3d_backwardr    s    #'71'<X>XX<k9---a###{q   v;!w<1qzz|&&&$%JJL!VUFE&q+vw	FK ';FK 'q+vw	FK ))+Kg,K+KKK4?4H4H4J1Qm\AJJL!HKKME3
 q3/M=-  -/-?KS+	
 		
# #JU U Un 
		%%'	
B Irp   c                   | j                         }t        |t              r|g}n|st        t	        |            }t	        |      dk(  rt        |      dv s
J d|        g S t        |      }t        t	        |            D ]_  }||   dk  r#||xx   t	        |      rt	        |      ndz  cc<   d||   cxk  rt	        |      k  rFn t	        |      dk(  r	||   dk(  r_J  t	        t        |            t	        |      k(  sJ d       |S )Nr   )r   r  rz  zinvalid axis: r3   zreduction axis not unique)r   rt   r   r  r   r   rr   r)   )r   r  r   r|   s       rn   _validate_reduction_axisr    s    ::<D$vSY
4yA~T{//H>$1HH/	:D3t9 M7Q;GCIs4y14GDG'c$i'CINtAw!|LLM z$ CI-J/JJ-Krp   )r  c                  |t        | |      } | j                         t        t           t	        | |            }g }g g g t        t                    D ]Q  }||v r&j                  |       j                  |          -j                  |       |j                  |          S dt        |       xs- t        j                  |       dk(  xr t        j                  dk(  }|dv rt              dkD  r|rt        | j                  t              rdnt        | j                  t        j                         sRt        | j                  t        j"                        rat        | j                  j                  t        j$                        r3| j'                         }	|	j)                         xs |	j+                          fd}
r0t-              }D ]  }t.        j0                  j2                  ||<   ! n|}| j5                         t7        | j9                         |xs | j;                         | j;                         |
|      S )	NFr   cpp)rf  argminargmax_valueargmin_valueargmax_with_valueargmin_with_valuer3   Tc                |   t        |      t              k(  sJ 
r+t        |       t              k(  sJ D cg c]  }| |   	 } }t        |       t              k(  sJ d gt        |       t        |      z   z  }t        j                  t        |       t        |            D ]
  \  }}|||<     	|      }ru|D cg c]  }t	        j
                  |       }}|d   }t        dt        |            D ]  }||   z  ||   z   } |t        j                  |t        j                        fS |S c c}w c c}w r  )r   r  r  r  r   r3  r  rY   rX  r   r  )r@  reduction_indexr|   r`  r[  varr   rindex
linear_idxinner_loaderkeepdimskept_idxreduced_idxreduced_sizesshould_compute_logical_indexr   s            rn   rd  z%_make_reduction_inner.<locals>.loader  sL   ?#s;'7777u:T***'/0!U1X0E05zS]***Fc%j3+??@	!% #k?"C
 	!HC !IcN	! Y' (/>?!ell1o?F?  J1c&k* G'-*::VAYF
G 3>>*ekkBCC+ 1 @s   D4D9)r   rd  r  rb  rd  re  )r   r   r)   r   r  r  r   rx   r?   r6   get_device_typer4   cpu_backendrt   rv   rC   r   r   Bufferr  is_transposedis_contiguousrr   r   r/  r0  re  r  rw   r   )r   r  r  r   rQ  r  
kept_sizesr|    supports_logical_index_argreducer  rd  rj  r  r  r  r  r  r   s     `         @@@@@@rn   _make_reduction_innerr    s    Q::<Dc?3At<=DJHMK3t9 '9q!  a)OOAd1g&' $) '0| (
1&F6+=+=+F % 	
	
 ",affk*+/( 2 23qvvr}}-*QVV[["))2T\\^F$$&Df.B.B.D*D ) 6 : 	&A''++HQK	& ==?L||~'81;;=++-& rp   c                      dd dd fd}|S )Nr   c               V   dv rF| j                         t        j                  k(  r%t        |       rt	        | t        j
                        } t        | |||      }t        j                  d| d|}t        |j                  j                  t              r|j                          |S )N)rf  r  r  r  r   rQ  r  r  rc  r   )r   r   r   r?   r   r  r  rE   r   rt   rv   r  )r   r  r  r   r   r  rQ  r  s         rn   r  zmake_reduction.<locals>.inner>  s    
 22+!EKK(A&"7)
 !!XAXQWXKK
 NNrp   r   )r   rG   r   )r  rQ  r  s   `` rn   make_reductionr  ;  s    T  4 Lrp   c                     dd fd}|S )Nc                    t        | ||d | j                               }t        j                  j                  d| d|S )Nr  r  r   )r  r   r6   ArgReductionr   )r   r  r  r   r  s       rn   r  z,make_arg_with_value_reduction.<locals>.inner^  sP    &"#++-)
 %% 
)a
;A
 	
rp   r   )r   zSequence[TensorBox]r   )r  r  s   ` rn   make_arg_with_value_reductionr  [  s    
 Lrp   c                   |t        | |      } t        | |      }t        | j                         | j	                         f| j                         f| j                         |      S )N)r   dtypes	inner_fnsr   r  )r   r  r  rw   r   re  r   )r   r  r   s      rn   _make_scan_innerr  n  sZ    QD!D||~==?$ZZ\ rp   r   c                   |t        | |      } | j                         t        | |      }| j                         }|t        j
                  t        j                  fv rt        | t        j                        } t        | ||      }t        fd|D              }t        j                  || j                         | j                               }t        j                  |t        |j                                     }t        t!        ||      |      S )Nc              3  (   K   | ]	  }|     y wr   r   r   r|   r   s     rn   r   zmean.<locals>.<genexpr>       0a$q'0r   r?  )r   r   r  r   r   rN  rM  r;  sum_rW   r6   r=   rw   r<   r   rr   div)r   r  keepdimr   rE  
sum_resultdenomr   s          @rn   r  r  |  s    Q::<D#At,D;;=Lu~~66Q$aw'J0400Ee1;;=XEeT**=*=*?%@AEC
E*L99rp   c                  
 |d}| j                         
t        | |      }t        | |d      }|r|j                          t	        t        | |            }t        |||      }t        
fd|D              }|rt        ||z
  d      }t        j                  || j                         | j                               }t        j                  |t        |j                                     }t!        ||      }	|s|	fS |r|nt#        ||      }|	|fS )Nr3   T)r  c              3  (   K   | ]	  }|     y wr   r   r  s     rn   r   z var_mean_sum_.<locals>.<genexpr>  r  r   r   r?  )r   r  r  r  squarer  r  rW   r-   r6   r=   r   rw   r<   r   rr   r  r6  )r   r  
correctionr  return_meanx_meandiffsr  r  x_varr   s             @rn   var_mean_sum_r    s    
::<D#At,D!T4(F3q&>"EeT7+J0400EEJ&*e1;;=XEeT**=*=*?%@AE
E"ExVGFD$9F&=rp   c                @   t        | |      }t        | ||d d       }|d   }t        |d         }| j                         }d}d}	|t        j
                  t        j                  fv }
|r*|j                  dk(  rt        j                  j                  }ne|rS|j                  dk(  rDt        |       r9|
r7t        j                  j                  }	t        j                  j                  }d}nt        j                  }t        |t         j"                        syt%        |      }||kD  st        |      d	k(  ry|	rt        j                  |cxk  r|	k  ry |syt'        j(                  j*                  d|d
d|\  }}t,        j.                  j0                  j3                  |d	      S )Nr  r  r   rQ  rd  re  Fr   r   ry  Tr3   ri  )r)  r  r   )r  r  rW   rw   r   rM  rN  r   r4   r  use_two_step_variance_thresholdr?   tritonuse_two_step_variance_min_numelrn  rt   r   rj  r   r6   rE   
num_splitsrZ   rf   r   r  )r   r  r  input_dtyper   rd  r)  r   check_for_split	min_numelis_cuda_two_step_dtype	thresholdr  r  s                 rn   use_two_step_variancer    sv    $At,D"	wd$F HF#F+=$>?O\\^FOI(U^^U]],KK&++& JJ>>		FKK6)il?UMMAA	MMAA	66	ou}}5/*O"mF&;q&@V77/UIU V ||&& ' HAu
 7700::rp   c                  dt        | ||d d       }|j                  d      }|j                  d       |j                  d       t        j                  j                  d|fd| j                         d|\  }}}	|j                          | j                         | j                         t        | |      }t        fd|D              d	 fd
}
 t        |
      |      }|r|j                          ||fS |fS )Nr3   r  rb  rd  r  welford_reduce)r  r  r   c              3  (   K   | ]	  }|     y wr   r   r  s     rn   r   z$var_mean_welford_.<locals>.<genexpr>  s     1q471r   c                    t        | t        j                        rD| j                  s8t	        j
                  t	        j                  | t        j                        |      S t	        j                  | |      S r   )
rt   r   r   	is_numberrY   r   rX  r   r  rY  rA  s     rn   get_constant_or_index_exprz5var_mean_welford_.<locals>.get_constant_or_index_expr  sG    a$Q[[<<q%++ >FF||Au%%rp   c                           }       }t        j                  d      }| t        j                  |||z
        z  S r  )rY   rY  r  )rv   cNzeror  r   r  rnumels       rn   r  z#var_mean_welford_.<locals>.scale_fn  sE    &z59&vu5||Au%ckk$A...rp   r   )r  r  r6   WelfordReductionr   r   r  r   r  rW   r  )r   r  r  r  r  r   rd  r  m2r  r  r  r   r  r  r   s     `         @@@@rn   var_mean_welford_r    s    
"	wd$F ZZ
#F
JJ{
JJ{%%,, )'kkm 	KD"a JJLKKME::<D#At,D1D11F&
/ #.
"2
&CDy6Mrp   c               .   | j                         t              }t        | |d      } t        | ||||      }t        j
                  j                  st        | ||      rt        di |n
t        di |}t        fd|D              }|s|d   S |S )NFr  )r   r  r  r  r  )r  r  r  c              3  :   K   | ]  }t        |d         yw)Fr  N)r   )r   r   r  s     rn   r   z#var_mean_helper_.<locals>.<genexpr>  s     F!8Ayu55Fs   r   r   )r   r   r   r  r4   mtiadisable_welford_reductionr  r  r  r   )	r   r  r  r  r  r   r   r6  r  s	           @rn   var_mean_helper_r    s    I))4MM.A
F KK11$g9 	 ((  FvFFF'6!93V3rp   )r  r  c               "    t        | |||d      S )NFr  r  r  r  r  r   r  r  r  s       rn   var_r    s    	W% rp   c               "    t        | |||d      S )NTr  r  r  s       rn   var_meanr  $  s    	W$ rp   c                   |dk  r!t        t        j                  |       | |      S |dk(  rt        j                  d|      S |dk(  r| S t        | |dz  |      }t        j                  ||      }|dz  dk(  rt        j                  ||       }|S )Nr   r3   r1   )pow_recursiverY   r  rY  r  )r   r7  r   r  s       rn   r  r  +  s    1uS^^A.E::Av||Au%%Av1a1fe,FWWVV$F	A!|#Mrp   c                .    t        j                  | |      S r   )rY   r  r   r5  s     rn   
pow_nativer  :  s    771a=rp   )r	  c                z    t        t              r%j                         rt         t	                    S t        t              rdk(  rt               S t        t              rdk(  rt               S t        d  fD              }t        |      }t        t              xr dcxk  xr dk  nc xs	 |xr dk\  }|rZ j                          fd}t        j                   j                          j                         | j                               S t         t              r: dk(  rt!        d      S  d	k(  r$t#        j                               rt%              S |rDt         t              rt'               S t        t              rt)               S t+               S t-               S )
Nr  r3   c              3  r   K   | ]/  }t        |t        j                        s|j                          1 y wr   )rt   r6   rG   r   r!  s     rn   r   zpow.<locals>.<genexpr>R  s#     N1*Q2MNs   77i    r   c                F    t         |       j                               S r   )r  r   )r[  r   r5  rd  s    rn   r   zpow.<locals>.fn\  s     a??rp   rc  r1   )rt   r;  r   r  r   sqrtr  rL  r!   re  rD   r   rw   r   r   r"   r@  r    exp2fallback_pow_scalarfallback_pow_tensor_scalarfallback_pow_tensor_tensorr  )r   r5  r   is_integer_powembed_exponentr   rd  s   ``    @rn   r  r  H  sv   !U1c!f~	Au	!s(Aw	As	QQx NANNE%e,N  3' a"32AF  	@ <<>++-::<	
 	
 !V6Q?"6nQ[[]37Na &q!,,6"-a33-a33arp   c                   t        | t              r| j                  }n| }t        |t              r|j                  }t        |t        j                        st        j                  | j                         | j                         |j                         | j                               }t        |t        t        f      sJ |j                  }t        |t        j                        sJ t        |t        j                        rg|j                         sW|j                         sGt        |j                  t        j                        s#|j!                          |j                  |_        | S t        j"                  j%                  |||       | S )Nrc  unsafe_alias)rt   rG   rv   r6   r   rD   r   rw   r   re  r   r:   r@   is_input_bufferis_module_buffer	NopKernelr  r  realize_into)changedr;  r  changed_datarl   s        rn   r  r  y  s*   '9%||#y!hhc2==)%%'##%__&##%	
 $: 6777ii#r}}---,.$$&((*l''6 	HH!!..\ /  Nrp   c                .    t        | t        | |            S r   )r  r@  )r   r  s     rn   r  r    s    Q	!Z011rp   c                    | |u r| S t        || j                               }t        || j                               }t	        || j                               }t        | |      S r   r  rw   r   r   r3  r   r  )r  r  r  s      rn   r  r    sR    
cz

C)
*C
3
(C
clln
%CS#rp   c                .    t        j                  | |      S r   )rY   floordivr  s     rn   r  r        <<1rp   c                .    t        j                  | |      S r   )rY   truncdivr  s     rn   r  r    r  rp   c                .    t        j                  | |      S r   )rY   div_rnr  s     rn   _div_rnr    s    ::arp   c                >   t        |       xr t        |      }t        |       xr t        |      }|dk(  r,|rJ d       |rt        | |      S t        t	        | |            S |dk(  r,|rJ d       |rt        | |      S t        t        | |            S t        | |      S )NrI  z5floordiv operands can not be boolean at the same timerN  z5truncdiv operands can not be boolean at the same time)r   r   r  rI  r  r  rN  r  )r   r5  rounding_modeboth_integerboth_booleans        rn   div_moder    s    "1%</!*<L"1%</!*<L X!XX
 ".x1~G5A3GGX!XX!-x1~C5Q3CCq!9rp   c                    t        |       xr t        |      }|rt        | |      S t        t        j                  j
                        } t        |      | |      S r   )r   logical_andrB   r  r  r  r  )r   r5  	both_boolr   s       rn   r  r    sN    "9q'9I1a  **+!~b!!Q''rp   c                f   t        | t        j                        rt        | j                        S t        | t        j
                        rt        | j                               S t        | t        j                        r| S t        | t        j                        syt        j                  j                  j                  | j                               }t        j                  |      5  t!        j"                  t        j$                  dd      5   | j&                  | j)                          }ddd       ddd       t        t        j                  j*                  j,                        sJ t        |j.                  t        j                        r|j.                  S y# 1 sw Y   rxY w# 1 sw Y   vxY w)z:Try convert an arbitrary IR node into an ir.Constant valueNallow_indexingT)rt   r6   r@   get_constant_valuerv   r:   r  r   Loopsr   	_inductorops_handlerExtractConstantsHandlerrw   rZ   set_ops_handlerr   objectr  rb  inner_fn_argsvirtualizedOpsValuer   )r   r  rz   s      rn   r#  r#    s/   
 !R]]#!!&&))!R[[!!!--/22!R[[! a"oo))AA!,,.QG	'"-R&&(8$?- ajj!//+,	- - c5??66??@@@#))R[[)yy- - - -s$   &&F'F*F'F$	 F''F0c                l   t        d | |fD              }|rt        | |      S t        |      x}q| j                         j                  dk7  rT|j
                  dk(  r*t        j                  t        d      |j
                        }nd|j
                  z  }t        | |      S d } t        |      | |      S )Nc              3  L   K   | ]  }t        |      xs t        |        y wr   )r   r   r!  s     rn   r   zdiv_prim.<locals>.<genexpr>  s"     O1oa(>OA,>>Os   "$r   r   infr  c                 &    t        j                  |  S r   )rY   r  r  s    rn   r   zdiv_prim.<locals>.fn  s    {{D!!rp   )r#  r  r#  rw   r   r   mathcopysignr;  r  r  )r   r5  is_integralrH  r  r   s         rn   rB  rB    s    OAOOK1~ &a((5!,,.:M:MQV:V ==AuU|W]]CJw}},J1j!!" >"a##rp   c                Z    t        | |ft        j                        \  } }t        | |      S r  )rT  r   INT_TO_FLOATrB  r  s     rn   r  r    s0     	
A$C$P$PDAq Aq>rp   c                h    t        |       xs t        |       }|rd }nd } t        |      | |      S )Nc                .    t        j                  | |      S r   )rY   modr  s     rn   r   zfmod.<locals>.fn%  s    771a= rp   c                .    t        j                  | |      S r   )rY   fmodr  s     rn   r   zfmod.<locals>.fn*  s    88Aq>!rp   )r   r   r  )r   r5  r3  r   s       rn   r:  r:    s9    !!$:(:K	!
	" >"a##rp   c                   t        | j                               st        | j                               r|t        j                  }t        d|      } || |||      S )Nri  r  r   r!   r   r   r   r  r  r   r  r  r   r   s        rn   r  r  0  sK     	'+;AKKM+J
-	U	;Baxu--rp   c                   t        | j                               st        | j                               r|t        j                  }t        | j                               dk(  r(|dv sJ |xs | j                         }t        | |d      S d }t        | ||      }t        j                  j                  d	i |d|i\  }|t        | ||      S |S )
Nr   r   rz  Tr  c                @    | \  }|\  }t        j                  ||      fS r   )rY   r   a_tupleb_tupler   r5  s       rn   
combine_fnzcumsum.<locals>.combine_fnN  #    1rp   r  r   rD  r   r   r   )r!   r   r   r   r  r   r   r   r  r6   Scanr   fallback_cumsumr   r  r   rD  r   r  s         rn   cumsumrK  B  s     	'+;AKKM+J
-
1::<Aw&5t,, 
 ad%8F??J?IV~qd%88Mrp   c                   t        | j                               st        | j                               r|t        j                  }t        | j                               dk(  r(|dv sJ |xs | j                         }t        | |d      S d }t        | ||      }t        j                  j                  d	i |d|i\  }|t        | ||      S |S )
Nr   r?  Tr  c                @    | \  }|\  }t        j                  ||      fS r   )rY   r  rA  s       rn   rD  zcumprod.<locals>.combine_fnf  rE  rp   rF  rD  rG  r   )r!   r   r   r   r  r   r   r   r  r6   rH  r   fallback_cumprodrJ  s         rn   cumprodrO  Z  s     	'+;AKKM+J
-
1::<Aw&5t,, 
 ad%8F??J?IV~t599Mrp   c                
   d }| j                         }t        | j                               dk(  r|dv sJ t        |       S t	        | ||      }t        j                  j                  di |d|i\  }|t        | |      S |S )Nc           	     *   | \  }|\  }t        j                  ||      }t        j                  ||      }||k7  t        j                  |       z  }t        j                  |t        j
                  t        j                  ||z
              |z   |      fS r   )rY   r  r  r>  r  log1pexp)rB  rC  r   r5  min_vmax_vr  s          rn   log_add_exp_helperz(logcumsumexp.<locals>.log_add_exp_helpert  s{    Aq!Aq!CIIe$4#45		$		#''%%-*@ AE I1MOOrp   r   r?  rF  rD  r?  r   )	r   r   r   r  r  r6   rH  r   fallback_logcumsumexp)r   r   rV  r   r   r  s         rn   logcumsumexprX  r  s    P KKME
1::<Ag~~Qxac7FGG4FGIV~$QC00Mrp   c                   t        | j                               dk(  r,dv sJ t        |       t        | t        j
                        fS | j                         }t        j                  d|d      }t        | |      }|t        j
                  f|d<   | j                         fd	f|d
<   t        j                  j                  di |d|i\  }}|t        |       S ||fS )Nr   r?  r   rf  Fr   arg_break_ties_leftrF  r  c                R    t        j                  |    t        j                        S r   rY   rX  r   r  r[  r  s    rn   rI  zcummax.<locals>.<lambda>      CNN3t9ekk: rp   r  rD  r?  r   )r   r   r  r  r   r  r   r6   get_reduction_combine_fnr  re  rH  r   fallback_cummaxr   r  r   rD  r   r  r!  s    `     rn   cummaxrc        
1::<AwQxAU[[999KKME,,5J ad%8Fu{{+F8	:F; ggnnEvE*EOFG~qd++7?rp   c                   t        | j                               dk(  r,dv sJ t        |       t        | t        j
                        fS | j                         }t        j                  d|d      }t        | |      }|t        j
                  f|d<   | j                         fd	f|d
<   t        j                  j                  di |d|i\  }}|t        |       S ||fS )Nr   r?  r   r  FrZ  rF  r  c                R    t        j                  |    t        j                        S r   r]  r^  s    rn   rI  zcummin.<locals>.<lambda>  r_  rp   r  rD  r?  r   )r   r   r  r  r   r  r   r6   r`  r  re  rH  r   fallback_cumminrb  s    `     rn   cumminrh    rd  rp   c                   t        | j                               st        | j                               r|t        j                  }t        d|      } || |||      S )Nr  r  r   r<  r=  s        rn   r  r    sK     	'+;AKKM+J
-	e	<Baxu--rp   c                   t         j                  j                  }|yt               |j                  D ]  }|j
                  dk(  r|j                  t        j                  u rmt        |j                        dk\  rUt        |j                  d   t              r8t        |j                        dkD  s|j                  |j                  d           y t        fd| D              S )NTre   r1   r3   r   c              3  &   K   | ]  }|v  
 y wr   r   )r   r@  used_indicess     rn   r   z7_current_node_uses_all_tuple_outputs.<locals>.<genexpr>  s     :u$:r  )rZ   rf   rg   r)   rh   ri   rj   rP  getitemr   r}   rt   r   r   r#  )r!  rg   rm   rl  s      @rn   $_current_node_uses_all_tuple_outputsrn    s    77''L$.LL"" 
GG&x///DII!#499Q<-4::"  1.
 :':::rp   c                ^    t        | t        j                        }  t        d      | ||      S )Nr$  r  r  )r   r   r   r  r   r   r  s      rn   
reduce_anyrr    s(    EJJA >% w??rp   c                   |}t        |       rV| j                         t        j                  k7  r5t	        d      rt        | ||      S t        | ||      t        | ||      fS t        | ||      t        | ||      fS t        | d |      S N)r   r3   rp  )	r?   r   r   r   rn  reduce_argmax_with_valuereduce_argmax_valuereduce_argmaxreduce_amaxrq  s      rn   
reduce_maxry        
Q<AKKMUZZ73F;/gNN#AC'BacG< 
 g6!#8
 	

 qtg66rp   c                   |}t        |       rV| j                         t        j                  k7  r5t	        d      rt        | ||      S t        | ||      t        | ||      fS t        | ||      t        | ||      fS t        | d |      S rt  )	r?   r   r   r   rn  reduce_argmin_with_valuereduce_argmin_valuereduce_argminreduce_aminrq  s      rn   
reduce_minr    rz  rp   xor_sumr  r  rf  r  r  r  r  r  r  
logical_or)r  r  r  stabler   
descendingc          	        |d}| j                         }| j                         }t        t        |      |      }t        |      dk(  r't	        |       t        d|t        j                  |      fS t        |      r||   nd}t        j                  j                  rt        j                  }nt        j                  }t        j                  j                  j!                  t#        j$                  |t        j&                  |      j(                              st+        | |||      S t-        |dd||d      }dgt        |      z  }	t        |      r||	|<   t/        ||	      }t1        ||      }t2        j4                  j7                  || j8                  |j8                  f| j;                         |j;                         f||||      \  }
}|
t+        | |||      S |J |
t=        |t        j                        fS )NFr   r3   r  r  )r   r  r  r   r  r  r  )r   rw   r   r   r  r  r   r  r4   r  decompose_sort_opsr  int16rZ   rf   r   r1  r   r  r  r  sort_fallbackr  r  r3  r6   Sortr   r   re  r   )r   r  r   r  rY  r   r  	idx_dtyper!  
view_shaper  s              rn   sort_stabler    s   ~JJLE\\^F
3u:s
+C
5zQQxq&%++u=== ZuSzQH }}''KK	KK	77**5;;y1556 Qv3:NN6QVG s5z!J
5z"
37J'GWe$Gggnn'==?G$7$7$9: % OFG ~Qv3:NN8GU[[111rp   c                     t        | d||      S )NFr  )r  )r   r   r  s      rn   sortr  I  s    qCJGGrp   c                @   t         j                  j                  st        |       S | j	                         }t        j                  t        j                  |t        j                  d            }t        | |g      }t        |d      \  }}|dz
  dz  }t        |d|      S )Nr3   r   r?  r1   )r4   r  r  median_fallbackr   r  r'  rP  r  r   rj  r  r  r  )r  r   r  flatsorted_valsr  r   s          rn   median_defaultr  ]  s    ==++t$$==?DX\\4q1ABEugD 1-NK	qA+q!$$rp   c                   t         j                  j                  st        | ||      S | j	                         }t        |      }|dk(  r5t        |       t        d| j                         t        j                  |      fS t        ||      }t        | d|      \  }}||   }|dz
  dz  }t        |||      }	t        |||      }
|rt        |	|      }	t        |
|      }
|	|
fS )Nr   Tr  r   r3   r1   )r4   r  r  median_dim_fallbackr   r   r  r  rw   r   r  r   r  r  rX  )r  r   r  rY  r  r  sorted_idxsr  r   r  r!  s              rn   
median_dimr  i  s    ==++"4g66MMOEu:DqyT{E!T__%6UKKK
4
%C*4#FKc
A	
Q1AKa(F[#q)G63'GS)7?rp   c                p    t         j                  j                  st        | ||      S | j	                         }t        |      }| j                         }|dk(  r't        |       t        d|t        j                  |      fS t        ||      }t        | d|      \  }}||   }t        |ddt        j                  |d      }	t        j                  d      g|z  }
||
|<   t!        |	|
      }	t#        |	|      }	|	j%                         fd}t'        j(                  t+        |      t        j                  ||      }t-        |||      }|j%                          |j%                         |	j%                          fd	}t'        j(                  t+        |      t        j.                  ||      }|j%                         |	j%                         fd
}t'        j(                  t+        |      t        j                  ||      }t1        ||      \  }}|	j%                         |j%                         fd}t'        j(                  t+        |      t        j                  ||      }t3        ||d      }t-        |||      }t-        |||      }|st5        ||      }t5        ||      }||fS )z9Lower aten.mode via sort-based decomposition or fallback.r   Tr  r3   Fr  c           	         t        j                  t        j                   |       t        j                  dt        j
                              t        j                  dt        j
                              S r4  )rY   r  r  rY  r   r  )r[  positions_loader0s    rn   prev_pos_fnz!mode_default.<locals>.prev_pos_fn  sI    {{GG%c*CLLEKK,HILLEKK(
 	
rp   rc  c           
         t        j                  t        j                   |        |             t        j                   |       t        j                  dt
        j                                    S r  )rY   or_ner  rY  r   r  )r[  positions_loadershifted_loadersorted_loaders    rn   is_boundary_fnz$mode_default.<locals>.is_boundary_fn  sO    wwFF=%~c':;FF#C(#,,q%++*FG
 	
rp   c                    t        j                   |        |       t        j                  dt        j                              S rW  )rY   r  rY  r   r  )r[  is_boundary_loaderpositions_loader2s    rn   boundary_pos_fnz%mode_default.<locals>.boundary_pos_fn  s6    yys#c"LLU[[)
 	
rp   c                    t        j                  t        j                   |        |             t        j                  dt        j
                              S r  )rY   r   r  rY  r   r  )r[  last_boundary_loaderpositions_loader3s    rn   
run_len_fnz mode_default.<locals>.run_len_fn  s@    wwGG%c*,@,EFLLEKK(
 	
rp   rp  )r4   r  r  mode_fallbackr   r   rw   r  r  r   r  r   r  r  r   rj  r  r3  re  rD   r   rL   rG  r   rc  rw  r6  )!r  r   r  rY  r  r   r  r  r  	positionspos_view_shaper  prev_positionsshifted_valsr  is_boundaryr  boundary_poslast_boundaryr  r  run_lenmax_pos	mode_vals	mode_idxsr  r  r  r  r  r  r  r  s!                            @@@@@@@@rn   mode_defaultr  }  s    ==++T300MMOEu:D__FqyT{E!VU[[%@@@
4
%C*4#FKc
A 	%++fEI mmA&'$.NN3Y/Iy%(I "--/
 %%V$kk	N +sN;L++-M!--/N ,,.
 ""V$jj	K %002!--/
 ##V$kk 	L lC0M1 "--/(446
 V$kk	G G#=G{C1I{C1IIs+	Is+	irp   c                z   t         j                  j                  st        | ||||      S | j	                         }t        |      }|dk(  r5t        |       t        d| j                         t        j                  |      fS t        ||      }t        | d||      \  }}t        ||d|      }	t        ||d|      }
|	|
fS )Nr   Tr  )r4   r  r  topk_fallbackr   r   r  r  rw   r   r  r   r  r  )r  r   r   largestr9  rY  r  r  r  r  r!  s              rn   topkr    s    ==++T1c7F;;MMOEu:DqyT{E!T__%6UKKK
4
%C*Tsw K Ka+F[#q!,G7?rp   c                   t         j                  j                  st        | |||      S | j	                         }t        |      }|dk(  r5t        |       t        d| j                         t        j                  |      fS t        ||      }t        | d|      \  }}t        |||dz
        }t        |||dz
        }	|rt        ||      }t        |	|      }	||	fS )Nr   Tr  r3   )r4   r  r  kthvalue_fallbackr   r   r  r  rw   r   r  r   r  r  rX  )
r  r   r   r  rY  r  r  r  r  r!  s
             rn   kthvaluer    s    ==++ q#w77MMOEu:DqyT{E!T__%6UKKK
4
%C*4#FKKa!e,F[#q1u-G63'GS)7?rp   c                <    t        | |t        j                  |      S )Nr   r   r  )r  r   r5  )ri   r   r  s      rn   register_pointwise_numericr    s!    
;HH'	 rp   c                b    t        | j                         t        | t        j                        S r  )rV   r  r  r   r5  rN  s    rn    register_pointwise_numeric_ldf64r    s'    '4
;HH rp   )r  )r   c                 	
 t        | ||t        j                        | j                         |j                         |j                         	| j	                         }j
                  xr |duxr |j                  dv 
	
fd}t        j                  | j	                         || j                               S )az  
    Computes self + value * tensor1 * tensor2 using FMA for better precision.

    Matches eager CUDA kernel order: self + value * (tensor1 * tensor2)
    This is computed as: fma(value, tensor1 * tensor2, self)

    Note: FMA is only used for floating-point types on non-AMD GPUs. For integer types,
    we fall back to regular arithmetic since FMA doesn't support integers.

    For floating-point types, we use mul_rn (round-to-nearest multiplication)
    to force rounding of the product before the FMA. This prevents Triton's
    compiler from fusing the multiplication with the FMA, matching eager's
    rounding behavior.
    rV  Nry  xpuc                    |       } |       } 	|       }dk(  r
rt        j                  |||      S 
rt        j                  ||      }nt        j                  ||      }t	        t
        j                        rt        j                        }nt        j                        }
rt        j                  |||      S t        j                  |t        j                  ||            S r  )
rY   rZ  mul_rnr  rt   r   r   rX  rY  r   )r[  self_valt1_valt2_valt1_times_t2rX  r   r  	t1_loader	t2_loaderuse_fmar   s         rn   rb  zaddcmul.<locals>.inner_fnN  s    s#33A:'7766844  **VV4K''&&1K eU[[)u5JeU3J77:{H== 778SWWZ%EFFrp   rc  )
r   r   rK  re  rw   r  r   rD   r   r   r  tensor1tensor2r   r   rb  r   r  r  r  r  s      `  @@@@@rn   addcmulr  +  s      ;CC	E ""$K##%I##%I __F 	+$	+KK?* G G<  }}	 rp   c                 	
 t        | ||t        j                        | j                         |j                         |j                         	| j	                         }j
                  xr1 t        j                  j                   xr |duxr |j                  dv 
	
fd}t        j                  | j	                         || j                               S )a  
    Computes self + value * (tensor1 / tensor2) using FMA for better precision.

    Matches eager CUDA kernel order: self + value * (tensor1 / tensor2)
    This is computed as: fma(value, tensor1 / tensor2, self)

    For value=1: self + tensor1 / tensor2 (no FMA needed, just add the division)
    For value!=1: fma(value, div_rn(tensor1, tensor2), self)

    Note: FMA is only used for floating-point types on non-AMD GPUs. For integer types,
    we fall back to regular arithmetic since FMA doesn't support integers.

    We use div_rn (round-to-nearest division) to force proper rounding, preventing
    Triton from fusing operations in ways that change the rounding behavior.
    rV  Nr  c                    |       } |       } 	|       }
rt        j                  ||      }nt        j                  ||      }dk(  rt        j                  ||      S t	        t
        j                        rt        j                        }nt        j                        }
rt        j                  |||      S t        j                  |t        j                  ||            S r  )rY   r  r  r   rt   r   r   rX  rY  rZ  r  )r[  r  r  r  	t1_div_t2rX  r   r  r  r  r  r   s         rn   rb  zaddcdiv.<locals>.inner_fn  s    s#33 

662IFF3IA:778Y// eU[[)u5JeU3J77:y(;; 778SWWZ%CDDrp   rc  )r   r   r5  re  rw   r  r   r  r  r   rD   r   r   r  s      `  @@@@@rn   addcdivr  t  s    " ;HH	E ""$K##%I##%I __F 	+!!!	+$	+ KK?*	 E E:  }}	 rp   r   r  logical_not)r  )r   r
  rQ  identityc                L   | j                         t        j                  t        j                  t        j                  t        j
                  t        j                  fv rt        | dt        j                        S t        d      } t        |t        j                        |       S )NFr   signbitr  )
r   r   r   r0  uint16uint32uint64r@  rB   r  rA  s     rn   r  r    so    {{}

  E44	Y	B?>"EJJ?BBrp   )r  pointwise_overrides_datac              #  .  K   t         |   t        | j                  d       }|y fd}t        |t        j
                  j                        r-|j                         D ]  }|j                   ||      f  y |j                   ||      f y w)Nc                4    j                   t        |       S y r   )r  r  )ri   rv   s    rn   make_triton_fallbackz6_get_pointwise_overrides.<locals>.make_triton_fallback0   s    ;;#B'' rp   )	r  r  r   rt   r   r   r   r   r   )nsr   ri   r  olrv   s        @rn   _get_pointwise_overridesr  *   s     #D)D	TYY	%B	z( "ejj112//# 	IBd..0DR0HHH	I $**,@,DDDs   BBr  c                d    | t         |<   t        j                  |        fd}t        | |       y )Nc                      | i |}g }t        | d   |      D ]"  \  }}|j                  t        ||d             $ |S )Nr   Tr  )r  rx   r  )r}   r   resultsmut_resultsr   r  outplace_ops         rn   r   z$register_foreach_inplace.<locals>.fn|   sV    t.v.tAw0 	JKCyf4HI	J rp   )rc   r  r   r  )aten_opoutplace_aten_opr  r   s     ` rn   register_foreach_inplacer  x   s.    07,-G$ w+rp   c                2    t        | d       fd       }|S )NrV  c                 l     | i |}t        || d   j                               }t        | d   |      S r  )r   r   r  )r}   r   r  r  s      rn   r   zregister_inplace.<locals>.fn   s<    d-f-&$q'"3"3"56a&))rp   )r*  )r  r  r   s    ` rn   register_inplacer     s#    wD9* :*
 Irp   c                     y r   r   r  s      rn   sym_constrain_ranger         rp   c                    t         j                  j                  j                  d   }t	        |t
        j                        r|j                  j                  S t        |      S r  
rZ   rf   rg   ri  rt   r   r  rl   r  r   r   r   r;  s      rn   r  r     A    
''


#
#E
*C#u||$xx}}3xrp   c                    t         j                  j                  j                  d   }t	        |t
        j                        r|j                  j                  S t        |      S r  r  r  s      rn   
sym_strider     r  rp   c                "    | j                         S r   )rE  r   s    rn   	sym_numelr     s    ;;=rp   c                    t        | t              r| j                  j                  S t        | t        t
        t        f      rt        j                  |       S | S r   )	rt   r(   rl   r  r   r;  r   r   r  r   s    rn   _unwrap_symbolic_magic_argr 	     s?    !Xvv{{!c5$'(}}QHrp   )_funcc                   t         j                  j                  }||j                  j	                  d      nd }t        |t              r|j                  j                  S t        d t        j                  ||j                               D              r: | d |D        i |j                         D ci c]  \  }}|t        |       c}}S  | |i |S c c}}w )Nr;  c              3  \   K   | ]$  }t        |t        t        j                  f       & y wr   )rt   r(   r   r   r!  s     rn   r   zwrapped.<locals>.<genexpr>   s(      
 q8U[[12
s   *,c              3  2   K   | ]  }t        |        y wr   )r 	  r!  s     rn   r   zwrapped.<locals>.<genexpr>!  s     >A,Q/>s   )rZ   rf   rg   ri  rj  rt   r(   rl   r  r$  r  r  r  r  r 	  )r	  r}   r   rl   r  r   r  s          rn   r  r     s    ww##+/+;499=='h)==%%% 
__T6==?;
 
 >>@FO11033O  d%f%% Ps   2Cc                 ~    t        |       dk(  rt        | d   t        t        f      r| d   } t	        j
                  |  S r4  )r   rt   rr   r   r   Addr  s    rn   sym_sumr	  	!  s8     4yA~*T!WtUm<Aw99drp   c                    t        d      )NzHelpful for debuggingr   )r  r}   r   s      rn   foobarr		  !  s    
5
66rp   c                8    | j                          t        |       S r   )r  r  r   s    rn   _realizer	  !  s    IIK8Orp   c                R    | j                          t        j                  | |       | S r   )r  r6   ResizeStorageBytes)variablerj  s     rn   resize_storage_bytes_r	  !  s#    (H-Orp   c                    | j                          |j                          t        j                  t        j                  | |            S r   )r  rG   r   r6   SetSourceTensorKernel)r  source_tensors     rn   set__source_tensorr	  $!  s5    LLNB44T=IJJrp   r  c                    | |u r| S t        || j                               }t        || j                               }t	        || j                               }t        | |      S r   r  )r  r  s     rn   
fsdp_copy_r	  -!  sR    #:JS^^-.sCMMO,S#,,.)c""rp   c               8   t        | t              sJ t        |t        t        f      sJ |t        j
                  }|t        j                  k(  rt        d|       |t        j                  k(  rt        |      dk(  sJ |t        j                  k(  rt        |      dk(  sJ | j                         | j                         }| j                         }t	        j                         rit        j                  j                   j"                  rEt%        |      rt'        d      n0t)        |      r t	        j*                  |      j,                  ndndt.        j0                  j2                  j5                  d      rt7        |||      S | j9                         }|d uxr t;        d	 |D              }|rt=        | g      }nRt        | j>                  t@        jB                        r| j>                  jE                         | _        tG        | gd
g      }|jI                         t@        jJ                  jM                  ||      }tA        jN                  ||||      jQ                         fd}	tS        jT                  |||	t        |            }
|
S )Nzunsupported memory format: r   r   nanTr  r   r`  c              3  p   K   | ].  }t         j                  j                  j                  |d        0 ywr*  )rZ   rf   r   r  rh  s     rn   r   zresize.<locals>.<genexpr>]!  s,      2;<00A62s   46r3   c                    |       t        j                  t        j                        }t        j                  t        j                        }t        j                  ||      }t        j
                  |fd      S )Nc                       g      S r   r   )
flat_indexflat_loaders   rn   rI  z*resize.<locals>.inner_fn.<locals>.<lambda>q!  s    ZL(A rp   )rY   rX  r   r  r  r  )	r[  flat_index_exprlimitr  r	  r	  	old_numelout_indexeruninitialized_vals	       @rn   rb  zresize.<locals>.inner_fnl!  sZ     %
..U[[Ay%++6vvou-zz$ ACTUUrp   rc  )+rt   rG   rr   r   r   contiguous_formatpreserve_formatr  channels_lastr   channels_last_3drE  r   rO  rT  r  rS  fill_uninitialized_memoryr    r;  r!   r  r  rZ   rf   r   r  r=  r  r$  r  rv   r6   r:   r  r  re  r   stride_ordered_for_memory_formatr  r  rD   r   )r   r   r  r   r   rl  has_overlappingx_flat
out_striderb  rz   r	  r	  r 	  r!	  s              @@@@rn   resizer+	  8!  s0   a###dT5M***//---8HII+++4yA~~...4yA~~IKKME""$F 	224KK%%??%  %ee$ %E 2 6 6 $  ww//	1=D+5HH  "GT) c 2@G2 /O a)% affbkk*VV'')AFA	{QC0$$&K""CCD-XJ..jANNPKV 

UXd4jC Jrp   )auto_functionalizedc                    ddl m} |j                  |      }t        j                  | ||i ||       |j                         D ci c]  \  }}t        |t              s|| c}}S c c}}w )Nr   )kernel_side_table)
kernel_idxgridtma_descriptor_metadatakernel_args)*torch._higher_order_ops.triton_kernel_wrapr.	  get_constant_argsr6   UserDefinedTritonKernelr  rt   rG   )	r/	  constant_args_idxr0	  r1	  r   r.	  constant_argsr  r;  s	            rn   triton_kernel_wrap_r8	  !  sj     M%778IJM 7/v//	 &,\\^Rcz#y7QCHRRRs   A*!A*c                ^   t        d | g|D              rTd}t        j                  j                  j                  j                  dd       x}r| d| }|t        j                  _        t        j                  j                  | |||      }t        t        t        j                  |            S )Nc              3  V   K   | ]!  }t        |t              xr t        |       # y wr   rv  r!  s     rn   r   zcond.<locals>.<genexpr>!  s#     
Ma:a 1Yq\1
Mrx  z"control flow operator: torch.cond.r  r  )r$  rZ   rf   rg   ri  rj  r  r6   Conditionalr   rr   maprG   )predtrue_fnfalse_fnoperandsr   r  r  s          rn   r   r   !  s     
MD;L8;L
MM2''..3377tLL;LE*;-8C,/)^^""4(HEFI$$f-..rp   c                   t         j                  sit        d ||z   D              rTd}t        j                  j
                  j                  j                  dd       x}r| d| }|t        j                  _        t        j                  j                  | ||||      }t        |t              sJ t        t        t        j                  j                   |            S )Nc              3  V   K   | ]!  }t        |t              xr t        |       # y wr   rv  r!  s     rn   r   zwhile_loop.<locals>.<genexpr>!  s+      * 	1f.)A,.*rx  z(control flow operator: torch.while_loop.r  r  )r4   r  r$  rZ   rf   rg   ri  rj  r  r6   	WhileLoopr   rt   r   rr   r<	  _maybe_wrap_as_tensor_box)cond_fnbody_fncarried_inputsadditional_inputsstack_outputr   r  r  s           rn   
while_looprJ	  !  s     !!c *"33* ' 9''..3377tLL;LE*;-8C,/)\\  .*;\F fh'''BLL::FCDDrp   )rI	  c                    t        j                  j                  | g| }t        t	        t
        j                  |            S r   )r6   InvokeSubgraphr   rr   r<	  rG   )subgraph_fn
identifierr@	  r  s       rn   invoke_subgraphrO	  !  s5    %%k=H=FI$$f-..rp   c                l   t         }t        | j                  j                        D ]]  \  }}|j                  dk(  r?|t
        j                  j                  vsJ ||   t
        j                  j                  |<   U|j                  dk(  r]t
        j                  j                  |      \  }}t        j                  j                  j                  t
        j                  |||      }|t
        j                  j                  vsJ t
        j                  j                  }	 |t
        j                  _        t
        j                  j                  |      t
        j                  j                  |<   |t
        j                  _        ` |t         u rt        d      |S # |t
        j                  _        w xY w)a  Process nodes from a FX graph by executing them through V.graph.

    This is a common pattern for executing a subgraph's nodes:
    - Placeholder nodes are mapped to the provided args
    - Output nodes return their result
    - Other nodes are executed via V.graph.run_node

    r  r6  zNo output node found in graph)_MISSINGrs   rf   nodesri   rZ   envfetch_args_kwargs_from_envr   r^  Interpreterr6  rg   run_noder  )r  r}   r6  r|   rl   output_argsr   saved_current_nodes           rn   process_subgraph_nodesrY	  !  s@    F\//556 :477m#qww{{*** $QAGGKKWW "#''"D"DT"JKXX))00$VTFqww{{***!"!5!5:'+$$%GG$4$4T$:D!'9$!:$ :;;M (:$s    AFF3)control_depsc                   t         j                  j                  j                  d   }t	        |t
              sJ g }t        | |d      D ]  \  }}t        j                  |      D cg c]  }t	        |t              r| }}|D ]1  }	|	j                          |j                  |	j                                3 t	        |t        j                  j                        st         j                  j                   j#                  |g       }
|j%                  |
        t         j                  j                  j                  }d}t'        |      |z   t'        |      k(  sJ t'        t         j                  j(                        }t'        |j*                  j                  j-                  d            t'        |      k(  sJ t/        |j*                  t1        |            }| sJ t         j                  j(                  |d }|D cg c]\  }t	        |t2        j4                        r@|j6                  4t	        |j8                  t2        j:                        r|j                         ^ }}|r5|t         j                  j                   t         j                  j                  <   |D ]E  }|D ]>  }|j<                  }|J t         j                  j>                  |   jA                  |       @ G |D cg c]h  }t	        |t2        j4                        rL|j6                  @t	        |j8                  t2        j:                        rt	        |t2        jB                        s|j }}|r|rtE        d |j*                  j                  jF                  D              }|r|D ]  }t	        |t2        jH                        sJ |D ]  }t        j                  |      D ]l  }t	        |t              s|jK                         }|'t3        jB                  t3        j:                  |	      ||      }|jL                  j                  |       n   |S c c}w c c}w c c}w )
aS  
    Lower control_deps_op by ensuring dependencies are realized and tracking them.

    The control_deps_op HOP makes dependencies explicit in the graph. During lowering:
    1. Realize all additional dependencies to ensure they're computed
    2. Execute the target operation normally
    3. Track the dependencies for the scheduler
    r   T)strictr1   r  rN  Nc              3     K   | ]O  }|j                   d k(  xr: |j                  t        j                  j                  j
                  j                  u  Q yw)re   N)ri   rj   r   rY   streamswait_streamr  )r   r  s     rn   r   z+control_deps_op_lowering.<locals>.<genexpr>O"  sQ      
  DDO# BEII--99AAAB
s   AAr2  )'rZ   rf   rg   r}   rt   r   r  rk  rl  r>   r  rx   r  r   r^  r_  _void_ctrl_dep_op_namesrj  r  r   
operationsr  
find_nodesrY	  rr   r6   r  r   r  
NoneLayoutoperation_nameadditional_buffer_depsr   MutationOutputr$  rR	  r  rw   mutation_outputs)additional_depsrM	  r}   original_dep_nodes	dep_namesdep	orig_nodedep_leafdep_ir_nodesdep_ir_nodefoundoriginal_args
arg_offsetoperation_lenr6  new_opsri   
void_namesdep_nameop_namevoid_opshas_wait_streamvoid_opr   arg_leafr   mut_outs                              rn   control_deps_op_loweringr}	  !  s    --2215(%000Io/A$O $Y #..s3
(F+ 
 

 ( 	5K![1134	5 i/GG3377	2FEU#$ GG((--MJt9z!S%7777**+M{''--88M8JKsSWyXXX $K$<$<d4jIF?gg  0G" b"))$GGryy"--0	 	J  @J''(<(<=  B! 	BH''G&&&GG**7377A	BB  b"))$GGryy"--02r001 	H  D 
 !--3399
 

 # A!'2??;;; 
AC$*$6$6s$; 	A)(F;$!)!4!4!6!>$"$"3"3MM8(G#  0077@	A
AA MI
N>s   'P= A!Q(A-Q)schemec                  d }t         j                  j                  j                  j	                  dd       }|J t        | j                  j                  j                        D ]  \  }}|j                  dk(  r!||   t         j                  j                  |<   7|j                  dk(  rt         j                  j                  |      \  }}t        j                  ||j                               D ]  }	|	j                          |j                  r7t         j                  j                   j#                  |	j%                                t         j                  j&                  j#                  |	j%                                 t(        j*                  j,                  j/                  t         j                  |||      }Xt         j                  j1                  |      t         j                  j                  |<    |S )Nquant_optionsr  r6  )rZ   rf   rg   ri  rj  rs   r  rR	  ri   rS	  rT	  r  r  r  r  codegen_low_precisionlow_precision_codegen_opsr   r  invoke_quant_opsr   r^  rU	  r6  rV	  )
rM	  r~	  r@	  r6  r	  r|   rl   r}   r   r  s
             rn   invoke_quant_tracerr	  f"  sl   FGG((--11/4HM$$$[55;;AAB 7477m# (AGGKKWW 77==dCLD&__T6==?; E		 66GG5599!:N:N:PQ((,,Q-A-A-CDE XX))00$fMF ! 0 0 6AGGKK%7( Mrp   c                |   ddl m}m} dt        |      z  }| j                  j
                  j                  D cg c]  }|j                  dk(  s| }}||d  }t        |      t              k7  rt        d      g }	t        |      D ]C  \  }
}t        |
t        t        j                  f      r|j                  s3|	j                  |
       E |	rt        d      t!        j"                  ||      D cg c](  } ||j%                         |j'                               * }}|j)                          || |      fd}t+        |d	   d	d 
      }t-        d |D              |d<   t-        d |D              |d<   t/        j0                  j2                  d|dd|}|d	   t        d      |S c c}w c c}w )Nr3   )InputDescriptorlower_pointwise_subgraphr1   r  z/Unable to generate code for associative_scan opz_Unable to generate code for associative_scan op, because there are unsupported lifted argumentsr`  c                h     g t        j                  |       t        j                  |       S r   )rk  rl  )lhsrhsrH	  lowered_combine_fns     rn   wrapped_combine_fnz,associative_scan.<locals>.wrapped_combine_fn"  s@    ! 
$
$
 
 	
rp   r   rF  c              3  <   K   | ]  }|j                           y wr   r   r!  s     rn   r   z#associative_scan.<locals>.<genexpr>"  s     7qQ[[]7r$  r  c              3  <   K   | ]  }|j                           y wr   r  r!  s     rn   r   z#associative_scan.<locals>.<genexpr>"  s     <A<r$  r  F)rD  can_fallback_to_atenr   )r  r	  r	  r   r  rf   rR	  ri   r  r  rt   r   r   r   rh   rx   r  r  r   rw   r  r  r   r6   rH  r   )rD  xsrH	  r	  r	  num_scan_inputsrl   placeholdersadditional_placeholdersunsupported_inputsr   r  r   subgraph_inputsr	  r   r  r	  s     `              @rn   associative_scanr	  "  s   L#b'kO#0066<<=@XL  +?+;<
"#s+<'==LMM  13JK +[#U[[12k6G6G%%c*+ +
 	
 R( 	akkmALLNCO  ,-1*oN
 be!48F7B77F8<<<F;WW^^ %" F
 ayLMMMW&s   F4F46-F9c                     y r   r   )tokenss    rn   _sink_tokensr	  "  r  rp   c                      y r   r   r   rp   rn   _make_tokenr	  "  r  rp   c                L   ddl m}m}  ||      }||t        j                  j
                  j                  u rddlm}m	} |j                         }	|	r|	j                  j                  t        j                  j
                  j                        }
|
rMt        |
|      sJ |
j                  |d         }|r)t        |      dk(  sJ d       t!        t#        |            }t        t$        j&                  j(                        }|t*        v r+t+        |   |i |}t-        j.                  t0        d |       n:d }t-        j2                  |t5        j6                  j8                  |g|i |      }t        t$        j&                  j(                  |d       dkD  sJ d	| d
       |rt$        j&                  j:                  j=                  |      }t$        j&                  j(                  |d D ]W  }d |_        |s|jA                         }t$        j&                  jB                  |   jE                  |jA                                Y t$        j&                  j:                  |<   	 d }t-        j2                  |||f      \  }} ||||      }t        |jR                        dk(  r| |fS t        |jR                        dk(  r| |fS | g|S # tF        $ rQ}tI        |      }tJ        jM                  d||       t        |tN        tP        f      r| g|cY d}~S | |fcY d}~S d}~ww xY w)z
    We lower the operator directly, and then we add StarDep dependencies to all
    the newly created nodes in the graph.
    r   )_get_effect_get_schemaN)InvokeSubgraphCacheTracingContextr3   zMultiple effects NYIc                "    | j                         S r   )r  r  s    rn   rI  zwith_effects.<locals>.<lambda>"  s    !))+ rp   c                Z    t        | t        j                        r| j                         S | S r   r  r   s    rn   r  z"with_effects.<locals>.wrap_tensors"  r  rp   zCNo operation nodes were generated when lowering effectful operator .c                      yr   r   r   rp   rn   rI  zwith_effects.<locals>.<lambda>"  s    rp   c                <   t        | t        j                        r| j                         S t        | t              rK	 | j
                  }t        |d      r0t        |j
                  d      r|j
                  j                         S | S | S # t        t        f$ r Y | S w xY w)Nrv   get_example)
rt   r6   TorchBindObject	get_valuerG   rv   r   r	  AttributeErrorr   )r   r  s     rn   convert_ir_to_valuez)with_effects.<locals>.convert_ir_to_value"  s    !R//0{{}$Ay)ffGw/Gm5  '||7799 H	 '(;< s   AB BBz5Failed to get schema for %s: %s. Assuming list output)*torch._higher_order_ops.effectsr	  r	  r   rY   higher_orderrO	  torch._guardsr	  r	  try_gethop_dispatch_set_cache	get_cachert   get_effectsr   rL  r  rZ   rf   ra	  r`   rk  r  rG   r  r6   r  r   effectful_opsrj  has_side_effectsr  additional_star_depsr   r  rQ  r  r  r   rr   returns)tokenri   r}   r   r	  r	  effect_typer	  r	  tracing_ctxinvoke_subgraph_cacheeffectsrs	  r  r  prev_effect_buffernew_oprw	  r	  schema_argsschema_kwargsschemae	error_msgs                           rn   with_effectsr	  "  s    I b/KrUYY%;%;%K%KKE$,,.$/$F$F$P$P		&&66%! %!"79LMMM/;;DGDw<1,D.DD,"&tG}"5K **+M 
Y2//Y(=vF	L "++222GGG
 qww!!-.12Q6 
MbTQRS6 WW2266{Cgg((8 	YF&2F#! //+,,W599:L:U:U:WX	Y  	
k* #	$ &,__$&
"] Rm< 6>>av	V^^		!v  #F	CR	
 fudm,#F##6?"#s*   #(K	 		L#=LL#LL#L#)register_comm_loweringsregister_symm_mem_loweringsc                   t        | |ddd      }t        j                  j                  j	                  t        |d               }t        j                  j                  j                  |t        j                        rt        j                  d| dd|\  }}||fS t        | |d      }t        t        j                     t        | |            }t!        ||d      }||fS )	z
    Lowering inductor_prims.prepare_softmax_online to compute max/sum with
    online softmax reductions when profitable.
    TNr  re  r1   )rc  
num_output)r  r   )r  rZ   rf   r   simplifyrW   statically_known_geqr4   rn  rA   r   rx  r`   r  rS  r  r  )	r   r   r   r  
max_tensor
sum_tensorr  rS  xsums	            rn   prepare_softmax_onliner	  .#  s     #	d$dF WW&&}V<N5O'PQFww,,22 "8!>!> "
Q"
*0"

J :%%1cD1!#a,/Ct,Tzrp   c                   t               st        d      | j                         }|t        j                  t        j
                  t        j                  fvrt        d|       |t        j                  k7  rt        | t        j                        } t        j                  t        j                  ddt        j                  dd      } t        |      |       }t        |t        j                        S )z
    Lowering for cvt_e8m0_rceil. Uses PTX cvt.rp.satfinite.ue8m0x2.f32 on NVIDIA SM100+.

    The PTX instruction takes 2 float32 and outputs 2 e8m0 packed in uint16.
    Currently we pass 0.0 as the second input and only use the low byte result.
    zMcvt_e8m0_rceil requires NVIDIA SM100+ (Blackwell) for PTX instruction supportzAcvt_e8m0_rceil requires float32, float16, or bfloat16 input, got z)cvt.rp.satfinite.ue8m0x2.f32 $0, 0.0, $1;z=h,rTr3   )asmconstraintsr   is_purepack)rO   r   r   r   r  rN  rM  r  r   r  r)  rY   inline_asm_elementwiser  r  r0  )r   r   r   r  s       rn   cvt_e8m0_rceil_loweringr	  H#  s     $%![
 	
 MMOEU]]EMM5>>BBOPUwW
 	

 sEMM*			""7ll
B  ^B$FFEKK((rp   )r	  r	  c           	     <   	 t        | }t        d |D              |D cg c]  }|j                          c}	 	fd}t        j                  j                  |d   j                         |t        |d   j                                     S c c}w )Nc              3  <   K   | ]  }|j                           y wr   r	  rw  s     rn   r   z/lower_inline_asm_elementwise.<locals>.<genexpr>v#  s     ;S;r$  c           
          t         fdD              }t        j                  |	d}t        j                  t        j
                  fv r$t        j                  |t        j                        }|S )Nc              3  .   K   | ]  } |        y wr   r   )r   rd  r[  s     rn   r   zAlower_inline_asm_elementwise.<locals>.inner_fn.<locals>.<genexpr>z#  s     7VVC[7rY  )r	  r	  r   r	  r	  input_dtypes)r   rY   r	  r   rN  rM  r   r  )
r[  valsr  asm_strr	  r   r	  r	  r  r	  s
   `  rn   rb  z.lower_inline_asm_elementwise.<locals>.inner_fny#  sh    7w77++#%
 U]]ENN33\\&%--8Frp   r   rc  )	r  r   re  r6   rD   r   rw   rr   r   )
r	  r	  r   r	  r	  rP  r   rb  r	  r  s
   `````   @@rn   lower_inline_asm_elementwiser	  n#  s     'F;F;;L,23Ss 3G $ <<ay##%F1I&&()	   ) 4s   B)r  )quantized_lowerings)jagged_loweringsmkldnn_loweringsc              #  X  K   t        | t        j                  j                        sJ d       t        j                  |       }	  t        |       t        |              d |r
|t        | <   yt        j                  |        y# |r
|t        | <   w t        j                  |        w xY ww)z^
    A context manager to force fallback an op. Used in unit test
    for FallbackKernel.
    z+Only OpOverload to make the clean up easierN)	rt   r   r   r   r`   rj  r*  r  r  )ri   old_handlers     rn   force_fallbackr	  #  s      b%**//0 50 --#K".r23'IbMMM" 'IbMMM"s   AB*B ""B*#B''B*rh  )ry   zIterable[Any]r   z4defaultdict[tuple[Any, bool], list[tuple[int, Any]]])r   Callable[..., Any]r   Callable[..., Any] | None)r   ztorch._C.Tagr   z%Callable[..., tuple[Any, Any]] | None)r   r   r   rQ  r   None)r   z3torch._ops.OpOverloadPacket | torch._ops.OpOverloadr   zCallable[..., tuple[Any, Any]]r   r	  )r   zint | torch.dtyper   rR  )r   r	   r   z'TypeGuard[TensorBox | sympy.Expr | int])r   r	   r   zTypeGuard[TensorBox | bool])r}   r	   r   r   r   r   r   rR  )ri   z9Any | torch._ops.OpOverloadPacket | torch._ops.OpOverloadr   rQ  r   r   )r   rG   r   torch.devicer   rG   )r}   	list[Any]r   zdict[str, Any]r	  r   r   &ELEMENTWISE_TYPE_PROMOTION_KIND | Noner
  r   r   z tuple[list[Any], dict[str, Any]])r   rj  r  r	  r   r	  )
r  r	  r	  r   r   r	  r
  r   r&  r_   )r   r	  r   z.Callable[[Callable[_P, _T]], Callable[_P, _T]]rt  )rP  zSequence[_T]rQ  torch.dtype | Noner   r	  r   z&Sequence[_T | BaseView | BaseConstant])rl   ztorch.fx.Node | Noner   r   )NNNFFN)r   r	  rQ  r	  r  ztorch.device | Noner  r	  r  r   r  r   r  zCallable[..., _T] | Noner   zCallable[..., TensorBox | _T])Fr`  )r  Callable[..., foreach_node_t]r  r   r  rQ  r   z#Callable[..., list[foreach_node_t]])
r  z0Mapping[tuple[Any, bool], list[tuple[int, Any]]]r  r   r  r	  r  r   r   r  )FT)r   rG   r   rR  r  r   r~  r   )r   rG   r   rR  )r   rG   r   r	  )F)r   rG   r  rG   r   )r   rG   r(  Sequence[sympy.Expr]r   rG   r  )r  rG   r  rG   r  rG   r  r   r  r   r  r   r   rR  r   rG   )r  rG   r  rG   r  rG   r  r   r  r   r  r   r   rR  r  r	  r   rG   )r  rG   r  r;  r  r   r  r   r  r   r   rR  r   rG   )r  rG   r  r;  r  r   r  r   r  r   r   rR  r  r	  r   rG   )r  rG   r  rG   r  rG   r  r   r  r   r   rR  r   rG   )r  rG   r  rG   r  rG   r  r   r  r   r   rR  r  r	  r   rG   )rP  zlist[TensorBox]r   r   r   
str | None)r   r   r3   )r  r   rz  r   r{  r   r  )T)r   ztorch.Tensor)rl   ztorch.fx.Node)NTFN)r   r	  )
r   	list[int]r  rG   rQ  rQ  r  r   rE  rR  )
rZ  r   rY  r   r   r	  r  rG   r  r   )r^  rG   r   z.tuple[str, sympy.Expr, sympy.Expr, sympy.Expr])r^  rG   r   rG   )r^  rG   r   ztuple[str, sympy.Expr])r^  rG   r@  r	  rk  rR  r   r	   )r{  rG   r  rG   rp  r   rq  r   rr  r	  rs  TensorBox | Noner   rG   )r  rG   r  rG   rp  r   rq  r   )r3   )r   r   r@  r   )r   NNr3   )NNN)rz  FF)r   r   )r  rj  r   r   r'  r	  r  r   )r   r   r'  r	  )r   r   r  r   )r1   F)r  ztuple[float | None, ...]r  r   r  r   )r  float | None)r  r	  r  r	  )r  r	  r  r	  r  r	  )r   rG   r  zSequence[int]r  r;  r   r	  )r|   
sympy.ExprrZ  zsympy.Expr | int)r|   r	  rY  r	  )r|   r	  rY  r	  rZ  r	  )Nr  N)
rj  rG   rI  Sequence[int | torch.SymInt]rw  r	  rv  zlCallable[[Sequence[int | torch.SymInt], Sequence[int | torch.SymInt]], torch._inductor.virtualized.OpsValue]r   rG   )Nr   r3   F)r  )NNNN)r   r   FTN)r  r\   r   zCallable[..., TensorBox])r  r\   r   z"Callable[..., Sequence[TensorBox]]r   )r   rg  r   zir.Constant | None)r!  ztuple[int, ...]r   r   )rz  F)rz  TT)ri   ztorch._ops.OpOverloadPacket)r   z-list[ir.TensorBox | ir.ShapeAsConstantBuffer])rM	  ir.SubgraphrN	  rQ  )r  ztorch.fx.GraphModuler}   r	  )rM	  r	  )rD  r	  rH	  ztuple[Any, ...]ri  (b  
__future__r   r@  r3  r  r  loggingr1  rP  r  r  r  collectionsr   collections.abcr   r   r   r   typingr	   r
   r   r   r   typing_extensionsr   unittest.mockr   r   r   $torch.ao.quantization.fx._decomposedtorch.fxtorch.utils._pytreer  _pytreerk  torch._dynamo.utilsr   *torch._functorch._aot_autograd.descriptorsr   r   (torch._higher_order_ops.associative_scanr   r3	  r   "torch._library.fake_class_registryr   torch._library.opaque_objectr   torch._library.utilsr   torch._prims_commonr   r   r   r   r   r   r   r   r    r!   r"   torch.fx.experimental.sym_noder#   r$   rv  r%   r&   r'   r(   torch.utils._ordered_setr)   torch.utils._sympy.functionsr*   r+   r,   r-   r.   r/   r0   _dynamo.utilsr2    r4   r5   r6   r7   decompositionr8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   r+  rY   rZ   r[   r&  r\   r]   r^   FALLBACK_ALLOW_LIST	getLoggerr  r  r`   __annotations__ra   rb   r   r   r  r  tr_c10dr  r   _higher_order_opsr  rk   r  rc   quantized_decomposedrm  ro   r   r   r   r   r   r   r  r  r  r`  bmmconvolutionconvolution_backwardr  r  r  r
  r  r  _int_mmr0  r/  r  r  r  rN  r  r  	complex32	complex64r   rM  r   r   r   r   r   r   r   r   r  r  r'  rK  r*  r8  rT  rf  rr  r  r  r  r  r   r  r  r  r  r   r  r  
device_putr  r  r5  r  r  r  r  r  r  aliasdetachdetach_liftview_ofr*  r   r+  r6  r8  r<  r>  rD  rF  rI  rL  r  rN  r3  r\  r^  rm  _unsafe_viewreshaperp  r  slicer  r  r  r  quantize_per_channelr  r  r   r  _functional_assert_asyncr  dequantize_per_channelr  quantize_per_tensorr  dequantize_per_tensorr	  rF  r  r  r,  rL  r  r  r  r  r  r  r  r  rX  r  r  r  r  cacher  r  r  r  r  r  rngprimsr
  r  r  	bernoullir  r  r  	lru_cacher  r  r  r%  r"  r$  r*  r)  r(  randint	rand_like
randn_likerandint_likerrelu_with_noise_functionalr^	  record_event
wait_eventsynchronize_eventsynchronize_deviceforce_stride_orderr0  r  r3  r:  r6  lookup_seedr;  rD  randomrT  rand_eager_offsetrand_eager_offsetsr[  r_  rc  re  rn  r  r  rz  	NO_OPMATHr  r  r  r   rk  r  r   r   r  r  _adaptive_avg_pool3dadaptive_max_pool3d*_scaled_dot_product_attention_math_for_mpsuniformexponential_pdist_forwardsoft_margin_loss_backward_fused_rms_normr  _is_compiledembedding_dense_backwardr  native_layer_norm_cdist_forward_cdist_backward
_trilinearsegment_reduce_segment_reduce_backwardhistc	histogrambin_ct_histogramdd_bin_edges_histogramdd_from_bin_ctsaddbmm_addmm_activation_grouped_mm
_cudnn_rnn_cudnn_rnn_backward
miopen_rnnmiopen_rnn_backward_embedding_bag_embedding_bag_forward_only_embedding_bag_backward*_embedding_bag_per_sample_weights_backward_fused_moving_avg_obs_fq_helper*_fused_moving_avg_obs_fq_helper_functional max_pool3d_with_indices_backward_adaptive_avg_pool2d_backward_adaptive_avg_pool3d_backwardadaptive_max_pool2d_backwardadaptive_max_pool3d_backwardfractional_max_pool2d_backwardfractional_max_pool3d_backwardreplication_pad1d_backwardreplication_pad2d_backwardupsample_linear1d_backwardupsample_bicubic2d_backwardupsample_trilinear3d_backwardgrid_sampler_2d_backward_pdist_backward	nanmedianmultinomialr  resize_
resize_as__linalg_detlinalg_householder_productlinalg_inv_exlinalg_ldl_factor_exlinalg_ldl_solve	linalg_lulinalg_lu_factor_exlinalg_lu_solvelinalg_matrix_exp	linalg_qr_linalg_slogdet_linalg_solve_exlinalg_solve_triangular_linalg_svd	lu_unpackormqr_linalg_check_errorslinalg_pinvatol_rtol_tensor_linalg_eightriangular_solvelinalg_cholesky_excholesky_inversecholesky_solvegeqrf_fft_r2cnonzerobincount_unique2
unique_dimunique_consecutiveunique_dim_consecutivegcd_thnn_fused_lstm_cell_prims	rng_primsrun_and_save_rng_staterun_with_rng_stategraphsafe_run_with_rng_staterun_dtensor_rng_op_amp_update_scale__amp_update_scalemasked_scattermasked_scatter_backwardr  angle_efficientzerotensor(_sparse_coo_tensor_with_dims_and_tensors	to_sparse
_to_sparser  r  #_scaled_dot_product_flash_attention	quantized,_scaled_dot_product_flash_attention_backward#_scaled_dot_product_cudnn_attention,_scaled_dot_product_cudnn_attention_backward+_scaled_dot_product_flash_attention_for_cpu4_scaled_dot_product_flash_attention_for_cpu_backward0_scaled_dot_product_fused_attention_overrideable9_scaled_dot_product_fused_attention_overrideable_backward_flash_attention_forward_flash_attention_backward_efficient_attention_forward_efficient_attention_backwardindex_reducerepeat_interleave_weight_norm_interface_backwardr  r  r  r  r  r  arange
start_stepr  r  r  r  scalar_tensorr  
LongTensorr  r  r  r  r  r@  r  ri  r  r   r  r  
zeros_liker-  r1  r$  r7  r;  r=  rG  rP  r\  rn  rq  rp  r@  rz  r  r  r  r  r  r  r  r  fallback__unsafe_masked_indexr  ,fallback__unsafe_masked_index_put_accumulater  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r'  rM  r-  r.  r;  r@  rM  rS  rY  rk  ro  rz  r}  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r+  r3  
avg_pool1dr?  r.  r[  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  Tensor_Tensorr  Scalarr   Tensor_Scalarr  r  r  r  r  r  r  r  r  r  r#  rB  true_divider:  ri  r  rK  rI  rO  rN  rX  rW  rc  ra  rh  rg  r  rn  r$  rr  r  ry  r  r  r  r  rx  r  r  rf  rw  r  r~  rv  r}  ru  r|  r   r  r  r  r  r  r  r  r  medianr  r   r  rQ  r  r  r  r  r  r  rsqrtrS  r  expm1relur  r  r  r  r  r  _foreach_addcmul_foreach_addcmul_scalar_foreach_addcdiv_foreach_addcdiv_scalarcossinabsbitwise_andbitwise_left_shiftbitwise_not
bitwise_orbitwise_right_shiftbitwise_xorlgammaerfspecial_erfrR  tantanhr  r  r  logical_xorr  r  	clamp_min	clamp_maxnegr  	remaindersignr  	_neg_viewler  r  r  r  r  coshsinhacosacoshasinasinhatan2atanatanhr2  erfcerfinvhypotlog10log2	nextaftercodegen.commonr  r  r  r   ri   r   r  _foreach_addListforeach_add_listforeach_add_scalar_foreach_mulforeach_mul_listforeach_mul_scalar_foreach_sub_foreach_neg_foreach_abs_foreach_powScalarAndTensor_foreach_divforeach_div_listforeach_div_scalar_foreach_sqrt_foreach_rsqrt_foreach_maximum_foreach_minimum_foreach_clamp_min_foreach_clamp_max_foreach_reciprocal_foreach_sign_foreach_clone_foreach_copyforeach_copyr  _foreach_add__foreach_mul__foreach_div__foreach_copy__foreach_addcmul__foreach_addcdiv_r  add_bitwise_and_bitwise_left_shift_bitwise_not_bitwise_or_bitwise_right_shift_bitwise_xor_mul_div_Tensor_modelogical_and_logical_not_logical_or_logical_xor_sub_relu_sigmoid___and__
__lshift____or__
__rshift____xor____iand____ilshift____ior____irshift____ixor__r  r  r   r  r  r 	  r  methodfuncr  r	  r		  _inductor_testr  r	  r  r	  set_source_Tensorr	  fsdpr	  r+	  *torch._higher_order_ops.auto_functionalizer,	  r8	  r	  r   rJ	  while_loop_stack_outputr)  rO	  r)  rQ	  rY	  .torch._inductor.fx_passes.control_dependenciesrZ	  r}	  invoke_quantr	  r	  r	  r	  r	  comm_loweringr	  r	  r	  cvt_e8m0_rceilr	  r	  r	  r  r	  register_quantized_opsregister_woq_mm_opsr	  r	  register_jagged_opscontextmanagerr	  r   rp   rn   <module>r2     s?   "        	 
  # A A ? ? '    +  $ $ ( I U ? 8 :    M  0   - 8 8 =    $    "    * T]t_ !   g!@B	= BBD? DTV Q V-Juzz,,-/	yy~~
))

		9
5::#8#89; /j../
))*
 8j!6!679 NP K Pyy55 '* &90B	B*BF
83;3.3 
3   !!$$$$--&&, {{zz{{{{{{}}}}}}

."# "'CC8C C 	C(ANQ	(O
OO O @	O
  O &Od"/A42!2 2 @	2
  2 F2n ,44 4*#8 15BFCC-C @C ,	CL6B 15+/=A#04rr-r )r ";	r
 r r .r #rj )8
 9(99 9 )	9x#/<#/#/ ,#/ 	#/
 #/N UYEE$E,0EMQE2 5**77TR) S)X 5--4HN IN* @E 50 499??=& >& ;@e K 5##>F ?F 
7?? $,^ $7DD 4::4H I>>   	
 4::DI J0 4))UPTU V. DJJT\\499emmTU V 4&doo&s+ 4<<T:B ;B, 4$$$?" @" DMM?# $ 4::C C 4::C C 499! ! 4::! ! 4::%%&% '% 4::! ! 4;;D9> :>> 5))tD
 E
 4>>t<# =# 4;;4  4n 4$$$?499$74<<T:1 ; 8 @1 4<<T:> ;>&!R 4::481#++AT ~ 9~B 4??=
 	0J .20J >0Jf 4##> ? 4''TB C
BJ '<<RVW/// / 	/
 / / / / X/d$ 4%%))*$ +$ 400445$ 6$ //T %)/// / 	/
 / / / "/ //d ,,44$!!! ! 	!
 ! ! !!H ..66D %)!!! ! 	!
 ! ! "! !!H ,,33/// / 	/
 / / //d ..554 %),,, , 	,
 , , ", ,,^L^ 488v: v:r 4==d;5E <5Ep 4%%4@6 A6 4((dC D 4;;D9I :IX 4::48& 9&R 4((dC  D  4;;D9 : 4;;D9D :D8 4>>t< = 4??= > 488 $  #L
3%6T 	5Np	, 599%%11tL#+ M#+L 4&&DAG BG 4??= > 4>>##>' ?' 4<<  ! QB B/
 )):):; *499+>+>? )$***<*<= +DJJ,@,@A  dll  dnnd 3 doot 4 d 6 d.. / eii,,44 5 eii**22 3 eii1199 : eii22:: ; 499J J 4::J J >44$OL PL
 >&&DAK BK >''TBJ CJ
 >--4H	 I	( >((dC $}}5
5
5 5
 5 5 D5p n.. / n// 0 >))tDLM	(09FI E88.	8	8.	8=H	8	8 4$$++F
 #ZZ
Z 	Z
 Z Z Z Z GZz NN/N/X/X 555 	5
 55p D= !K
dP d'' ( d&& = d== > dll ' d&&U 3 d!!#5 6 d,,5 9 d"" /99%%E 	::U
 d!! " d"" # doo  d!!)) * d++33 4 djj  dnn## $ d))11 2 d,,44 5 dkk  d$$5 1 d . doo} - d&&(: ; doo} - d&&(: ; d!!#5 6 d..0B C d** + d== > d== > d22 3 d== > d33 4 d00- @ d00 1 d// 0 d// 0 d11 2 d11 2 d-- . d-- . d-- . d..0B C d00 1 d++ , d""$6 7 dnn  d&&U 3 dmm  dll  doo  d  d-- . d   ! d'' ( d## $ dnn  d&& ' d"" # d$$ % dnn  d"" # d## $ d** + d  dnn  djj  d'' ( d// 0 d   d## $ d%% & d## $ d!! " djj  dmm  dll"" # dmm##% 0 dmm##% 0 doo%%E 2 d%%--E : d))11 > dhhU + d((- 8 ell$$;; < ell$$77 8 ell$$AA B ell$$77 8
 d%%--E : d$$,,5 9
 d!! " d** + d""$6 7 djj  d'' ( d;; < dnn  doo  0088	
 99AA	
 ,,44	
 ,,66	 55==	
 ,,44	
 55==	
 44<<	
 ==EE	
 99AA	
 BBJJ	
 d++33_ E d++55 6 d,,44o F d//77 I d0088/ J d   d$$++T B d22::<N O
 499$7 8* 4::" 	 	3
* 4"#+d**+O< 5:: ( 4;;))tD 

  E. 4&&DA  B F 4%%4@A AAH ELL$"4"4e6I6IJKtDU < L<~ 5??#4 $4 5##$+ %+ 4++,)" -)"X 4&&' ( 4//0%)59$ 1<K> EKK,-  . .> 0t/0B50IJ
1!45	 21 56
  4>>" $T$4  # 4%%&Td '@ 4))*"4 +" 5%%--.A /A EJJ		*+: ,:
 4;;D9! :!H 4>>t< =B#F FR"1J 4::48
 9
 4%%4@/ A/ 4>>t< = 4))* +;. 4??= > >44$O Psl !1%%5!  0@,,44%0 ,
 4,,$G H, 4;;QUV	P W	P 1 1 4**E F 4<<T:9 ;9 & 
  D 4==d;AE : <:" 4##>3 ?3 4$$$?6 @6 4&&DAP BP 4''TBPT q Cqp , ', 	,
 ,^ 4**223> 4> 41199:J ;J 4**223LPI*I=II 4I 41199:LPU*U=IU ;U 4**223 "!!S S 	S
 S 4S 41199: "!!	 	 		
 	 ;	7 599$$% &*]](]6;]]@2)2)(2)6;2)2)j 4''TB8 C8v :>@ MQ < JN@:HV 5::PTU 5 V5D- -	 > 	11t$< 4//TJ 
 K
 4//TJ 
 K
 -=))11- ) 488dSN TNb8B&R2j  0%%5  
 4,,-B .BJ  0$$%  
 4++,N -Nb)X 4--.V /V 4--.V /VAH 433;;<BF. =.b 4??=  >* 4??=  >, T__,,%HT__,,%HT__,,%H A#H  0$$%  
 4++F b GbJ  0$$%  
 4++F D GDN& HLZ| :>!@!'& 499:t : : 4/;d(V42 DHHeii()T5  * 4==!u  "   .HH  'txxER -HH 
 488t,- --`"J 4::2 2 4::48 9       488t, -& DHH:.( /(@ EII;$/$ 0$. 	txx'7DD

 DIIuzz*d;$ <$  DHHeii(). . *. #4;;#6#67#DLL$8$89 ():):)B)BC "4;;#6#67"4;;#6#67 4;;  . 4<<  !. 4$$% &* 4;;D9 :, 4;;D9 :, 499. . .;* 488@ @
 48867 77" 48867 77" ! %--  	!: ;*		*>%+@A*		*>%+@A.!$++.85;;? /!$++.85;;? %^4 $^4 89LM 89LM HH ,	 !!1!1uM 499##>!re -2 ?-2` 499$$$?H @H !!2!2N$T]]%:%:PUV "4;;#6#6ER&t{{ER  !2!2N 4;;&&DA% B% 4;;??= >& 499$$$?j  @j Z 499$$$? @  4==((dC D$ 	#4::.&txx0!$)),"4::.$))$
*4<<
8'		2	DKK	(t4 4<<40-. E 1EP 4<<40-. F 1FR 5  't'  5  't' 
 ! *   *" !1!12'(?(?@  -  0
()A)AB  !1!12 4;; ' * *I*V*V 4:: & 488 $ 499 %   * **	 !**	  OO**	
 !**	 T\\
*
T\\
* ! $.. !' * ! $.. !' *""'8
 4>> "$))L 499  4<< 
C !
C " $.. !# & 477%** = 477%** = 477%** =uzzB 477%** = 477%** = 499 % 499 % 499 % 4:: & 499 % 4:: & 4:: & 499 % 4:: & 4== ) 499 % 4;; ' 4:: & 4:: & 499 % 4>> * D DE" % 
D4Ld5 
0 	 3+		

 5Mt5 
0 	 3+		


, .CT  0ct  4,,33Sd K-d.?.?.D.DcJ  4,,33S 9/0A0A0H0H#N  4,,113D I 4,,33Sd K 4,,44c : 4,,44c : 4,,33S 9 4,,113 7 4,,<<c B-d.?.?.D.DcJ  4,,33S 9/0A0A0H0H#N  4--t 4 4.. 6 40055w ? 40077 A 40055w ? 40077 A 42277 A 422997 C 42277 A 422997 C 433Z @ 4--t 4 4.. 6)$*<*<dC, T..335E t00779K T..335E t00779K T..335E t00779K !3!3!;!;\ !!  
 !!   C   ""K 0 ))+= > ""K 0 !!: . **,? @ ""K 0 C   !!3 ' && 1 ""K 0 ""K 0 !!: . ""K 0 C   T "  (   $,,  , " $// "#5 6  $++ z * " $// "#6 7  $,,  ,  - !!4?? 3 t{{ + !!4?? 3  - 4++, - 4==$$% & 4??&&' ( 4>>" # (M'') &LFD)&12! & 3&&& 5==! " 4<< 7 !7 599++334 5
 599%%;;< = 599>>&&445K 6K 599>>7#uyy~~++334# 5# 599>>(()%) = *=@ K J ! " 12S 3S( 599))..DI/2/ J/ 599))44$OE PE& 	II22)JT24
 599))99tT/ U/
 8 H H G <T:u ;up 5**77TRDH  S8 &DA/ B/d 599??//778 9 599??..667 8 599))66DQc  Rc L P O     >88dS T2 >00dK") L")J 	22 37QF     ! ! +  * * , '  ' ' )  %  $ $ &  rp   