
    ^j             -          d dl Z d dlmZmZ d dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlZd dlmZ d dlmZmZmZ d dlmZmZmZmZ d d	lmZ d d
lmZmZ d dlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z* d dl+m,Z,m-Z-m.Z.m/Z/m0Z0 d dl1m2Z2m3Z3 d dl4m5Z6 d dl7m8Z8m9Z9 d dl:m;Z<  e	d      Z= ed      Z>ej~                  j                  Z@ej                  j                  ddd      ZC eDd      \  ZEZFZGd ZHd ZIdeee>e=f   gee>e=f   f   fdZJde!fdZKd ZLd ZM eJe@j                  e@j                  g       e0       dddej                  ddfd               ZQ eJe@j                  j                  e@j                  j                  g       e0       d!               ZU eJe@j                  j                  e@j                  j                  g       e0       did"              ZW eJe@j                  j                  e@j                  j                  g       e0       	 	 djd#ed$eYd%eYd&ed'ed(ed)ed*eZd+eZd,e[d-ed.ed/ed0ed1ed2ed3ed4ed5ej$                  dz  d6eYdz  def*d7              Z\ eJe@j                  j                  e@j                  j                  g       e0       d8d9d:              Z] eJe@j                         e0       d;               Z^ eJe@j                  j                  e@j                  j                  e@j                  j                  e@j                  j                  g       e0d<d=      d>               Za eJe@j                  j                  e@j                  j                  g       e0       d?               Zbd@ ZcdkdAedBedeY   dCeZfdDZe eJe@j                  j                  e@j                  j                  g       e0       dE               ZgdZhdBedeY   fdFZi eJe@j                  j                  e@j                  j                  g       e0       dG               Zk eJe@j                  j                        ddHdI       Zn eJe@j                  j                        ej                  ddddJdK       Zp eJe@j                  j                  e@j                  j                  g       e0       ej                  ddddJdL              Zr eJe@j                  j                  e@j                  j                  g       e0       ej                  ddddJdM              Zu eJe@j                  j                  e@j                  j                  g       e0       dddddJdN              Zw eJe@j                  j                        dO        Zy eJe@j                  j                        dP        Z{dQ Z| eJe@j                  j                        dldR       Z~ eJe@j                  j                        dldS       Z eJe@j                  j                  e@j                  j                  g       e0       dAedBedeY   dTeYdUeYfdV              Z eJe@j                  j                        dkdW       ZdX Z eJe@j                  j                        dY        Z eJe@j                        	 	 	 dmdZed[ed\ed]edz  d^edz  d_ej                  dz  fd`       Z eJe@j                        	 didaedbedced_ej                  dz  fdd       Z eJe@j                        dededdfdZedaedbedced_ej                  dz  f
dg       Z eJe@j                         	 	 	 	 	 	 	 dndhej$                  diej$                  d]edz  djedz  d_ej                  dz  dkeZdleYdmeYdneYfdo       Z eJe@j$                  j                        dpdqdAedBeYdredsej$                  dtedueZdefdv       Z eJe@j(                  j                        dpdqdAedBeYdredsej$                  dtedueZdefdw       Z e0        eJe@j,                  j                        dx               Z eJe@j0                  j                        dddd dddydzedted{edz  d=edz  d|edz  d}eYd~eZdefd       Z eJe@j4                  j                  e@j4                  j6                  g       e0       d               Z eJe@j4                  j:                        dkd       Z eJe@j>                  j                  e@j>                  j6                  g       e0       d               Z eJe@j>                  j:                        dkd       Z eJe@jD                  j                        d        Z eJe@jD                  j                        d        Z eJe@jJ                  j                        d        Z eJe@jJ                  jN                        d        Z eJe@jR                  j                        d        Z eJe@jV                  j                        ddddddd       Z eJe@jZ                  j                        djd       Z eJe@j\                  j                        dmd       Z eJe@j`                  j                        djd       Z eJe@jb                  j                        d        Z eJe@jf                  jN                        d        ZdAedefdZdAededefdZ	 dodededeZfdZdpdededefdZdededeZdefdZ	 dqdededZedefdZdefdZ eJe@jx                  j                  e@jx                  jz                  g       e0dd      drdededeZfd              Z eJe@j~                  j                  e@j                  j                  g       e0       dZedefd              Z eJe@j                  g       e0dd      dZefd              Zd#edefdZ eJe@j                         e0       dAededeZdefd              Z eJe@j                         e0       dkdAededeZdefd              Z eJe@j                         e0       dkdAedeZdefd              Z eJe@j                         e0       dkdAedeZdefd              Z eJe@j                  j                        dsdedeZdeZfd       Z eJe@j                  j                  e@j                  j                  g       e0       dZededefd              Z eJe@j                  j                        dkdedeZfd       Z eJe@j                  j                  e@j                  j                  g       e0ddd      ddddAedeZdeZdeeeef   fd              Z eJe@j                  j                  e@j                  j                  g       e0       dddedededeZdef
d              Z eJe@j                  j                  e@j                  j                  g       e0ddd      dpddedeZdeeeef   fd              Z eJe@j                  j                  e@j                  j                  g       e0ddd      dpddÜdedeZdeZdeeeef   fdĄ              Z eJe@j                  j                  e@j                  j                  g       e0       dpddŜdedededeZdeZdefdǄ              Z eJe@j                         e0ddd      	 	 dtdededeZdeZdeeeef   f
dʄ              ZdedeeZeZf   fd̄Z eJe@j                  j                  e@j                  j                  g       e0ddΫ      dudededeeef   fdτ              Z eJe@j                  j                  e@j                  j                  g       e0dddd      dedeeeeef   fd҄              Z eJe@j                  j                        	 	 	 dvdedeZdeZdedz  fdք       ZdededeedeY   edeY   f   fdلZdedededz  deeef   fdڄZdZededeZfd܄Z eJe@j                        dpddddddݜdededeZdeZdedz  dedz  dedz  dedz  deeeeef   fdބ       Z eJe@j                  j                  e@j                  j                  g      dpdddߜdededeZdeZdeZdedz  defd       Z eJe@j                         e0dddp      	 	 	 dwdAededeZdeZdeZdeeef   fd              Z eJe@j                  j                        d        Z eJe@j                         e0       	 	 dxdZedededeZdeZdefd              Zd Zd Z eJe@j                         e0       d               Z eJe@j                         e0       d               Zd Z eJe@j                         e0d      d               Z eJe@j                         e0d      d               Zd Z eJe@j                         e0       d               Z eJe@j                         e0       d               Z eJe@j                  j                        d        Z eJe@j                  j                  e@j                  j                  e@j                   j                  e@j                   j                  g       e0d      d               Zd Z eJe@j                         e0       d               Z eJe@j
                         e0       d               Z eJe@j                  j                  e@j                  j                  e@j                  j                  e@j                  j                  g       e0d      d               Z	 eJe@j                         e0       dydAede[defd              Z eJe@j                         e0       dedAede[dedef
d              Z eJe@j                  j                  e@j                  j                  g       e0dp      deded d              Z eJe@j                   j                  e@j                   j                  g       e0       ddHd              Z eJe@j$                  j                        dzd       Z eJe@j                   j(                        dzd       Z eJe@j,                  j                  e@j,                  j                  g       e0       did              Z eJe@j0                  j                        	 	 dsd       Z eJe@j4                         e0dp      did_ej                  dz  fd              Zd Zd{d	Z	 did
ej$                  d[ej$                  dedeY   eYz  dedeY   eYz  dedeY   eYz  deZdeYdedeY   eYz  dz  fdZd Z eJe@j@                  j                        d
ej$                  d[ej$                  d]ej$                  dz  dej$                  dz  dej$                  dz  deZde[de[fd       Z! eJe@jD                  j                        d
ej$                  d[ej$                  d]ej$                  dz  dedeY   dedeY   dedeY   deZdedeY   deYfd       Z# eJe@jH                  j                        	 dod
ej$                  d[ej$                  d]ej$                  dz  dedeY   dedeY   dedeY   deZdedeY   deYdeZdeZdeZdeZfd       Z%ejL                  jN                  rej                  j                  d dd      Z( eJej~                  jR                  jT                  j                        d!        Z+ eJej~                  jR                  jX                  j                        d"        Z-ejL                  j\                  rOej                  j                  d#dd      Z/ eJej~                  j`                  jb                        d$        Z2ej                  j                  d%dd      Z3 eJej~                  jh                  jj                  j                         eJej~                  jh                  jl                  j                         eJej~                  jh                  jl                  jn                        d&                      Z8 eJej~                  jh                  jj                  jr                         eJej~                  jh                  jj                  jt                        d'               Z; eJej~                  jh                  jx                  j                         eJej~                  jh                  jx                  jn                        d(               Z= eJej~                  jh                  jx                  jr                         eJej~                  jh                  jx                  jt                        d)               Z> eJej~                  jh                  j~                  j                         eJej~                  jh                  j                  j                        d*               ZAej                  j                  d+dd      ZB eJej~                  j                  j                        	 	 	 	 d|d.       ZE eJej~                  j                  j                        d/        ZGd0 ZH eJe@j                        dZej$                  d1e[d2eYd3ej                  dej$                  f
d4       ZJ eJe@j                  j                        	 	 	 	 	 d}d5       ZLd6 ZM eJe@j                  j                        d7        ZO eJe@j                         e0       	 	 	 	 	 d}d8              ZQ eJe@j                         e0d      d9               ZS eJe@j                  j                        d:        ZU eJe@j                  j                        d;        ZW eJe@j                  j                        d<        ZY eJe@j                         e0d      d=               Z[d>edefd?Z\ eJe@j                         e0dd=      d@               Z^ eJe@j                         e0d      dA               Z` eJe@j                         e0dd=      dB               Zb eJe@j                         e0d      dC               Zd eJe@j                  j$                        didD       Zf eJe@j                  j                  e@j                  j                  g       e0       dE               Zh eJe@j                  j                  e@j                  j                  g       e0       d8dFdGeYfdH              Zi eJej~                  j                  j                  j                  ej~                  j                  j                  j                  g       e0       dI               Zj eJe@j                  j$                  e@j                  j$                  g      dJ        Zm eJe@j                  j                  g      dK        Zo eJe@j                  j                  e@j                  j                  g       e0dp      deded dL              Zq eJe@j                  j$                  g      dM        Zs eJe@j                  j                  e@j                  j                  g      dddNdO       Zv eJe@j                  j                  g      dddNdP       Zx eJe@j                  g       e0       dQ               Zz eJe@j                  g      dR        Z| eJe@j                  g      dS        Z~ eJe@j                  g      dT        Z eJe@j                  g      dU        Z eJe@j                  g      dV        ZdWeYdXeYdeYfdYZdZ Z eJe@j                  g      d]edz  fd[       Z eJe@j                  g      d\        Z eJe@j                  g      d]        Z eJe@j                  j                        d^        Z eJe@j                         e0       d_               Z eJe@j                   j                        	 	 	 	 	 	 d~d`       Z eJe@j$                  j                        da        ZdodbZ eJe@j*                  j                  e@j*                  j                  g       e0       dddcdd              Z eJe@j.                  j                  e@j0                  j                  g      de        Z eJe@j.                  j:                  e@j.                  j4                  e@j0                  j:                  e@j0                  j4                  e@j6                  j                  e@j6                  j8                  g       e0d<d=      ddf              Z eJe@j<                  j                        dg        Z eJe@j@                  j                        dh        Z eJe@jD                  j                        di        Z eJe@jH                  jJ                  e@jH                  j$                  e@jL                  j                  e@jN                  j                  e@jP                  j                  g      dj        Z eJe@jT                  jJ                  e@jT                  j$                  g      dk        Z eJe@jX                  jJ                  e@jZ                  jJ                  e@jX                  j$                  e@jZ                  j$                  g      d-dl       Z eJe@j^                  jJ                  e@j`                  jJ                  g      d-dm       Z eJe@jd                  j                  e@jd                  jf                  g      dn        Zdo Z eJe@jl                  j$                  e@jl                  jJ                  g      dp        Z eJe@jp                  j$                  e@jp                  jJ                  g      dq        Z eJe@jt                  j                        dr        Z eJe@jx                  j$                  e@jx                  jJ                  g      ds        Z eJe@j|                  j$                  e@j|                  jJ                  g      dt        Z eJe@j                  j                        du        Z eJe@j                  j$                         e0       d-defdv              Z eJe@j                  g       e0       	 ddw              Z eJe@j                  g      	 ddx       Z eJe@j                  g      	 ddy       Z eJe@j                  j                  e@j                  j                  g      dkdz       Z eJe@j                  jJ                        d{        Z eJe@j                  j                        d|        Z eJe@j                        d}        Z eJe@j                         e0       d~               Z eJe@j                        d        Z eJe@j                  j                        dkd       ZؐdjdZ eJe@j                  j                  e@j                  j                  g       e0dp      d               Z eJe@j                  j                  e@j                  j                  g       e0dp      d               Zݐd Zސd Zߐd Zd Z	 dkdZedeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdedeZf,dZd ZdZed>edeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdeYdef&dZd Z eJe@j                         e0d      d               Z eJe@j                  j                        	 	 	 	 d|d       Z eJe@j                  j                        d        Z eJe@j                         e0dd=      	 	 	 	 d|d              Z eJe@j                         e0d      d               ZdZedefdZ G d de      ZdZededeYfdZ eJe@j                  j                        d        Z eJe@j                         e0       d               Z eJe@j                         e0dd      d               Z eJe@j                  j                  g      d        Z eJe@j                  j                        	 	 	 	 	 dd       Z eJe@j                  j                  e@j                  j                  g       e0       ddddddd              Z eJe@j                  j                  e@j                  j                  g       e0       ddddddd              Z eJe@j                  j                        d        Z eJe@j                  j                        dd       Z  eJe@j                  j                        dd       Zd ZdodBeYdeYdeZfdZd Zd Z eJe@j                  j                        dkd       ZdkdZ	didZ
d ZdidZddZ eJe@j                  j                        dÄ        Z eJe@j                         dĄ        Z eJe@j$                  j&                  e@j$                  j(                  e@j$                  j*                  e@j$                  j,                  g       e0       didń              Z eJe@j0                  j&                  e@j0                  j(                  e@j0                  j*                  e@j0                  j,                  g      didƄ       Z eJe@j4                  j                  g      	 	 	 	 ddededede[deZdeZd1e[dz  fd̈́       Z eJe@j4                  j                  g      	 	 	 	 ddededededz  dedz  dedz  de[deZdeZd1e[dz  fdф       ZdedeeYdf   fdԄZ eJe@j<                  g      	 	 	 	 ddededededz  deZde[deZdeZd1e[dz  fdׄ       Z eJe@j@                  g      	 	 	 	 	 ddededededz  de[deZdeZd1e[dz  fd؄       Z! eJe@jD                        ddٜdededededededeZ   dededededeYdeYde[deZdeded1e[dz  f"d       Z# eJe@jH                  g      	 didededededededededeYdeYde[deZdeded1e[dz  fd       Z% eJe@jL                  g      	 	 	 	 ddededede[deZdedz  d1e[dz  fd       Z' eJe@jP                  g      	 	 djdedededededede[deZdedz  d1e[dz  fd       Z) eJe@jT                  g      	 	 	 	 	 	 ddededededz  de[deZdedz  d1e[dz  deZdeeef   fd       Z+ eJe@jX                  g      	 	 	 ddededededz  deZdeZd1e[dz  fd       Z- eJe@j\                  g      	 	 ddedededededz  dedededede[dedeZ   deZd1e[dz  fd       Z/ eJe@j`                  g      	 didedededededededededededeYdeYde[deZd1e[dz  f d       Z1 eJe@jd                  j                  g      	 	 	 	 	 	 	 ddededededz  dedz  deYdeYde[deZdeZd1e[dz  deYdz  deYdz  dedz  dedz  dedz  deYdz  f"d       Z3 eJe@jh                  j                  g      	 	 	 	 	 	 	 ddedededededz  dedz  deYdeYde[deZdeZd1e[dz  deYdz  deYdz  dedz  dedz  dedz  deYdz  f$d       Z5 eJe@jd                  j                  g      	 	 	 	 	 ddededededz  dedz  deYdeYde[deZdeZdedz  dedz  dedz  d1e[dz  deYdz  deYdz  dedz  dedz  f$d       Z6 eJe@jn                  g      	 	 	 dmdededededededededeYdeYde[deZdeded1e[dz  deYdz  deYdz  f"d       Z8 eJe@jr                  g      	 	 	 	 	 ddededed]edz  dedz  dedz  deYdz  deYdz  de[deYdeZd1e[dz  dedz  dedz  deYdz  fd        Z: eJe@jv                  g      	 	 	 ddedededed]edz  dedz  dedz  dejx                  dejx                  dede[dededeYdeZd1e[dz  deYdz  deZf$d       Z=	 	 	 	 ddAej$                  dcej$                  dej$                  dej$                  d]ej$                  dz  dej$                  dz  d_ej                  dz  deZfd	Z> eJe@j~                  j                  g      	 	 	 	 ddAej$                  dcej$                  dej$                  dej$                  d]ej$                  dz  dej$                  dz  d_ej                  dz  deZfd
       Z@	 	 	 	 	 ddAej$                  dcej$                  dedej$                     dede8   dedej$                     dede8   d]ej$                  dz  d_ej                  dz  dede9   dz  dede9   dz  deZfdZA eJe@j                  j                  g      	 	 	 	 ddAej$                  dcej$                  dedej$                     dede8   dede9   dedej$                     dede8   dede9   d]ej$                  dz  dej                  dz  dedeY   dz  deZfd       ZC eJe@j                  j                  e@j                  j                  g       e0       dod              ZG eJe@j                  j                        dod       ZI eJe@j                  j                  e@j                  j                  g       e0       dkddHd              ZKd ZLd ZM eJe@j                  j                  e@j                  j                  g      did       ZN eJe@j                  j                  e@j                  j                  g      djd       ZP eJe@j                  j                  e@j                  j                  g      	 	 djd>edeeYejx                  z     deeYejx                  z     de[dz  de[dz  f
d       ZR eJe@j                  j                  e@j                  j                  g      dmd       ZT eJe@j                  j                  e@j                  j                  e@j                  j8                  e@j                  j                  g      dd        ZYd! ZZ eJe@j                  j                        	 	 djd"       Z\ eJe@j                  j                        d#        Z] eJe@j                  j                        d$        Z^ eJe@j                  j                        d%        Z_d& Z`d' Za eJe@j                  j                  e@j                  j                  g      dd(       Zd eJe@j                  j                        dd)       Ze eJe@j                  j                        dd*       Zg eJe@j                         e0       	 dd+              Zi eJe@j                  j                  e@j                  j8                  g       e0d<d=      dd,              Zkej                  Zmd- Zn eJe@j                  j                        d.        Zo eJe@j                  j                        d/        Zp eJe@j                  j                        d0        Zr eJe@j                  j                        d1        Zs eJe@j                  j$                  e@j                  j                  g       e0       ddd2d3              Zv eJe@j                  jJ                  e@j                  j                  g      ddd2dAe'd4ed5eZd6eZfd7       Zx eJe@j                  g       e0       dd8              Zz eJe@j                  j                  e@j                  j                  e@j                  j                  g      	 	 djd9       Z~ eJe@j                  j                  e@j                   j                  g      	 	 djd:       Z eJe@j                  j                        d;        Z eJe@j                  j                  e@j                  j                  g       e0       dmd<              Z eJej~                  j                  j                        d=        Z eJej~                  j                  j
                        d>        Z eJe@j                         e0       ddddd?d@              ZdA Z eJe@j                        dB        Z eJe@j                        	 ddC       Z eJe@j                        	 ddD       Z eJe@j                        	 ddE       Z eJe@j"                         e0       dddFdG              Z eJe@j&                         e0       dHeYdAedefdI              Z eJe@j*                        dAefdJ       Z eJe@j.                         e0dp      dAedefdK              Z eJe@j0                         e0       dAedefdL              ZdM Z	 	 	 	 	 ddNedOedej$                  dz  dej$                  dz  dPedz  d]edz  dej$                  dz  d_ej                  dz  deZfdQZ eJe@j6                         e0       	 	 	 dmdNedOedPedz  d]edz  d_ej                  dz  defdR              Z eJe@j:                  g      	 	 	 	 	 ddNej$                  dOej$                  dej$                  dej$                  dPej$                  dz  d]ej$                  dz  dej$                  dz  d_ej                  dz  deZfdS       Z eJe@j>                  jJ                        ddT       Z eJe@jB                         e0       dUedBeYdVeZdefdW              Z eJe@jF                         e0       d,dX              Z eJe@jJ                         e0       	 	 	 dd[ed=edYeYdZeZd[eZdefd\              Z eJe@jL                  j                        	 dd<ed|ede   d]edeY   d^e[fd_       Zd` Z eJe@jR                  j                        	 	 	 	 dda       Zdb Z ee@jX                          ee@jZ                          ee@j\                          ee@j^                          ee@j`                          ee@jb                          ee@jd                          ee@jf                          ee@jh                          ee@jj                          ee@jl                          ee@jn                          ee@jp                          ee@jr                          ee@jt                          ee@jv                          ee@jx                          ee@jz                          ee@j|                          ee@j~                          ee@j                         dc Z eJe@j                         e0       dd               Z eJe@j                         e0       dededf              Z eJe@j                         e0       dededg              Z ee@j                        Z ee@j                        Z ee@j                        Zd dl1Zd dlZd dlZdh Z eʫ        y(      N)CallableSequence)Enum)wraps)TypeVar)	ParamSpec)SymBoolSymFloatTensor)_add_op_to_registry_convert_out_paramsglobal_decomposition_table
meta_table)
OpOverload)_prim_elementwise_meta$ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND)BoolLikecorresponding_complex_dtypecorresponding_real_dtypeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KIND	FloatLikeIntLikemake_channels_last_strides_formake_contiguous_strides_forNumber
NumberTypesuggest_memory_formatsym_min
TensorLike)_maybe_convert_to_dtype_maybe_resize_out_resize_output_check_safe_copy_outout_wrapper)_broadcast_shapes_maybe_broadcast)_config)ScalingTypeSwizzleType)_pytree_T_PatenIMPLMeta   c                     | |z   dz
  |z  S N    abs     d/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_meta_registrations.pyceil_divr:   :   s    EAI!    c                     | |z   dz
  |z  |z  S )z$Rounds up x to nearest multiple of yr4   r5   xys     r9   round_upr@   >   s    UQY1!!r;   returnc                       fd}|S )Nc                 V     t                 fd}t        j                  |        S )Nc                 (    t        t        |        y N)r   r   )opfns    r9   registerz0register_meta.<locals>.wrapper.<locals>.registerG   s    
B3r;   )r   pytree	tree_map_)rG   rH   rF   s   ` r9   wrapperzregister_meta.<locals>.wrapperD   s)     $	4 	2&	r;   r5   )rF   rK   s   ` r9   register_metarL   C   s     Nr;   type_promotionc                     t        j                  |d| i\  }}|D cg c]  }t        ||       }}t        | }t	        |dt
        j                  iS c c}w )Ntype_promotion_kindrM   )utilsr   r!   r'   r   r   DEFAULT)rM   args_result_dtyper>   s        r9   elementwise_metarU   P   ss    
 ..	*OA| ?CC#A|4CDC T"D "	BJJ  Ds   Ac                     t         j                  t         j                  t         j                  t         j                  t         j
                  t         j                  i}|j                  | |       S rE   )torch	complex32halfcfloatfloatcdoubledoubleget)dtypefrom_complexs     r9   toRealValueTypera   d   sE    ekku||L
 E5))r;   c                 l     t        t         g|       t        j                   k(   fd       y )Nc                      d d  S )Nzoutput with shape z# doesn't match the broadcast shape r5   )broadcasted_shape
self_shapes   r9   <lambda>z)check_inplace_broadcast.<locals>.<lambda>q   s    $ZL0STeSfg r;   )tupler&   rW   _check)re   
args_shaperd   s   ` @r9   check_inplace_broadcastrj   m   s0    /
HZHI	LLZ'gr;   Fc	                    	 t         t        j                        r(t        j                   j	                         dk(  d        t        t        j                        r(t        j                  j	                         dk(  d        t        d  fD              rZt        j                  t        j                               		nFt        j                  t        j                        	fd       nxs t        j                         t        t        j                        st        dt                     t        j                  t        t               fd       t        t              st        dt                     t        j                  dk\  d	        t        j                  f|d
||      S )Nr   c                       yNz:linspace only supports 0-dimensional start and end tensorsr5   r5   r;   r9   rf   z(meta_linspace_logspace.<locals>.<lambda>       r;   c                       yrm   r5   r5   r;   r9   rf   z(meta_linspace_logspace.<locals>.<lambda>   rn   r;   c              3   <   K   | ]  }t        |t                y wrE   )
isinstancecomplex).0args     r9   	<genexpr>z)meta_linspace_logspace.<locals>.<genexpr>   s     
C:c7#
Cs   c                      d  d S )Nzlinspace(): inferred dtype z& can't be safely cast to passed dtype r5   )default_complex_dtyper_   s   r9   rf   z(meta_linspace_logspace.<locals>.<lambda>   s    56K5LLrsxryz r;   zdtype must be torch.dtype, got c                      dt              j                   dt               j                   dt              j                   dS )Nz4received an invalid combination of arguments - got (, ))type__name__)endstartstepss   r9   rf   z(meta_linspace_logspace.<locals>.<lambda>   sD     u+r$s),,-RU0D0D/EQH r;   zsteps must be IntLike, got c                       y)Nz$number of steps must be non-negativer5   r5   r;   r9   rf   z(meta_linspace_logspace.<locals>.<lambda>   rn   r;   metar_   layoutdevice
pin_memoryrequires_grad)rq   rW   r   rh   dimanyrP   r   get_default_dtypeis_complex_dtyper_   AssertionErrorr{   _check_typer   empty)
r~   r}   r   baser_   r   r   r   r   rw   s
   ``` `    @r9   meta_linspace_logspacer   u   sn    %&IIK1P	
 #u||$GGINP	

 
CsE/B
CC % A A##%!
 =)ELL&&u-z
 2002eU[[)>tE{mLMM 
5'"	H
 eW%:4;-HII	LL!KL;;	# r;   c                    t        j                  j                  t         j                  k(  fd       t        j                  | j                         dk(  xr j                         dk7   d        | j                  j                        S )Nc                  "    d j                    S )Nz2take(): Expected a long tensor for index, but got r_   indexs   r9   rf   zmeta_take.<locals>.<lambda>   s    DU[[MR r;   r   c                       y)Nz*take(): tried to take from an empty tensorr5   r5   r;   r9   rf   zmeta_take.<locals>.<lambda>   rn   r;   )rW   rh   r_   long_check_indexnumel	new_emptyshape)selfr   s    `r9   	meta_taker      sm     
LLuzz!R
 
ZZ\Q55;;=A#56< >>%++&&r;   c                 ,    t        j                  |       S rE   rW   
empty_liker   	generators     r9   meta__standard_gammar           D!!r;   src	embed_dim	num_heads
qkv_weightqkv_biasproj_weight	proj_biasuse_gelu
norm_firstepsnorm_weight_1norm_bias_1norm_weight_2norm_bias_2ffn_weight_1
ffn_bias_1ffn_weight_2
ffn_bias_2mask	mask_typec                     | j                   rt        d      | j                         dk(  r| j                         S t	        j
                  |       S )NzR_transformer_encoder_layer_fwd fake implementation does not support nested tensorsr   )	is_nestedNotImplementedErrorr   clonerW   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   s                       r9   #meta__transformer_encoder_layer_fwdr      sH    : }}!`
 	
 yy{ayy{C  r;   r   c                T     j                   }j                   }t        j                  ||k(  d        t        j                   j                        dk(  xr j                        dk(   fd       t	         j
                  j
                        } j                  |      S )Nc                       y)Nz=linalg.cross: inputs must have the same number of dimensions.r5   r5   r;   r9   rf   zlinalg_cross.<locals>.<lambda>   rn   r;   r1   c                  V    d  dj                          dj                          S )Nzlinalg.cross: inputs dimension z must have length 3. Got  and size)r   otherr   s   r9   rf   zlinalg_cross.<locals>.<lambda>   s6    -cU 399S>"%

3'8: r;   )ndimrW   rh   r   r&   r   r   )r   r   r   x_dy_d	out_shapes   ```   r9   linalg_crossr      s     ))C
**C	LLs
O 
LL		#!4

31 4	
 "$**ekk:I>>)$$r;   c                 |    t        | d       t        | d       t        j                  | t        j                        S )Nzlinalg.matrix_expmemory_format)squareCheckInputscheckFloatingOrComplexrW   r   contiguous_formatr   s    r9   linalg_matrix_expr     s3     d/04!45D0G0GHHr;   valuesindicesc                 Z   t        j                  | j                  | j                  | j                        }t        j                  | j                  | j                  t         j
                        }| j                         dk7  r%| j                  dk7  rt        || j                         ||fS )Nr   r_   r   )	rW   r   r   r   r_   int64r   r   maybe_wrap_dim)r   r   r   r   s       r9   	cummaxminr   	  sp    
 [[DKKtzzJFkk$**T[[LGzz|qTYY!^sDII&7?r;   c                 x    t        || j                         t        j                  | t        j                        S Nr   )r   r   rW   r   r   )r   r   s     r9   logcumsumexpr     s+     3		"D0G0GHHr;   c                $   |j                   }t        |      }||z
  }t        t        |            }t        |      D 	cg c]  }	d }
}	|D ]  }d|
|<   	 |D cg c]
  }|
|   r	| }}|t        |      z   }t        |      }|j	                         |d | }|j                  fdd       |||d  z   }|j                  |      }dgt        |j                  |d        z   }|j                  |      }|j                  d      }||d<   t        |      }t        t        |            D ]  }|||      ||dz   <    | j                  |t        j                         t        |      D 	cg c]  }	d }}	d}|dz
  }|dk\  r0|| j	                  d      z  |||   <   ||||      z  }|dz  }|dk\  r0t        ||      D ]  }| j	                  d||z
  z         |||   <   ! | j                  ||| j                                | S c c}	w c c}w c c}	w )	NFTc                     |    S rE   r5   )r>   self_stridess    r9   rf   z_exec_fft.<locals>.<lambda>6  s    <? r;   keyreverser   r   r4   r   )r   lenlistrangestridesortpermuter   reshaper   resize_rW   r   as_strided_storage_offset)outr   	out_sizesr   forwardr   signal_ndim
batch_dimsdim_permuterS   is_transformed_dimdleft	batch_endtmpinputbatched_sizes
batch_sizebatched_out_sizesiout_stridesbatch_numelr   s                         @r9   	_exec_fftr   #  sG   99Dc(K#J uT{#K).t5A%55 % $1% #@!*<Q*?A@D@c"KD	I;;=L
jy
!CHH*DH9IJ//KLL%E D4JK 899MMM-(EAJ!M!]+3s8_ 5#,SV#4!a% 5KK!1H1HKI $Dk*1*K*KQA
q&&1CJJqM&AKN#yQ00	Q q& :t$ G&)jja*n1E&FKN#GOOI{C,>,>,@AJM 6
 A, +s   	H
H%H+	Hr   r   exclude_lastc                     t        |      }| j                         t        |      t        |      z
  }t	        |d | fdd      |d | |S )Nc                     |    S rE   r5   )r   r   s    r9   rf   z_sort_dims.<locals>.<lambda>Y  s    a r;   Tr   )r   r   r   intsorted)r   r   r   sorted_dimsr}   r   s        @r9   
_sort_dimsr  T  sS    s)K;;=L
k
S.
.CDS8$K r;   c                   
 t        j                  | j                  j                         |s| j	                         S t        |       dk(  rGt         j                  j                  j                         s| j                  | j                               S | j                         }| j                  |      }t        |       dk7  rt        | |      }t        || |||      S t        |      }| }	 |j                         
|j                  
fdd       t!        t"        t%        |            }|t%        |      |z
  d  }	t        ||||	|       |d t%        |      |z
   }|s|S || u r|}| j                  |      }n||}})Ncpucudar   Tc                     |    S rE   r5   r   stridess    r9   rf   zmeta_fft_c2c.<locals>.<lambda>v  s    wqz r;   r   )rW   rh   r_   
is_complexr   device_hintbackendsmklis_availabler   r   r  r   r   r   r   mincufft_max_ndimr   )r   r   normalizationr   r   outputr  working_tensormax_dims	last_dimsr  s             @r9   meta_fft_c2cr  `  s[    
LL&&'zz|4E!%..*<*<*I*I*K~~diik**		I^^I&F4F" s+y+wOO s)KN
 '')14@~s;'78K 08 ; =>	&.)YP!"?C$4x$?@MT!#N^^I.F%3VNF! r;   c                 f    t        |       t        kD  st        |       dk\  r| d   dk(  r	| d   dk(  ryy)N   r   r4   FT)r   r  r   s    r9   use_optimized_cufft_pathr    s3    
3x. SX]s1v{s1vQR{r;   c                 f   t        j                  | j                  j                         t	        | j                               }t	        |      }|d   }||   dz  dz   }t	        |      }|||<   |r|||<   t        |       dk(  st        |       dk(  rz| j                  |t        j                  | j                              }	| }
t        |       dk(  rt        |      rt        |	|
||d       nt        |      dk(  r|n|}t        |	|
||gd       t        |      dkD  r0| j                  |t        j                  | j                              }
|d d }|rx|
|	}
}	|
j                         |j                  fd	d
       t        t         t        |            }|t        |      |z
  d  }t        |	|
||d       |d t        |      |z
   }|rx|s:|	j                  |      ||   k7  r#|
j#                  |t         j$                         |
}	|	S t         j&                  j(                  j+                         rNt-        | |d      }| j                  |t        j                  | j                              }	t        |	| ||d      S | j                  |t        j                  | j                              S )Nr   r  r4   r	  xpur   Tr
  c                     |    S rE   r5   r  s    r9   rf   zmeta_fft_r2c.<locals>.<lambda>  s    '!* r;   r   r   )r   )rW   rh   r_   is_floating_pointr   r   r  r   rP   r   r  r   r   r   r   r  r  r   r   r  r  r  r  )r   r   r  onesidedinput_sizesr   last_dimlast_dim_halfsizeonesided_sizesr  r  target_sizesr  r  r  r  s                  @r9   meta_fft_r2cr(    s    
LL--.tyy{#K[!I2wH#H-2Q6+&N0N8/	(4F"k$&75&@ U>>tzzJ   
 t&+CC+HfnidK ),CA9>LfnlXJPTU3x!|!%U%F%Ftzz%R "0 "
 cr(K)7(//1  ,d !  ~s;/?@'K(88(C(EF	NNIt **GC,<x,GH  {{8$	((;;&&y@W@W&X'				(	(	* s>U>>tzzJ   
 y+tLL ~~U>>tzzJ  
 	
r;   )r   c                B    t        |t        j                  | g            S rE   )r"   rW   Size)nr   r   s      r9   meta_randpermr,    s    S%**aS/22r;   r_   r   r   r   c                6    t        j                  | ||||      S Nr-  rW   r   )r+  r_   r   r   r   s        r9   meta_randperm_defaultr1    s      ;;	vf r;   c                x     dt        j                   kD   fd       t        j                  |||||      S )Nr   c                      d d  S Nz:random_ expects 'from' to be less than 'to', but got from=z >= to=r5   highlows   r9   rf   zmeta_randint.<locals>.<lambda>      LSEQXY]X^_ r;   r-  rW   rh   r   )r6  r   r_   r   r   r   r7  s   `     @r9   meta_randintr:    s>     C	LLs
_ ;;E&J r;   c                t     t        j                   kD   fd       t        j                  |||||      S )Nc                      d d  S r4  r5   r5  s   r9   rf   z"meta_randint_low.<locals>.<lambda>  r8  r;   r-  r9  )r7  r6  r   r_   r   r   r   s   ``     r9   meta_randint_lowr=    s9     
LLs
_ ;;E&J r;   c                6    t        j                  | ||||      S r/  r0  )r   r_   r   r   r   s        r9   meta_rand_defaultr?    s      ;;E&J r;   c                 n    t        j                   j                         dk\  xr  j                  d   dk(   fd       t        j                   j                  t         j
                  k(   fd       t        j                  dkD  fd        j                  d d } j                  g|d      S )Nr4   r   r  c                  "    d j                    S )Nz>_philox_key_split: key must have shape (*batch, 2), got shape r   r   s   r9   rf   z'meta_philox_key_split.<locals>.<lambda>  s    PQTQZQZP[\ r;   c                  "    d j                    S )Nz3_philox_key_split: key must have dtype uint64, got r   rC  s   r9   rf   z'meta_philox_key_split.<locals>.<lambda>#  s    Ecii[Q r;   r   c                      d  S )Nz4_philox_key_split: num_splits must be positive, got r5   )
num_splitss   r9   rf   z'meta_philox_key_split.<locals>.<lambda>'  s    FzlS r;   )rW   rh   r   r   r_   uint64r   )r   rF  batch_sizess   `` r9   meta_philox_key_splitrI    s    	LL	Q-399R=A-\ 
LL		U\\!Q 
LLQS ))CR.K==*6{6A677r;   c                     t        j                   j                         dk\  xr  j                  d   dk(   fd       t        j                   j                  t         j
                  k(   fd       t        j                         S )Nr4   r   r  c                  "    d j                    S )Nz@_philox_key_fold_in: key must have shape (*batch, 2), got shape rB  rC  s   r9   rf   z)meta_philox_key_fold_in.<locals>.<lambda>1  s    RSVS\S\R]^ r;   c                  "    d j                    S )Nz5_philox_key_fold_in: key must have dtype uint64, got r   rC  s   r9   rf   z)meta_philox_key_fold_in.<locals>.<lambda>5  s    G		{S r;   )rW   rh   r   r   r_   rG  r   )r   datas   ` r9   meta_philox_key_fold_inrN  -  se    	LL	Q-399R=A-^ 
LL		U\\!S C  r;   c           	          t        j                  j                  j                   fd       t        j                  j                  t         j                  k(   fd       t        j                  j
                  j
                  k(   fd       t        j                  j                         dk\  xr j                  d   dk(   fd       j                         dkD  rt        j                  j                         j                         dz   k(   fd       j                  d j                          t        j                  t        d	 t        j                        D               fd
       y y )Nc                  &      dj                    S )Nz,: self must be a floating point tensor, got r   )op_namer   s   r9   rf   z1_check_philox_distribution_args.<locals>.<lambda>=  s    7)G

|T r;   c                  &     d j                    S )Nz": key must have dtype uint64, got r   r   rQ  s   r9   rf   z1_check_philox_distribution_args.<locals>.<lambda>A  s    7)=cii[I r;   c                  @     dj                    d j                    S )Nz/: self and key must be on the same device, got r   r   r   rQ  r   s   r9   rf   z1_check_philox_distribution_args.<locals>.<lambda>E  s*    i ;;-uSZZL2 r;   r4   r   r  c                  &     d j                    S )Nz5: key must have shape (2,) or (*batch, 2), got shape rB  rS  s   r9   rf   z1_check_philox_distribution_args.<locals>.<lambda>L  s    iLSYYKX r;   c                  @     d j                    dj                    S )Nz?: batched key must have ndim == output ndim + 1, got key shape z with output shape rB  rV  s   r9   rf   z1_check_philox_distribution_args.<locals>.<lambda>S  s,    ) !!$+>tzzlL r;   c              3   :   K   | ]  \  }}|d k(  xs ||k(    ywr4   Nr5   )rs   kssss      r9   ru   z2_check_philox_distribution_args.<locals>.<genexpr>Z  s$     MBa#28#Ms   c                  >     dt                dj                   S )Nz: key batch shape z( is not broadcastable with output shape r   r   )	key_batchrQ  r   s   r9   rf   z1_check_philox_distribution_args.<locals>.<lambda>[  s-    )-d9o-> ?::>**G r;   )
rW   rh   r_   r!  rG  r   r   r   allzip)rQ  r   r   r_  s   ```@r9   _check_philox_distribution_argsrb  :  s   	LL

$$T 
LL		U\\!I 
LLszz!	
 
LL	Q-399R=A-	
 wwy1}GGIa'	
 IIl
+	M#i2LMM	
 r;   c                      t        d| |       | S )N_philox_normal_rb  )r   r   meanstds       r9   meta_philox_normal_rh  b  s    #$5tSAKr;   c                      t        d| |       | S )N_philox_uniform_re  )r   r   r7  r6  s       r9   meta_philox_uniform_rk  h  s    #$6cBKr;   r  lastdimc                    t        j                  | j                  j                         t	        |       dk(  rt        | j                               }|||d   <   | j                  |t        | j                              }t        |      r.t        || j                  t         j                        ||d      S t        |      dkD  rt        | |d d dd      }n | j                  t         j                        }t        ||||d   gd      S t         j                  j                   j#                         r| }t        |      dkD  r|d d }t        | ||d      }|dd  }t        |j                               }|||d   <   | j                  |t        | j                              }	t        |	|||d      S t        | j                               }|||d   <   | j                  |t        | j                              S )	Nr	  r   r   r   Fr
  r4   r   )rW   rh   r_   r  r  r   r   r   ra   r  r   r   r   r   r  r  r  r  )
r   r   r  rl  r   r  tempr   c2c_dimsr   s
             r9   meta_fft_c2rrp  n  s    
LL&&'4F"%	$	#b'	1LM#C(

)@)@
A  3x!|#D#cr(AuEzz0G0GzHVT9s2wiOO				(	(	*s8a<3BxH xNEbc(C&	$	#b'nnYodjj.InJeYUCC %	$	#b'~~itzz/J~KKr;   c                    ddl m}  ||       s#t        j                  |       dk(  rt	        d      t        |t              r@|j                  | |      }t        j                  j                  || j                                | S )Nr   free_unbacked_symbolsr4   zQmore than one element of the written-to tensor refers to a single memory location)%torch.fx.experimental.symbolic_shapesrs  rW   _debug_has_internal_overlapRuntimeErrorrq   r   to_refsexpandr   )r   r   non_blockingrs  intermediates        r9   
meta_copy_r|    sr     L "$'E,M,Md,SWX,X_
 	
 #vvvdL1 	<5Kr;   c                     t        | j                               }t        | j                               }|| j                         k\  rdn
||   ||   z  }|j	                  |d       |j	                  ||       ||fS r3   )r   r   r   r   insert)tensorr   result_sizesresult_strides
new_strides        r9   inferUnsqueezeGeometryr    sq    &L&--/*NVZZ\)|C/@>RUCV/VJQ#z*''r;   c                     t        || j                         dz         }t        | |      \  }}| j                  ||       | S r3   )r   r   r  r   )r   r   g_sizes	g_stridess       r9   meta_unsqueeze_r    s>    
dhhj1n
-C/c:GYWi(Kr;   r   weight_metabias_activation_opt	out_dtypec                 (   t        | j                        }|R|j                  d      |j                  d      k7  r/t        d|j                  d       d|j                  d             |j                  d      | j                  d      dz  k7  r2t        d|j                  d       d| j                  d      dz         |j                  d      |d<   t	        | j                        dk7  r"t        d	t	        | j                         d
      d| j                  d      f}|K| j
                  t        j                  k(  r|t        j                  k(  st        d| j
                   d|       | j                  |||| j
                  n|      }|S )Nr   z%output size mismatch: weight.size(0)= != bias.size(0)=r4   r   r  zweight.size(1)=z != input.size(-1)/2=z0we can only handle the squashed input case, got D inputzKout_dtype is only supported for i8i8->i32 linear operator, got input.dtype=, out_dtype=r   )
r   r   r   r   r   r_   rW   int8int32new_empty_strided)	r   r  r  r  r  r  output_sizestransposed_stridesr  s	            r9   meta_sparse_structured_linearr    s    $L;;q>TYYq\) 7A7GGXY]YbYbcdYeXfg  {{1~B!++fkk!n--B5::b>TUCUBVW
 	
 {{1~L 5;;1>s5;;?O>PPWX
 	
 UZZ]+uzz)i5;;.F ]^c^i^i]jjv  xA  wB  C  $$&.ekkI % F Mr;   mat1	mat1_metamat2c                 2   t        | j                        dk7  r"t        dt        | j                         d      t        |j                        dk7  r"t        dt        |j                         d      t        |j                        dk7  r"t        dt        |j                         d      | j                  d      |j                  d      dz  k7  r2t        d| j                  d       d	|j                  d      dz         | j                  d      |j                  d      g}|K|j                  t
        j                  k(  r|t
        j                  k(  st        d
|j                   d|       |j                  |||j                  n|      }|S )Nr  mat1 must be 2D, got Dmat1_meta must be 2D, got mat2 must be 2D, got r4   r   mat1.size(1)= != mat2.size(0)/2=Jout_dtype is only supported for i8i8->i32 linear operator, got mat2.dtype=r  r   	r   r   r   r   r_   rW   r  r  r   )r  r  r  r  r  r  s         r9   meta_sparse_structured_mmr    sv    4::!4S_4EQGHH
9??q 9#ioo:N9OqQRR
4::!4S_4EQGHHyy|tyy|a''DIIaL>)<TYYq\A=M<NO
 	
 IIaL$))A,/L

ejj(Y%++-E \]a]g]g\hhtu~t  A  ^^%-djj9  F
 Mr;   r4   )alphabetar  c                J   t        | j                        dk7  r"t        dt        | j                         d      t        |j                        dk7  r"t        dt        |j                         d      t        |j                        dk7  r"t        dt        |j                         d      t        |j                        dk7  r"t        dt        |j                         d      | j                  d	      |j                  d	      k7  r/t        d
| j                  d	       d|j                  d	             |j                  d      |j                  d	      dz  k7  r2t        d|j                  d       d|j                  d	      dz         |j                  d	      |j                  d      g}|K|j                  t
        j                  k(  r|t
        j                  k(  st        d|j                   d|       |j                  |||j                  n|      }|S )Nr4   zKonly input broadcasted to columns of mat1 * mat2 product is supported, got r  r  r  r  r  r  r   zUonly input broadcasted to columns of mat1 * mat2 product is supported, input.size(0)=z != mat1.size(0)=r  r  r  r  r   r  )	r   r  r  r  r  r  r  r  r  s	            r9   meta_sparse_structured_addmmr    s    5;;1YZ]^c^i^iZjYkkrs
 	
 4::!4S_4EQGHH
9??q 9#ioo:N9OqQRR
4::!4S_4EQGHHzz!}		!$"ZZ]O+<TYYq\NL
 	
 yy|tyy|a''DIIaL>)<TYYq\A=M<NO
 	
 IIaL$))A,/L

ejj(Y%++-E \]a]g]g\hhtu~t  A  ^^%-djj9  F
 Mr;   compressed_Adense_Br  transpose_resultalg_idsplit_ksplit_k_modec	           	         |j                   t        j                  t        j                  t        j                  t        j
                  t        j                  hvrt        d|j                          | j                   |j                   k7  r%t        d| j                    d|j                          t        |j                        dk7  r"t        dt        |j                         d      | j                   t        j
                  t        j                  fv }	|j                  d      }
| j                  d      }|4||j                  d      k7  r t        d	| d
|j                  d             |z|	rO|t        j                  t        j                  t        j                  t        j                  t        j                  hv s)t        d| j                    d|j                    d| d      |r|
|fn||
f}|j                  ||      S )NzA_cslt_sparse_mm only supports fp16, bf16, int8, and fp8e4m3, got z%inputs must have the same dtype, got r   r  z-_cslt_sparse_mm only supports 2d inputs, got r  r4   r   zbias size mismatch: m=r  zout_dtype is not supported for  x z -> z matmul!r   )r_   rW   float32float16bfloat16r  float8_e4m3fnr   r   r   r   r  r   )r  r  r  r  r  r  r  r  r  is_8bit_input_typer+  moutput_shapes                r9   meta__cslt_sparse_mmr  I  s    }}

  OPWP]P]_
 	
 W]]*3L4F4F3GuW]]O\
 	
 7==Q;C<N;OqQ
 	
 &++

E<O<O/PPQA!A		! (+<TYYq\NK  ## !1,2D2D1ESW[\e[ffno  .Aq6Aq6L\;;r;   T)include_selfr   sourcereducer  c                L    t        j                  | t         j                        S r   rW   r   r   r   r   r   r  r  r  s         r9   meta_index_reducer    s     D0G0GHHr;   c                    | S rE   r5   r  s         r9   meta_index_reduce_r    s	     Kr;   c                     t        | j                               }| j                         dkD  r|j                         ||<   | j	                  |      S Nr   )r   r   r   r   r   )r   r   r   result_sizes       r9   meta_index_selectr    s@     tyy{#KxxzA~ ;;=C>>+&&r;   )lengthsr   offsetsaxisunsafeinitialrM  r  r  r  r  c                     |t        d       fd}| ||j                           S | ||j                     dz
        S t        d      )Nz?segment_reduce(): indices based reduction is not supported yet.c                     t        j                        }| |<   t        j                  |j                  dt        j
                        S )Nr   r_   r   r   )r   r   rW   r   r_   r   )segment_countr  r  rM  s     r9   segment_reduce_outputz2meta_segment_reduce.<locals>.segment_reduce_output  sB    DJJ'*T{{**11	
 	
r;   r4   z<segment_reduce(): Either lengths or offsets must be defined.)r   r   rv  )	rM  r  r  r   r  r  r  r  r  s	   `    `   r9   meta_segment_reducer    si     !M
 	

 $W]]4%899 $W]]4%81%<==
U
VVr;   c                 $    | j                  d      S Nr5   r   r   s    r9   meta_maxr         >>"r;   c                     t        j                  | j                  |f      }t        | ||      }| j	                  |      | j	                  |t
        j                        fS Nr   rP   reduction_dimsr   _compute_reduction_shaper   rW   r   r   r   keepdimr  s       r9   meta_max_dimr    R    


tzzC6
2C+D#w?L|$|5::6 r;   c                 $    | j                  d      S r  r  r   s    r9   meta_minr    r  r;   c                     t        j                  | j                  |f      }t        | ||      }| j	                  |      | j	                  |t
        j                        fS r  r  r  s       r9   meta_min_dimr    r  r;   c                     | j                         rt        | j                        }nt        | t        j
                        \  }}t        j                  | |      S NrO   r   )r  r   r_   r   r   INT_TO_FLOATrW   r   )r   rT   rS   s      r9   
meta_angler    sI    /

;, ? L L
< D55r;   c                     t        j                  || j                         | j                         |j	                  t        j
                  |             S rE   )rW   _resize_output_r   r   copy_angle)r   r   s     r9   meta_angle_outr    s6    	#tyy{DKK899U[[&''r;   c                      y rE   r5   vals    r9   assert_asyncr        
r;   c                      y rE   r5   )r  
assert_msgs     r9   assert_async_metar    r  r;   c                      y rE   r5   )ss    r9   
print_metar  	  r  r;   r_   r   r   r   r   c                 0    t        j                  dd      S )Nr5   r   rU  r0  r  s        r9   make_dep_tokenr    s     ;;r&))r;   c                 h    ddl m} t        | t        t        f      rt        d       || ||       y )Nr   )constrain_range'Constraining SymFloat or Symbool is nyir  max)rt  r  rq   r
   r	   
ValueError)r   r  r  r  s       r9   sym_constrain_ranger     s/     F$7+,BCCDcs+r;   c                 6    t         j                  | ||       |S Nr  )r.   r   r   r  r  	dep_tokens       r9   functional_sym_constrain_ranger  $  s    Ts4r;   c                 .   ddl m} ||t        j                  | dk\         y t	        | t
        t        f      rt        d      t        |       t        u r5|t        j                  | |k\         |t        j                  | |k         y  || ||       y )Nr   )_constrain_range_for_sizer  r  )
rt  r  rW   rh   rq   r
   r	   r  r{   r  )r   r  r  r  s       r9   sym_constrain_range_for_sizer  *  s     P
{s{TQY$7+,BCCDzS?LL%?LL%d5r;   c                 6    t         j                  | ||       |S r  )r.   r  r  s       r9   'functional_sym_constrain_range_for_sizer
  >  s    %%d%=r;   c                     |S rE   r5   )r  r  r  s      r9   functional_assert_async_metar  D  s    r;   f_namec                       j                         dk  rt         d j                                t        j                   j	                  d       j	                  d      k(   fd       y )Nr  z8: The input tensor must have at least 2 dimensions, got r   c                  V      dj                  d       dj                  d       dS )Nz5: A must be batches of square matrices, but they are r   by r   	 matricesr   )r  r   s   r9   rf   z#squareCheckInputs.<locals>.<lambda>R  s1    6( 		"d499R=/D r;   )r   r   rW   rh   r   )r   r  s   ``r9   r   r   J  s^    xxzA~hNtxxzl[
 	
 
LL		"2&	Dr;   Anamec                     t        j                   j                  j                  k(   fd       t        j                   j                  j                  k(   fd       t        j                  j	                  d      j	                  d      k(  fd       t        j                  j	                  d       j	                  d      k(   fd       y )Nc                  >    dj                    d j                    dS )Nz:Expected b and A to be on the same device, but found b on z
 and A on 	 instead.rU  r  r   s   r9   rf   z(linearSolveCheckInputs.<locals>.<lambda>]  s%    H{{m:ahhZy: r;   c                  >    dj                    d j                    dS )Nz=Expected b and A to have the same dtype, but found b of type z and A of type r  r   r  s   r9   rf   z(linearSolveCheckInputs.<locals>.<lambda>e  s%    Kzzl/!'')= r;   r   r  c                  R    d j                  d       d j                  d       dS )Nz3A must be batches of square matrices, but they are r  r  r   r  r   r  s   r9   rf   z(linearSolveCheckInputs.<locals>.<lambda>m  s0    FF2J<tAFF2J<yB r;   c                      d d j                  d       d j                  d       dj                  d       dj                  d       
S )NzIncompatible matrix sizes for z: each A matrix is r   r  z but each b matrix is r  r   )r  r  r   s   r9   rf   z(linearSolveCheckInputs.<locals>.<lambda>u  sR    ,TF 3D$TYYr]O4		"H r;   )rW   rh   r   r_   r   )r   r  r  s   ```r9   linearSolveCheckInputsr  Z  s    	LLqxx	
 
LL

agg	
 
LL	r
affRj 	
 
LL	r
diim#	
r;   tallow_low_precision_dtypesc                 J   | j                   t        j                  | j                         xs | j	                         fd       |sYt        j                  t        j
                  t        j                  t        j                  t        j                  fv fd       y y )Nc                       d  S )Nz<: Expected a floating point or complex tensor as input. Got r5   r_   r  s   r9   rf   z(checkFloatingOrComplex.<locals>.<lambda>  s    6(VW\V]^ r;   c                       d  S )Nz*: Low precision dtypes not supported. Got r5   r"  s   r9   rf   z(checkFloatingOrComplex.<locals>.<lambda>  s    vhHP r;   )	r_   rW   rh   r!  r  r[   r]   rZ   r\   )r  r  r  r_   s    ` @r9   r   r   ~  sn    
 GGE	LL	/^ &ekk5<<u}}MMP	
 &r;   arg_namec                 ^    t        j                  | j                         dk\  fd       y )Nr  c                       d  dS )Nz: The input tensor z! must have at least 2 dimensions.r5   )r$  r  s   r9   rf   zcheckIsMatrix.<locals>.<lambda>  s    6(-hZ7XY r;   )rW   rh   r   )r  r  r$  s    ``r9   checkIsMatrixr'    s    	LL	1Yr;   Br   c                      t                t               t        j                  r# j	                  d      j	                  d      k(  n" j	                  d      j	                  d      k(   fd       y )Nr  r   c                       drdnd d j                  d       d j                  d       dj                  d       dj                  d       d	S )
Nz2: Incompatible shapes of A and B for the equation zAX = BzXA = Bz (r  r>   r   r   rz   r   )r  r(  r  r   s   r9   rf   z#checkInputsSolver.<locals>.<lambda>  s[    hHxX.AaffRj\qvvbzl!AFF2J<qJ r;   )r   r'  rW   rh   r   )r  r(  r   r  s   ````r9   checkInputsSolverr+    sY    a !V	LL$(r
affRj affRjAFF2J.F	
r;   resultfn_nameresult_namec                 r     t        j                  j                  j                  k(   fd       y )Nc            	      L      d d dj                    dj                    	S )Nz: Expected z5 and input tensors to be on the same device, but got z on z and input on rU  )r-  r   r,  r.  s   r9   rf   z!checkSameDevice.<locals>.<lambda>  s5    i{;-/dm4nU\\NL r;   )rW   rh   r   )r-  r,  r   r.  s   ````r9   checkSameDevicer1    s&     
LL%	
r;   UPLOc                       j                         }t        j                  t               dk(  xr |dk(  xs |dk(   fd       y )Nr4   ULc                      d  S )Nz1Expected UPLO argument to be 'L' or 'U', but got r5   )r2  s   r9   rf   zcheckUplo.<locals>.<lambda>  s    CD6J r;   )upperrW   rh   r   )r2  UPLO_uppercases   ` r9   	checkUplor9    s<    ZZ\N	LLD	QKNc1J^s5JJr;   eigenvalueseigenvectorsr5  	compute_vc                 T   t        | d       t        |       t        | j                        }|r/| j	                  |      }|j                  |t        |d             n| j	                  dg      }|j                          | j	                  |t        | j                              }||fS )Nzlinalg.eighF	row_majorr   r   )
r   r9  r   r   r   r   r   popra   r_   )r  r2  r<  r   vecsvalss         r9   meta__linalg_eighrC    s     a'dOME{{5! ;EU ST{{A3	IIK;;uOAGG$<;=D:r;   c                     t        | d       t        j                  | j                        r| j                  nt        j                  | j                        }| j                  | j                  d d |      S )Nzlinalg.eigvalsr   r   )r   rP   r   r_   r   r   r   )r   complex_dtypes     r9   meta__linalg_eigvalsrF    sc     e-. !!%++. 	..u{{; 
 ??5;;s+=?AAr;   c                    t        | d       t        j                  | j                        r| j                  nt        j                  | j                        }| j                  | j                  d d |      }| j                  | j                  |      }t        |       dk(  }|j                  | j                  t        | j                  |             ||fS )Nz
linalg.eigr   r   r	  r>  )
r   rP   r   r_   r   r   r   r  r   r   )r   rE  r   vectorsis_cudas        r9   meta_linalg_eigrJ    s     e\* !!%++. 	..u{{; 
 __U[["-]_CFooekko?G % F*G0P 7?r;   c                 v    | j                   j                  t        j                        j	                  dd      S )Nr   r  r   )mTr   rW   r   	transpose)r   s    r9   cloneBatchedColumnMajorrN    s*    66<<e&=&=<>HHRPPr;   r7  c                     t        |       S rE   )rN  )r   r  r7  s      r9   _cholesky_solve_helperrP    s     #4((r;   c                      t        j                   j                  dk\   fd       t        j                  j                  dk\  fd       t         d      \  }}t	        |||      S )Nr  c                  $    d j                    dS )Nz-b should have at least 2 dimensions, but has  dimensions insteadr   r   s   r9   rf   z cholesky_solve.<locals>.<lambda>  s    ?		{J]^ r;   c                  $    d j                    dS )Nz-u should have at least 2 dimensions, but has rS  rT  r  s   r9   rf   z cholesky_solve.<locals>.<lambda>  s    ?xGZ[ r;   cholesky_solve)rW   rh   r   !_linalg_broadcast_batch_dims_namerP  )r   r  r7  self_broadcastedA_broadcasteds   ``   r9   rV  rV    sh     
LL		Q^ 
LL	![ 'Ha!'#m ""2M5IIr;   c                     | j                         dk(  r%t        j                  | t        j                        S t	        | d       t        |       S )Nr   r   cholesky)r   rW   r   legacy_contiguous_formatr   rN  r   r7  s     r9   r[  r[  
  s@     zz|qE4R4RSSdJ'"4((r;   c                 0    t        | d       t        |       S )Ncholesky_inverse)r   rN  r]  s     r9   r_  r_    s     d./"4((r;   check_errorsc                    t        | d       t        | d       | j                  }t        |      }t	        |d      }| j                  |      }|j                  ||       | j                  |d|dz
   t        j                        }||fS )Nzlinalg.choleskyFr   r  r   )	r   r   r   r   r   r   r   rW   r  )r  r7  r`  A_shaper   	L_stridesr5  infoss           r9   linalg_cholesky_exre    s    a*+1/0ggGw<D ,GU;I	GAMM'9% KKD1H-U[[KAEe8Or;   tauc                 @    t        j                   j                  dk\  d        t        j                   j                  d       j                  d      k\  d        t        j                   j                  d      j                  d      k\  d        t        j                   j                  j                  z
  dk(   fd        j                  dkD  r: j                  d d }j                  d d t        j                  |k(  fd	       t        j                  j
                   j
                  k(   fd
       t        d d       t        j                   j                  t         j                  d       j
                   j                        S )Nr  c                       y)NzHtorch.linalg.householder_product: input must have at least 2 dimensions.r5   r5   r;   r9   rf   z,linalg_householder_product.<locals>.<lambda>4  rn   r;   r  r   c                       y)Nzbtorch.linalg.householder_product: input.shape[-2] must be greater than or equal to input.shape[-1]r5   r5   r;   r9   rf   z,linalg_householder_product.<locals>.<lambda>8  rn   r;   c                       y)Nz`torch.linalg.householder_product: input.shape[-1] must be greater than or equal to tau.shape[-1]r5   r5   r;   r9   rf   z,linalg_householder_product.<locals>.<lambda><  rn   r;   r4   c                  <    dj                    d j                    S )Nzptorch.linalg.householder_product: Expected tau to have one dimension less than input, but got tau.ndim equal to  and input.ndim is equal to rT  r   rf  s   r9   rf   z,linalg_householder_product.<locals>.<lambda>A  '    )),
2Nuzzl\ r;   c                      d  S )Nzltorch.linalg.householder_product: Expected batch dimensions of tau to be equal to input.shape[:-2], but got r5   actual_batch_tau_shapes   r9   rf   z,linalg_householder_product.<locals>.<lambda>K      66L5MO r;   c                  <    dj                    d j                    S )Nz,torch.linalg.householder_product: tau dtype z does not match input dtype r   rm  s   r9   rf   z,linalg_householder_product.<locals>.<lambda>S  s#    :399+*5;;-9 r;   z torch.linalg.householder_productrf  Fr>  r   r   r_   r   )
rW   rh   r   r   r   r_   r1  empty_stridedr   r   )r   rf  expected_batch_tau_shaperq  s   `` @r9   linalg_householder_productrw  -  sK   
 
LL

aZ 
LL

2%**R.(t 
LL

2#((2,&r
 
LL

SXX"	
 zzA~#(;;s#3 !$3B"&>>	
 
LL		U[[ 	
 6UEJ[[*5;;%Hkk||	 r;   c                 2   t        | d       t        | dd       | j                  | j                        }|j	                  | j                  t        | j                  d             | j                  | j                  d d t        j                        }||fS )Nzlinalg.inv_exF)r  r>  r  r   r   r   r   r   r   r   rW   r  )r  r`  r5  rd  s       r9   linalg_inv_ex_metarz  c  sq    a)1o%P	AGGAMM!''6qww%PQKKEKKK8Ee8Or;   LDpivotsinfo)	hermitianr`  r~  c                   t        | d       t        | d       t        j                  | j                  t        | j                  d      | j                  | j                        }| j                  | j                  d d t        j                        }| j                  | j                  d d t        j                        }|||fS )Nztorch.linalg.ldl_factor_exFr>  rt  r   r   r  )
r   r   rW   ru  r   r   r_   r   r   r  )r   r~  r`  r{  r|  r}  s         r9   linalg_ldl_factor_ex_metar  o  s     d894!=>			ZZ*4::Gjj{{	
B ^^DJJsO599^=F>>$**Sb/>;Dvtr;   )r~  c                d    t         d       t         d       t         d       t        j                  j
                  dk\  fd        j                  d d }t        j                  |j                  k(  fd       t        j                  t        j                  j                        fd       t        j                   j                  j                  k(   fd       t               \  }}t        j                  |t        |d	      j                  j                  
      S )Nztorch.linalg.ldl_solver  c                  $    d j                    dS )NzMtorch.linalg.ldl_solve: Expected B to have at least 2 dimensions, but it has rS  rT  )r(  s   r9   rf   z'linalg_ldl_solve_meta.<locals>.<lambda>      &&!46 r;   r   c                  $    d j                    dS )Nzjtorch.linalg.ldl_solve: Expected LD.shape[:-1] and pivots.shape to be the same, but got pivots with shape  insteadrB  r|  s   r9   rf   z'linalg_ldl_solve_meta.<locals>.<lambda>      ))/h@ r;   c                  "    d j                    S )Nz<torch.linalg.ldl_solve: Expected pivots to be integers. Got r   r  s   r9   rf   z'linalg_ldl_solve_meta.<locals>.<lambda>  s    Nv||n] r;   c                  <    dj                    d j                    S )Nz!torch.linalg.ldl_solve: LD dtype z does not match b dtype r   )r(  r{  s   r9   rf   z'linalg_ldl_solve_meta.<locals>.<lambda>  s"    3BHH:=UVWV]V]U^_ r;   Fr>  rt  )r   r   r  rW   rh   r   r   rP   is_integer_dtyper_   _linalg_broadcast_batch_dimsru  r   r   )r{  r|  r(  r~  expected_pivots_shapeB_broadcast_sizerS   s   ```    r9   linalg_ldl_solve_metar    s     b232781b":;	LL	!	
 HHSbM	LL-	
 
LLv||,] 
LL
AGG_ 7q"=a*+;uMggxx	 r;   Pr4  )pivotr  c                h    t        j                   j                  dk\   fd       t         j                        }|d   }|d   }t        ||      }||d<   |r j                  |      }n j                  dg      }||d<    j                  |      }||d<   ||d<    j                  |      }|||fS )Nr  c                  $    d j                    dS )Nz@linalg.lu: Expected tensor with 2 or more dimensions. Got size: r  rB  r  s   r9   rf   z linalg_lu_meta.<locals>.<lambda>  s    RSTSZSZR[[cd r;   r  r   r   )rW   rh   r   r   r   r   r   )	r  r  sizesr  r+  kr  r5  r4  s	   `        r9   linalg_lu_metar    s     
LL	!d
 MEb	Ab	A1AE"IKKKKE"I	EAE"IE"I	EAa7Nr;   LU)r  r`  c                    t        j                   j                  dk\   fd       t         j                        }|d   }|d   }t        j
                  |t        |d       j                   j                        }|j                          t        ||      |d<    j                  |t         j                        }|j                           j                  |t         j                        }|||fS )	Nr  c                  $    d j                    dS )NzFtorch.lu_factor: Expected tensor with 2 or more dimensions. Got size: r  rB  r  s   r9   rf   z*linalg_lu_factor_ex_meta.<locals>.<lambda>  s    XYZY`Y`Xaaij r;   r  r   Fr>  rt  r   )rW   rh   r   r   r   ru  r   r_   r   r@  r   r   r  )	r  r  r`  r  r  r+  r  r|  r}  s	   `        r9   linalg_lu_factor_ex_metar    s     
LL	!j
 MEb	Ab	A			*5EBggxx	
B 
IIK1E"I[[eii[0F 
IIK;;uEII;.Dvtr;   )r   adjointr  c                    t         d       t        j                   j                  j                  k(   fd       t        j                  j                  t        j                  k(  d        t         d       t         |d       t        j                   j                  d      j                  d      k(  d        t        j                   j                  d d j                  k(  fd       t               \  }}t        j                  |t        ||       j                  j                  	      }|j                         d
k7  r"|s |j                         r|j                         }|S )Nztorch.linalg.lu_solvec                  >    dj                    d j                    dS )NzPlinalg.lu_solve: Expected LU and B to have the same dtype, but found LU of type  and B of type r  r   )r(  r  s   r9   rf   z&linalg_lu_solve_meta.<locals>.<lambda>   s(    $$&HH:_QWWIXO r;   c                       y)NzElinalg.lu_solve: pivots should be a Tensor of scalar type torch.int32r5   r5   r;   r9   rf   z&linalg_lu_solve_meta.<locals>.<lambda>  rn   r;   zlinalg.lu_solver   c                       y)NzYlinalg.lu_solve: Number of pivots per batch should be same as the dimension of the matrixr5   r5   r;   r9   rf   z&linalg_lu_solve_meta.<locals>.<lambda>  rn   r;   c                  $    d j                    dS )Nzclinalg.lu_solve: Expected LU.shape[:-1] and pivots.shape to be the same, but got pivots with shape r  rB  r  s   r9   rf   z&linalg_lu_solve_meta.<locals>.<lambda>  r  r;   r>  rt  r   )r   rW   rh   r_   r  r   r+  r   r   r  ru  r   r   r   r  conj)r  r|  r(  r   r  r  rS   r,  s   ```     r9   linalg_lu_solve_metar    s.    267	LL
AGG	
 
LL		!W b12b!T#45	LL
v{{2&k 
LL
"%	
 7q"=a  *+;4xPggxx	F ||~4[[]FMr;   unpack_dataunpack_pivotsc                     t        j                   j                  dk\   fd       |r2t        j                  |j                  t         j                  k(  d        t         j                        }|d   }|d   }t        ||      }||d<   |r j                  |      }n j                  dg      }|r2||d<    j                  |      }	||d<   ||d<    j                  |      }
n$ j                  dg      }	 j                  dg      }
||	|
fS )Nr  c                  $    d j                    dS )NzFtorch.lu_unpack: Expected tensor with 2 or more dimensions. Got size: r  rB  )r  s   r9   rf   z lu_unpack_meta.<locals>.<lambda>5  s    XY[YaYaXbbjk r;   c                       	 y)Nztorch.lu_unpack: LU_pivots is expected to be a contiguous tensor of torch.int32 dtype.
Note: this function is intended to be used with the output produced by torch.linalg.lu_factorr5   r5   r;   r9   rf   z lu_unpack_meta.<locals>.<lambda>:  s    p r;   r  r   r   )	rW   rh   r   r_   r  r   r   r   r   )r  r|  r  r  r  r  r+  r  r  r5  r4  s   `          r9   lu_unpack_metar  +  s     
LL
1k LLEKK'	
 NEb	Ab	A1AE"ILLLL!b	LLb	b	LLLL!LL!a7Nr;   modec                       dk(  rd}d}||fS  dk(  rd}d}||fS  dk(  rd}d}||fS t        j                  d fd       fS )NreducedTcompleteFrc                      d  dS )Nzqr received unrecognized mode 'z=' but expected one of 'reduced' (default), 'r', or 'complete'r5   )r  s   r9   rf   z _parse_qr_mode.<locals>.<lambda>b  s    1$ 8N O r;   rW   rh   )r  	compute_qr  s   `  r9   _parse_qr_moder  U  s    y	 g 
		 g 
	 g 		
 gr;   QRc                    t        | d       t        | d       t        |      \  }}| j                  d   }| j                  d   }t	        ||      }|rMt        | j                        }|r|n||d<   | j                  |      }|j                  |t        |d             n| j                  dg      }t        | j                        }	|s|s|n||	d<   | j                  |	      }
|
j                  |	t        |	d             ||
fS )Nz	linalg.qrr  r   Fr>  r   )	r'  r   r  r   r   r   r   r   r   )r  r  r  reduced_moder  r+  r  Q_shaper  R_shaper  s              r9   linalg_qr_metar  j  s     ![!1k*,T2I|	A	A1Aqww-'aQKK 	g:7eTUKK 177mG#9!!GBK	GAMM'6w%PQa4Kr;   sign	logabsdetc                    t        | d       t        | dd       | j                  }| j                  |d d       }| j                  |d d t	        | j
                              }t        j                  |t        |d      | j
                  | j                        }| j                  |d d t        j                        }||||fS )Nzlinalg.slogdetFr  r   rt  r   )r   r   r   r   ra   r_   rW   ru  r   r   r  )r  r   r  r  r  r|  s         r9   _linalg_slogdetr    s     a)*1.6GGE;;uSbz"DE#2Joagg.FGI			*5%8ggxx	
B [[s5;;[7FB&&r;   full_matrices
compute_uvdriverc                 v   t        | d       t        | d       t        | j                  d d       }| j                  d   }| j                  d   }t	        j
                  ||      }|r|||r|n|gz   }| j                  |      }	|	j                  |t        |d             ||r|n||gz   }
| j                  |
      }t        |       dk(  }|j                  |
t        |
|             n$| j                  dg      }	| j                  dg      }| j                  ||gz   t        | j                              }|	||fS )	Nz
linalg.svdr  r   Fr>  r	  r   r   )r'  r   r   r   rW   r   r   r   r   r  ra   r_   )r  r  r  r  r   r  r+  r  U_shaper4  V_shapeVrI  Ss                 r9   _linalg_svd_metar    s(    !\"1l+aggcrl#J	A	AaA11==KK 	g:7eTU]1==KK 
 a.F*	g:7gVW KKKK 	
J!$OAGG,DEAa7Nr;   arg1arg2c                    | j                   d d }|j                   d d }t        ||      }t        |      }|| j                  d      | j                  d      gz  }t        |      }||j                  d      |j                  d      gz  }||fS )Nr  r   )r   r&   r   r   )r  r  arg1_batch_sizesarg2_batch_sizesexpand_batch_portionarg1_expand_sizearg2_expand_sizes          r9   r  r    s    
 zz#2zz#2,-=?OP012		"66012		"66---r;   c                     |rt        | ||       t        | |      \  }}|| j                  k(  r| n| j                  |      }||j                  k(  r|n|j                  |      }||fS rE   )r  r  r   ry  )r  r  r  r  r  arg1_broadcastedarg2_broadcasteds          r9   rW  rW    sv     tT40)EdD)Q&& !DJJ.DKK@P4Q  !DJJ.DKK@P4Q  ---r;   r   c                     | j                   d d }|j                  dk(  xs- | j                  dz
  |j                  k(  xr |j                   |k(  }|S )Nr   r4   )r   r   )r   r   expected_batched_rhs_shapevector_cases       r9   linalg_solve_is_vector_rhsr    sS    !&Sb!1**/ 

Q%**$R8R)R  r;   )r   r`  r,  r  r|  r}  c                    t         d       t        j                   j                  j                  k(   fd       t	               }|rj                  d      n}	t         |	|d       t        |	       \  }
}t        j                  |xs | d        |r|
d d n|
}t        j                  |t        ||       j                  j                        } j                  }t        j                  |t        |d       j                   j                        } j                  |d d t        j                        } j                  |d d t        j                        }||||f}||||f}t        d	 |D              rbt        ||      D ]S  \  }}t!        ||j                         |j#                  |j                  |j%                                t'        ||d
       U |S )Nzlinalg.solvec                  >    d j                    dj                    dS )NzKlinalg.solve: Expected A and B to have the same dtype, but found A of type r  r  r   )r  r(  s   r9   rf   z"_linalg_solve_ex.<locals>.<lambda>  s%    Ywwiqwwix9 r;   r   c                       	 y)Nzlinalg.solve: Vector broadcasting of the left hand side is not supported for left=False. In this case linalg.solve is equivalent to B / A.squeeze(-1)r5   r5   r;   r9   rf   z"_linalg_solve_ex.<locals>.<lambda>  s    K r;   rt  Fr   r  c              3   $   K   | ]  }|d u 
 y wrE   r5   rs   r>   s     r9   ru   z#_linalg_solve_ex.<locals>.<genexpr>  s     
&Q1D=
&s   )	copy_fromcopy_toexact_dtype)r   rW   rh   r_   r  	unsqueezer+  r  ru  r   r   r   r   r  r`  ra  r"   r   r   r$   )r  r(  r   r`  r,  r  r|  r}  r  B_B_broad_shaperS   result_shaperesult_r   LU_pivots_info_r   resr  os   ``                    r9   _linalg_solve_exr    s    1n-	LL	177	
 -Q2K'RQBaT>23B:M1	LLK	
 *5="%-L!!*<TBggxx	G GGE


*5%8ggxx	C kk%*EKKk8GKKcr
%++K6E2vt
$CC%
(C

&#
&&SM 	FDAqa)MM!''188:.QuE	F Jr;   )r   unitriangularr   r  r   c                   || j                  dg      }t        |t              st        dt	        |             t        | ||d       t        || d       \  }}|j                  dd      j                         xr |j                         }|rt        ||j                        }|S t        ||j                        r=|j                  |j                  dd      j                         |j                  dd       |S )Nr   zout must be TensorLike, got zlinalg.solve_triangularr  r   )r   rq   r    r   r{   r+  rW  rM  is_contiguousis_conjr"   r   r#   r   
transpose_)	r  r(  r7  r   r  r   r  A_avoid_copy_As	            r9   linalg_solve_triangular_metar  (  s     {kk1#c:&;DI;GHHaD";<.q!T:FB<<B'557HBJJLLRXX. J  RXX.KKR,223NN2r"Jr;   XM)r  rM  c                     t        j                   j                  dk\   fd       t        j                  j                  dk\  fd       t         d       j                  t         j
                  k(  rt               \  }}t        j                  |t        |d       j                   j                        }t        j                  |t        |d      j                  j                        }||fS j                  t         j                  k(  sj                  t         j                  k(  r+t        j                         } j                  dg      }||fS t        j                  dd	        fS )
Nr  c                  $    d j                    dS )NzMtorch.triangular_solve: Expected b to have at least 2 dimensions, but it has rS  rT  r   s   r9   rf   z'triangular_solve_meta.<locals>.<lambda>N  s    ))$79 r;   c                  $    d j                    dS )NzMtorch.triangular_solve: Expected A to have at least 2 dimensions, but it has rS  rT  r  s   r9   rf   z'triangular_solve_meta.<locals>.<lambda>U  r  r;   triangular_solveFr>  rt  r   c                       y)Nz+triangular_solve: Got an unexpected layout.r5   r5   r;   r9   rf   z'triangular_solve_meta.<locals>.<lambda>o  rn   r;   )rW   rh   r   r  r   stridedr  ru  r   r_   r   
sparse_csr
sparse_bsrr   r   )	r   r  r7  rM  r  self_broadcast_sizeA_broadcast_sizesolutioncloned_coefficients	   ``       r9   triangular_solve_metar  C  sL    
LL		Q	
 
LL	!	
 4$67xx5== 0LTST0U--&&$./BeT**;;	
 #00!./?5Q''88	
 ''' 
U%%	%U5E5E)E##D)!^^QC0 ''' 	UQR'''r;   c                 l   t        | d       t        | d       | j                  | j                  d d       }| j                  | j                        }|j	                  | j                  t        | j                  d             | j                  | j                  d d t        j                        }|||fS )Nz
linalg.detr  Fr>  r   r   ry  )r  detr  r|  s       r9   _linalg_det_metar  t  s    a&1l+
++aggcrl
#C	
QWW	BNN17775QR[["U[[[9FF?r;   c                 0    t        j                   j                  dk\  d        t        j                  j                  dk\  d        |rdndt        j                  j                     j                  d   k\  fd       t        j                  j                      j                  d   k(  fd       t        j                  j                  d    j                  d   k  d        t        j                   j                  j                  z
  d	k(   fd
       t        j                   j                  j                  k(   fd        j                  dkD  re j                  d d }j                  d d t        j                  |k(  fd       j                  d d t        j                  |k(  fd       t        j                  j                   j                  k(   fd       t        j                  j                   j                  k(   fd       t        d d       t        d d       t        j                  j                  t        j                  d      j                  j                        S )Nr  c                       y)Nz3torch.ormqr: input must have at least 2 dimensions.r5   r5   r;   r9   rf   zormqr.<locals>.<lambda>  rn   r;   c                       y)Nz3torch.ormqr: other must have at least 2 dimensions.r5   r5   r;   r9   rf   zormqr.<locals>.<lambda>  rn   r;   r  r   c                      d  dS )Ntorch.ormqr: other.shape[z0] must be greater than or equal to tau.shape[-1]r5   left_size_conditions   r9   rf   zormqr.<locals>.<lambda>  s    +,?+@@pq r;   c                      d  dS )Nr  z"] must be equal to input.shape[-2]r5   r  s   r9   rf   zormqr.<locals>.<lambda>  s    +,?+@@bc r;   c                       y)NzHtorch.ormqr: tau.shape[-1] must be less than or equal to input.shape[-1]r5   r5   r;   r9   rf   zormqr.<locals>.<lambda>  rn   r;   r4   c                  <    dj                    d j                    S )Nz[torch.ormqr: Expected tau to have one dimension less than input, but got tau.ndim equal to rl  rT  rm  s   r9   rf   zormqr.<locals>.<lambda>  rn  r;   c                  <    dj                    d j                    S )Nzhtorch.ormqr: Expected other to have the same number of dimensions as input, but got other.ndim equal to rl  rT  r   r   s   r9   rf   zormqr.<locals>.<lambda>  s+    ++0::,6RSXS]S]R^` r;   c                      d  S )NzWtorch.ormqr: Expected batch dimensions of tau to be equal to input.shape[:-2], but got r5   rp  s   r9   rf   zormqr.<locals>.<lambda>  rr  r;   c                      d  S )NzYtorch.ormqr: Expected batch dimensions of other to be equal to input.shape[:-2], but got r5   )actual_batch_other_shapes   r9   rf   zormqr.<locals>.<lambda>  s    66N5OQ r;   c                  <    d j                    dj                    S )NzPtorch.ormqr: Expected input and tau to have the same dtype, but input has dtype z and tau has dtype r   rm  s   r9   rf   zormqr.<locals>.<lambda>  s'    ##(;;-/B399+O r;   c                  <    d j                    dj                    S )NzRtorch.ormqr: Expected input and other to have the same dtype, but input has dtype z and other has dtype r   r  s   r9   rf   zormqr.<locals>.<lambda>  s'    ##(;;-/DU[[MS r;   ztorch.ormqrrf  r   Fr>  rt  )	rW   rh   r   r   r_   r1  ru  r   r   )	r   rf  r   r   rM  expected_batch_shaper  rq  r  s	   ```   @@@r9   ormqrr    s    
LL

aV 
LL

aV !%""	LL'(CIIbM9q 
LL'(EKKO;c
 
LL		"R(Z
 
LL

SXX"	
 
LL

ejj 	
 zzA~${{3B/!$3B"&::	
 $);;s#3 $(<<	
 
LL		U[[ 	
 
LLu{{"	
 M3u5M5%9[[*5;;%Hkk||	 r;   c                    t        j                  t              dz  k(  fd        j                  }|dz   k(  }|}| }ddlm}m} |r-t        d|      D ]  }	 || j                  |	      dk7        } n+t        |      D ]  }	 || j                  |	      dk7        } t        j                   |||       fd       y )Nr  c                  ,    dd z   dt               S )Nzpadding size is expected to be r  z, but got: r   )r   paddings   r9   rf   z,_padding_check_valid_input.<locals>.<lambda>  s    1!c'+c'l^T r;   r   sym_andsym_orr4   c                  :    d dz    d dz    dj                    S )N	Expected r4   zD or r  zcD (batch mode) tensor with possibly 0 batch size and other non-zero dimensions for input, but got: rB  )r   r   s   r9   rf   z,_padding_check_valid_input.<locals>.<lambda>  s2    aycAgY /AAFO r;   )	rW   rh   r   r   rt  r  r  r   r   )
r   r  r   	input_dimis_batch_modevalid_batch_modevalid_non_batch_moder  r  r   s
   ```       r9   _padding_check_valid_inputr"    s    	LLGCT
 

I#'*M$,,Eq)$ 	MA&'7A!9KL	M y! 	UA#*+?ARSAS#T 	U 
LL!56	
r;   c                   	
 ddl m} d}dd} j                  dk(  r j                  d      }dz  |dz  }t	         |d       |\  
 j                  |      } j                        
z   z   	|r)t        j                   |
k  k         
fd       t        j                  	dk\  	fd        j                  dk(  r j                  |	f      S  j                  ||	f      S )	Nr   r  r4   r1   r   c                  4    d d d  dj                    S NzcArgument #4: Padding size should be less than the corresponding input dimension, but got: padding (ry   ) at dimension 
 of input rB  dim_wr   pad_lpad_rs   r9   rf   z_pad1d_common.<locals>.<lambda>  2    %%*G2eWOE7*UZU`U`Tac r;   c                      d  d S )Nz
input (W: z%) is too small. Calculated output W: r5   )input_woutput_ws   r9   rf   z_pad1d_common.<locals>.<lambda>  s    *WI%J8*U r;   r  )rt  r  r   r   r"  rW   rh   r   )r   r  is_reflectionr  	dim_planenbatchnplaner*  r/  r0  r+  r,  s   `      @@@@@r9   _pad1d_commonr5    s    =IEFzzQA
Q	ug15LE5ZZ	"FjjG&HEGOUW_5	
 
LLAU
 zzQ1229::r;   c                     t        | |d      S NTr1  )r5  r   r  s     r9   meta_reflection_pad1dr:  '       t<<r;   c                      t        j                   j                  t         j                  k7   fd       t	         |d      S )Nc                  @    d j                   j                          dS )Nz)"replication_pad1d" not implemented for ''r_   __str__r   s   r9   rf   z(meta_replication_pad1d.<locals>.<lambda>2       =ekk>Q>Q>S=TTUX r;   Fr8  )rW   rh   r_   boolr5  r9  s   ` r9   meta_replication_pad1drD  -  5     
LLuzz!X u==r;   c                   	 ddl m} d|s#t        j                  t	        |      dk(  d        j
                  dk(  rdz  |\  	j                        }|z   	z   |r)t        j                   ||k  	|k        	fd       t        j                   j                        k(   fd       j                  j                        S )	Nr   r$  r4   r  c                       y)Nz padding size is expected to be 2r5   r5   r;   r9   rf   z(_pad1d_backward_common.<locals>.<lambda><  rn   r;   r1   c                  4    d d d  dj                    S r&  rB  r)  s   r9   rf   z(_pad1d_backward_common.<locals>.<lambda>I  r-  r;   c                  2    d dj                          S Nz(grad_output width unexpected. Expected: , Got: r   r*  grad_outputr0  s   r9   rf   z(_pad1d_backward_common.<locals>.<lambda>Q  "    :8*GKL\L\]bLcKde r;   )	rt  r  rW   rh   r   r   r   r   r   )
rM  r   r  r1  r  r/  r*  r0  r+  r,  s
   ``    @@@@r9   _pad1d_backward_commonrO  7  s    =ES\Q&(RSzzQ
LE5jjG&HEGOUW_5	
 
LLK$$U++e
 ??5;;''r;   
grad_inputc                      t        | ||d      S r7  rO  rM  r   r  s      r9   meta_reflection_pad1d_backwardrT  W  s     "+ugTRRr;   c                      t        | ||d      S )NFr8  rR  rS  s      r9   meta_replication_pad1d_backwardrV  ]  s     "+ugUSSr;   c                V   	
 ddl m} d
d	d}d}t         |d        j                  }|dk(  r  j	                  d      }
dz  
	dz  	|dz  }|\   j	                  |      } j	                  	       j	                  
      z   z   z   z   |rRt        j                   |k  k        
 fd       t        j                   |k  k        	 fd       t        j                  dk\  xs dk\  fd	        j                  d
k(  r j                  |f      S  j                  ||f      }t               dk7  rFt        j                         t
        j                  k(  r |j                  t
        j                        }|S )Nr   r$  r  r4   r      c                  4    d d d  dj                    S r&  rB  r)  s   r9   rf   z_pad2d_common.<locals>.<lambda>  r-  r;   c                  4    d d d  dj                    S NzcArgument #6: Padding size should be less than the corresponding input dimension, but got: padding (ry   r'  r(  rB  dim_hr   pad_bpad_ts   r9   rf   z_pad2d_common.<locals>.<lambda>  r-  r;   c                       d  d d d S )Nz
input (H:  W: z%) is too small. Calculated output H: r5   )input_hr/  output_hr0  s   r9   rf   z_pad2d_common.<locals>.<lambda>  s*    	gY /$$,:T(= r;   r1   r	  r   )rt  r  r"  r   r   rW   rh   r   r  rP   r   channels_lastrw  )r   r  r1  r  
dim_slicesr3  r   r4  r  r]  r*  rb  r/  rc  r0  r^  r+  r,  r_  s   `        @@@@@@@@@@r9   _pad2d_commonrf  c  s   =EEJFug15::DqyA

a
!(E5%ZZ
#FjjGjjG&H&HEGOUW_5	
 	EGOUW_5	
 
LLA&Q	
 zzQ(;<<&&(H!EF &(++E2e6I6IIYYU-@-@YAFr;   c                     t        | |d      S r7  )rf  r9  s     r9   meta_reflection_pad2drh    r;  r;   c                      t        j                   j                  t         j                  k7   fd       t	         |d      S )Nc                  @    d j                   j                          dS )Nz)"replication_pad2d" not implemented for 'r>  r?  rA  s   r9   rf   z(meta_replication_pad2d.<locals>.<lambda>  rB  r;   Fr8  )rW   rh   r_   rC  rf  r9  s   ` r9   meta_replication_pad2drk    rE  r;   c                 ^    t        j                  |      }t        j                  |      }||fS rE   r   )grad_wsaved_vsaved_gsaved_normsr   grad_vgrad_gs          r9   meta_weight_norm_backwardrs    s.     g&Fg&F6>r;   c                     ddd}|j                   }|j                         dk(  rdz  dz  |dz  }|\  }}}}|   }	|   }
|	|z   |z   |
|z   |z   t        j                   j	                        k(   fd       t        j                   j	                        k(   fd       |j                  |j                         S )Nr  r4   r   rX  c                  2    d dj                          S rJ  r   rL  s   r9   rf   z%meta_pad2d_backward.<locals>.<lambda>  rN  r;   c                  2    d dj                          S Nz)grad_output height unexpected. Expected: rK  r   r]  rM  rc  s   r9   rf   z%meta_pad2d_backward.<locals>.<lambda>  "    ;H:W[M]M]^cMdLef r;   )r   r   rW   rh   r   r   )rM  r   r  r2  re   r+  r,  r_  r^  rb  r/  r]  r*  rc  r0  s   `          @@@@r9   meta_pad2d_backwardrz    s     EEIJxxzQ

Q	!(E5%GG&H&H	LLK$$U++e 
LLK$$U++f >>$**%%r;   c          	      N   	
 ddl m} dd
d	d}t         |d        j                  dk(  }|r% j	                  d      }dz  
dz  
	dz  	|dz  }|\   j	                  |      } j	                  	       j	                  
       j	                        z   z   z   z   z   z   |r{t        j                   |k  k         fd       t        j                   |k  k        
 fd	       t        j                   |k  k        	 fd
       ddl m} t        j                   |dk\  dk\  dk\        fd       |r j                  |f      S  j                  |f      S )Nr   r$  r1   r  r4   r      c                  4    d d d  dj                    S r&  rB  r)  s   r9   rf   z_pad3d_common.<locals>.<lambda> 	  r-  r;   c                  4    d d d  dj                    S r[  rB  r\  s   r9   rf   z_pad3d_common.<locals>.<lambda>	  r-  r;   c                  4    d d d  dj                    S )NzcArgument #8: Padding size should be less than the corresponding input dimension, but got: padding (ry   r'  r(  rB  )dim_dr   pad_bkpad_fs   r9   rf   z_pad3d_common.<locals>.<lambda>	  s2    %%*G2fX_UG:V[VaVaUbd r;   r  c                  ,    d  d d d d d S )Nz
input (D:  H: ra  z%) is too small. Calculated output D: r5   )input_drb  r/  output_drc  r0  s   r9   rf   z_pad3d_common.<locals>.<lambda>	  s7    	gYd7) <$$,:T(4zK r;   )	rt  r  r"  r   r   rW   rh   r  r   )r   r  r1  r  r2  
batch_moder3  r4  r  r  r]  r*  r  rb  r/  r  rc  r0  r^  r  r  r+  r,  r_  s   `        @@@@@@@@@@@@@@@r9   _pad3d_commonr    s   =EEEIug15qJA


Q	07-E5%vZZ	"FjjGjjGjjG'H&H&HEGOUW_5	
 	EGOUW_5	
 	EGOVg%56	
 =	LLx1}h!mX];	
 	
 (HMNN(HEFFr;   c                     t        | |d      S r7  )r  r9  s     r9   meta_reflection_pad3dr  $	  r;  r;   c                      t        j                   j                  t         j                  k7   fd       t	         |d      S )Nc                  @    d j                   j                          dS )Nz)"replication_pad3d" not implemented for 'r>  r?  rA  s   r9   rf   z(meta_replication_pad3d.<locals>.<lambda>/	  rB  r;   Fr8  )rW   rh   r_   rC  r  r9  s   ` r9   meta_replication_pad3dr  *	  rE  r;   c                 T    t        j                  t        |      dk(  d        |j                  dk  rt	        d|j                          j                  |j                  k7  r%t	        d j                   d|j                         ddd|j                  d	k(  rdz  dz  dz  |\  }}}}}}|j                        }	|j                        }
|j                        }|	|z   |z   |
|z   |z   ||z   |z   t        j                   j                        k(   fd
       t        j                   j                        k(   fd       t        j                   j                        k(   fd       |j                  |j                        S )N   c                       y)Nz padding size is expected to be 6r5   r5   r;   r9   rf   z%meta_pad3d_backward.<locals>.<lambda>>	  rn   r;   r1   zinput.ndim must be > 3, got z,grad_output.ndim must equal input.ndim, got  != r  r4   r|  c                  2    d dj                          S rJ  r   rL  s   r9   rf   z%meta_pad3d_backward.<locals>.<lambda>Z	  rN  r;   c                  2    d dj                          S rw  r   rx  s   r9   rf   z%meta_pad3d_backward.<locals>.<lambda>^	  ry  r;   c                  2    d dj                          S )Nz(grad_output depth unexpected. Expected: rK  r   )r  rM  r  s   r9   rf   z%meta_pad3d_backward.<locals>.<lambda>b	  rN  r;   )rW   rh   r   r   r   r   r   r   )rM  r   r  r+  r,  r_  r^  r  r  r  rb  r/  r  r]  r*  r  rc  r0  s   `           @@@@@@r9   meta_pad3d_backwardr  4	  s    
LLW"$NOzzQ;EJJ<HII5::%:;;K;K:LDQVQ[Q[P\]
 	
 EEEzzQ


07-E5%vjjGjjGjjG'H&H&H	LLK$$U++e 
LLK$$U++f 
LLK$$U++e
 ??5;;''r;   pc                     t        j                  | j                         d        | j                  d      }| j	                  ||dz
  z  dz  f      j                  t         j                        S )Nc                       y)Nz(_pdist_forward requires contiguous inputr5   r5   r;   r9   rf   z%meta__pdist_forward.<locals>.<lambda>l	  rn   r;   r   r4   r  r   )rW   rh   r  r   r   rw  r\  )r   r  r+  s      r9   meta__pdist_forwardr  h	  sh     
LLP 			!A>>1A;!+-.1144 2  r;   gradpdistc                     t        j                  |j                         d        t        j                  |j                         d        t        j                  |t         j                        S )Nc                       y)Nz._pdist_backward requires self to be contiguousr5   r5   r;   r9   rf   z&meta__pdist_backward.<locals>.<lambda>x	  rn   r;   c                       y)Nz/_pdist_backward requires pdist to be contiguousr5   r5   r;   r9   rf   z&meta__pdist_backward.<locals>.<lambda>{	  rn   r;   r   )rW   rh   r  r   r\  )r  r   r  r  s       r9   meta__pdist_backwardr  t	  sW     
LLV 
LLX D0N0NOOr;   )r  r  c          
      0    ddl m}m} j                  d      }j                  d      }j                  d      }	 |t	        j
                   | j                  |||	f                  r j                  |||	f       t	        j                  j                         dk(  d        t	        j                  j                         dk(  d        t        j                  sGt	        j                   j                  j                  cxk(  xr j                  k(  nc  fd       j                  }
j                  |
d   |
d   t	        j                  d   k(  xr d   k(  fd	        j                   j                               S )
Nr   )guard_or_truesym_eqr4   r  r1   c                       yNzbatch1 must be a 3D tensorr5   r5   r;   r9   rf   zmeta_baddbmm.<locals>.<lambda>	  rn   r;   c                       yNzbatch2 must be a 3D tensorr5   r5   r;   r9   rf   zmeta_baddbmm.<locals>.<lambda>	  rn   r;   c                  V    dj                    d j                    dj                    S )Nz+Input dtypes must be the same, got: input: z
, batch1: z
, batch2: r   )batch1batch2r   s   r9   rf   zmeta_baddbmm.<locals>.<lambda>	  s0    A$**ZX^XdXdWeeopvp|p|o}~ r;   c            	      .    d d d d    d d    d	S Nz@Expected size for first two dimensions of batch2 tensor to be: [ry   z] but got: [r   r4   ].r5   batch2_sizesbscontraction_sizes   r9   rf   zmeta_baddbmm.<locals>.<lambda>	  s:    t2&'|LO3DB|TUFWWY[ r;   )rt  r  r  r   rW   sym_notr   ry  rh   r   
exp_config&skip_dtype_check_in_meta_registrationsr_   r   )r   r  r  r  r  r  r  dim1dim2dim3batch1_sizesr  r  r  s   ```        @@@r9   meta_baddbmmr  	  s7    L;;q>D;;q>D;;q>DU]]6$**tT46H#IJK{{D$-.	LL"$HI	LL"$HI<<JJ&,,6&,,6~	
 <<L<<L	aB#A	LLQ2E,q/5E"E	
 >>$))+&&r;   c                L    t        j                  | t         j                        S r   r  r   s     r9   meta_bernoullir  	  s     D0G0GHHr;   c                     | S rE   r5   r   r  r   s      r9   meta_bernoulli_r  	      Kr;   c                 L    t        j                  | t         j                        S r   r  r  s      r9   meta_bernoulli_pr  	  s     D0G0GHHr;   c                 ,    t        j                  |       S rE   r   r   s     r9   meta_poissonr  	  r   r;   c                     t        j                  |
| j                         k  d        t        j                  | t         j                        }t        j                  |       |fS )Nc                       y)NzJError in fused_moving_avg_obs_fake_quant_cpu: ch_axis must be < self.dim()r5   r5   r;   r9   rf   z6meta__fused_moving_avg_obs_fq_helper.<locals>.<lambda>	  rn   r;   r   )rW   rh   r   r   rC  )r   observer_onfake_quant_onrunning_minrunning_maxscale
zero_pointaveraging_const	quant_min	quant_maxch_axisper_row_fake_quantsymmetric_quantr   s                 r9   $meta__fused_moving_avg_obs_fq_helperr  	  sO      
LL$((*\ D

3DT"D))r;   c                 @   t        j                  | j                         dk(  d        t        j                  |j                         dk(  d        | j                  \  |j                  \  t        j                  k(  fd       |gt        j                  || j                  k(  xsA |t         j
                  k(  xr, | j                  t         j                  t         j                  fv d        || j                  n|}| j                  f|      S )Nr  c                       y)Nza must be 2Dr5   r5   r;   r9   rf   zmeta_mm.<locals>.<lambda>	  rn   r;   c                       y)Nzb must be 2Dr5   r5   r;   r9   rf   zmeta_mm.<locals>.<lambda>	  rn   r;   c            	      "    d d  d d d	S )Nz/a and b must have same reduction dim, but got [ry   z] X [r  r5   )M1M2Nr  s   r9   rf   zmeta_mm.<locals>.<lambda>	  s(    A!Brd%PRtSUVWUXXZ[ r;   c                       y)NzFout_dtype must be the same as input dtype or fp32 for fp16/bf16 inputsr5   r5   r;   r9   rf   zmeta_mm.<locals>.<lambda>	  rn   r;   r   )	rW   rh   r   r   r_   r  r  r  r   )r7   r8   r  rT   r  r  r  r  s       @@@@r9   meta_mmr  	  s     
LLA56	LLA56GGEArGGEB	LL
b[   U]]* ?GGu~~>>\	
 (/177YL;;1v\;22r;   c                      |r(t         fdt         j                        D              S t        j                   j
                        S )Nc              3   H   K   | ]  }|vrj                   |   nd   ywrZ  rB  )rs   r   dimsr   s     r9   ru   z+_compute_reduction_shape.<locals>.<genexpr>	  s$     UqatmTZZ]:Us   ")rg   r   r   rP   compute_reduction_output_shaper   )r   r  r  s   `` r9   r  r  	  s7    UE$))DTUUU//

DAAr;   c                    t        | t        j                  j                        r| j                  j
                  S t        | d      rEt        | j                  d      r/| j                  j
                  dk7  r| j                  j
                  S y)Nr   r{   r   r	  )rq   rW   _subclasses
FakeTensorfake_devicer{   hasattrr   )r  s    r9   r  r  	  sg    &%++667!!&&&!FMM6*MM&(}}!!!r;   input_tensorr   r  dilationis_transposedgroupsoutput_paddingc                 Z   dt         dt         dt         dt         dt         dt         fd}dt         dt         dt         dt         dt         dt         dt         fd	}	|j                  d
d  }
| j                  d
d  |r||j                  d   z  }nF|j                  d   }t        j                  |j                  d   |z  | j                  d   k(  d        | j                  d   |gt	        |t
              r|gt              z  }n t        |      dk(  r|d   gt              z  }t	        |t
              r|gt              z  }n t        |      dk(  r|d   gt              z  }t	        |t
              r|gt              z  }n t        |      dk(  r|d   gt              z  }d }|rCt	        |t
              r|gt              z  }n#t        |      dk(  r|d   gt              z  }n|}|sddlm} t        t                    D cg c]  }|   d
||   z  z    c}t        t                    D cg c]  }||   |
|   dz
  z  dz    c}t        j                   |t        d      D cg c]
  \  }}||k\   c}} fd       t        t                    D ]]  }|r/j                   |	|   ||   ||   |
|   ||   ||                4j                   ||   ||   ||   |
|   ||                _ ddlm} ddlm} t	        | |      r| j                  n| j                   }|j"                  dk(  xr t        j$                  j&                  d u }|s1t        j                   |d
d  D cg c]  }|dkD  	 c} fd       S c c}w c c}w c c}}w c c}w )Nlnr  r   r  r  rA   c                 6    | d|z  z   ||dz
  z  z
  dz
  |z  dz   S )a  
        Formula to apply to calculate the length of some dimension of the output

        See: https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html

        Args:
            ln: length of the dimension
            p: padding in that dim
            d: dilation in that dim
            k: kernel size in that dim
            s: stride in that dim
        Returns:
            The output length
        r  r4   r5   )r  r  r   r  r  s        r9   _formulaz+calc_conv_nd_return_shape.<locals>._formula	
  s.     QU
Q!a%[(1,2Q66r;   rF   c                 <    | dz
  |z  d|z  z
  ||dz
  z  z   |z   dz   S )a  
        Formula to apply to calculate the length of some dimension of the output
        if transposed convolution is used.
        See: https://pytorch.org/docs/stable/generated/torch.nn.ConvTranspose2d.html

        Args:
            ln: length of the dimension
            p: padding in that dim
            d: dilation in that dim
            k: kernel size in that dim
            s: stride in that dim
            op: output padding in that dim

        Returns:
            The output length
        r4   r  r5   )r  r  r   r  r  rF   s         r9   _formula_transposedz6calc_conv_nd_return_shape.<locals>._formula_transposed
  s2    " Q!|a!e#a1q5k1B6::r;   r  r4   r   c                       y)NzInvalid channel dimensionsr5   r5   r;   r9   rf   z+calc_conv_nd_return_shape.<locals>.<lambda>5
  rn   r;   r$  T)strictc                  n    ddj                  d D               ddj                  d  D               dS )Nz+Calculated padded input size per channel: (r  c              3   2   K   | ]  }t        |        y wrE   strr  s     r9   ru   z>calc_conv_nd_return_shape.<locals>.<lambda>.<locals>.<genexpr>g
  s     <!s1v<   z). Kernel size: (c              3   2   K   | ]  }t        |        y wrE   r  r  s     r9   ru   z>calc_conv_nd_return_shape.<locals>.<lambda>.<locals>.<genexpr>h
  s     +MqCF+Mr  z6). Kernel size can't be greater than actual input size)join)effective_kernelpadded_inputs   r9   rf   z+calc_conv_nd_return_shape.<locals>.<lambda>e
  sJ    JJ<|<<= >!!&+M<L+M!M N OFG r;   )r  r  r	  c                  .    dt                ddd   dS )NzGiven input size per channel: z&. Calculated output size per channel: r  z. Output size is too small)r   )r  	ret_shapes   r9   rf   z+calc_conv_nd_return_shape.<locals>.<lambda>
  s,    4T$ZL A33<QR=/ B'( r;   )r  r   rW   rh   rq   r   r   rt  r  r   ra  appendtorch._subclasses.fake_tensorr  r  r  r   r{   versionhip)r  r  r   r  r  r  r  r  r  r  kernel_sizeout_channelsoutput_padding_listr  r   r  r  r  r  r   is_cudnnr>   r  r  r  r  s                         @@@@r9   calc_conv_nd_return_shaper  	  s   7S 7S 7S 7S 7S 7S 7"; ; ; ; ; ; ;QT ;& ,,qr"Kab!DQ/||ALLOf$(:(:1(==0	

 ##A&5I&'"CI%	V	)s4y('7#)c$i'	W	1:,T*(G$:D	)	X!	QK=3t9,,0ng.#1"2SY"> A%#1!#4"5D	"A"0 A ;@D	:JKQQ!gaj.0K 3t9%
  QK;q>A-.2

 	%(7GPT%UVTQ!q&V	
	
 3t9 #GAJ QKN1I'* a'!*hqk;q>6RS9U%6 9< lJ/ 	      {{f$B):):d)BHIabM2qQU23(	
 A L
 Wd 3s   
N8N5N"=N(c                 b    t         j                  j                  |       t         j                  k(  S rE   )rW   _prims_commonr   rd  )tens    r9   is_channels_lastr	  
  s$    44S9U=P=PPPr;   running_meanrunning_vartrainingexponential_average_factorepsilonc                    ||j                   n|j                   }||j                   n|j                   }	t        |       t        j                  t        j                  j
                  j                  j                  |       fd       t        j                  |       }
|r#|j                  |      }|j                  |	      }n"|j                  d      }|j                  d      }|
||fS )Nr   c                      d  S )Nz/miopen_batch_norm: input must be contiguous in r5   )fmts   r9   rf   z(meta_miopen_batch_norm.<locals>.<lambda>
  s    A#G r;   r   )
r   r   rW   rh   opsr.   sym_is_contiguousdefaultr   r   )r  r  r  r
  r  r  r  r  save_mean_shapesave_var_shaper   	save_meansave_varr  s                @r9   meta_miopen_batch_normr  
  s     -9,Dl((&,,O*5*A[&&v||N
  
-C 
LL		((00S0QG
 

<s
;C$$_5	##N3$$T*	##D)	8##r;   c	           
          t        | |||||||r|nd       }	ddlm}
 d}d} |
| j                  |      dk(        rd|	|<   | j	                  |	      }|S )Nr   guard_or_falser4   )r  rt  r  r   r   )r  r  r  r   r  r  r  r  r  	shape_outr  input_channels_dimoutput_channels_dimr   s                 r9   	meta_convr!  
  sw     *'T	I El''(:;q@A)*	%& 
 
 
+CJr;   
transposed	benchmarkdeterministiccudnn_enabled
allow_tf32c                 (    t        | ||||||||	      S rE   )r!  )r  r  r  r   r  r  r"  r  r  r#  r$  r%  r&  s                r9   
meta__convr(  
  s,      
 
r;   mkldnnc
           
          t        | ||||d|g       }
| j                  |
      }t        j                  }| j	                         dk(  rt        j
                  }|j                  |      }|S )NFr|  r   )r  r   rW   rd  r   channels_last_3drw  )r  r  r  r  r   r  r  attrscalars	algorithmr  r   out_memory_formats                r9   meta_mkldnn_convolution_defaultr0    sp     .&&'8UFB
	 $$Y/!//" % 6 6ff#4f5
r;   c                 b    | j                  g | j                  d d |j                  d         S Nr   r   r   r   )r  r  r  r,  r-  r.  s         r9   meta_linear_pointwise_defaultr4  *  s5     %%&Q(:(:3B(?&Qa&QRRr;   r  c                 b    | j                  g | j                  d d |j                  d         S r2  r3  )r  packed_weightorig_weightr  r   s        r9   meta_mkl_linearr8  5  s:    ))@,$$Sb)@;+<+<Q+?@ r;   onednnc           
         t        | ||||	d|
d       }|| j                  }|t        j                  t        j                  t        j
                  t        j                  t        j                  fvrt        d|       | j                  ||      }t        |      dvrt        dt        |       d      t        j                  t        j                  t        j                  dt        |         }|j                  |      }|S )NFOoutput_dtype must be one of float32, bfloat16, uint8, int8, float8_e4m3fn, got r   )r1   rX  r|  z3Expect output to be 3d/4d/5d for conv1d/2d/3d, got r   r   )r  r_   rW   r  r  uint8r  r  r   r   r   r   rd  r+  rw  )r>   x_scalex_zpww_scalew_zpr  r   r  r  r  output_scaleoutput_zero_pointoutput_dtyper,  r-  r.  r  r   formats                       r9   meta_qconv_pointwiserF  ?  s   , .	
	 77LMMNNKKJJ 
 
 !abnaop  kk)<k8y>* Ec)nEUUVW  &&""%%
 i.	
 ff6f*
r;   c                 .    |dk7  rt        d| d      |S )Nsumz#binary_op_name must be 'sum', got 'r>  r   )r>   r=  r>  r?  r@  rA  accumr  r   r  r  r  rB  rC  rD  accum_scaleaccum_zero_pointbinary_op_namer  unary_op_nameunary_op_argsunary_op_algorithms                         r9   meta_qconv2d_pointwise_binaryrQ  x  s-    4 U" 5n5EQG  r;   c                 4   t        | j                        }|j                  d   |d<   |	t        j                  t        j                  t        j
                  t        j                  t        j                  fvrt        d|	       | j                  ||	      }|S )Nr4   r   zOoutput_dtype must be one of float32, bfloat16, int8, uint8, float8_e4m3fn, got r   )
r   r   rW   r  r  r  r<  r  r   r   )r>   r=  r>  r?  r@  rA  r  rB  rC  rD  post_op_namepost_op_argspost_op_algorithmr  r   s                  r9   meta_qlinear_pointwiserV    s    " AGG}771:RMMNNJJKK 
 
 !abnaop  kk,lk;
r;   c                 B   |dk(  r|S t        | j                        }|j                  d   |d<   |
t        j                  t        j                  t        j
                  t        j                  t        j                  fvrt        d|
       | j                  ||
      }|S )NrH  r4   r   r;  r   )
r   r   rW   r  r  r<  r  r  r   r   )r>   r=  r>  r?  r@  rA  x_2r  rB  rC  rD  x2_scalex2_zprM  r  rN  rO  rP  r  r   s                       r9   meta_qlinear_pointwise_binaryr[    s    , U"JAGG}771:RMMNNKKJJ 
 
 !abnaop  kk,lk;
r;   c                 v    t        | j                        }|j                  d   |d<   | j                  |      }|S )Nr4   r   )r   r   r   )r>   r?  r  r  r   s        r9   meta_linear_dynamic_fp16r]    s6     AGG}771:Rkk,'
r;   	quantizedr  r4   c                 .   t        | |||||      \  }}}| j                         dk(  r| j                  d      nd}	t        j                  }
| j                         dk(  r|||g}n|	|||g}t        j
                  || j                  | j                  |
      S NrX  r4   r1   r  )#max_pool2d_checks_and_compute_shaper   r   rW   rd  r   r_   r   r   r  r   r  r  	ceil_modenInputPlaneoutputHeightoutputWidthr3  r   r   s               r9   meta_quantized_max_pool2dri    s     0;9
		
 $)99;!#3B++99;!{;DK{CD{{++<<'	
 	
r;   c                     t        j                   j                         dk(   fd       t        j                  j                         dk(  fd       t        j                   j                  t         j                  t         j
                  t         j                  fv  fd       t        j                  j                  t         j                  k(  fd       t        j                  j                  t         j                  k(  fd       t        j                  j                   j                  k(  fd        j                   j                  d      j                  d       j                  	      S )
Nr  c                  ,    d j                          dS )Nzx must be a 2D tensor, got r  r   r>   s   r9   rf   z/meta_int4mm_packed_weight_cpu.<locals>.<lambda>      -H	QR+S r;   c                  ,    d j                          dS )Nzw must be a 2D tensor, got r  r   r?  s   r9   rf   z/meta_int4mm_packed_weight_cpu.<locals>.<lambda>  rm  r;   c                  "    d j                    S Nz#expected x to be f32/f16/bf16, got r   rl  s   r9   rf   z/meta_int4mm_packed_weight_cpu.<locals>.<lambda>  s    9!''C r;   c                  "    d j                    S Nzexpected w to be uint8, got r   ro  s   r9   rf   z/meta_int4mm_packed_weight_cpu.<locals>.<lambda>  s    .J177),T r;   c                  "    d j                    S )Nz q_group_size must be int64, got r   )q_group_sizes   r9   rf   z/meta_int4mm_packed_weight_cpu.<locals>.<lambda>  s    6|7I7I6JK r;   c                  "    d j                    S )Nz5q_scale_and_zeros must have the same dtype as x, got r   )q_scale_and_zeross   r9   rf   z/meta_int4mm_packed_weight_cpu.<locals>.<lambda>   s    KL]LcLcKde r;   r   r   )rW   rh   r   r_   r  r  r  r<  r   r   r   r>   r?  ru  rw  s   ````r9   meta_int4mm_packed_weight_cpury    s    QUUW\#STQUUW\#STGGu}}ennEEC	
 	GGu{{"$T	
 	%++-K	
 	##qww.e	
 {{166!9affQiqww{??r;   c                      t        j                   j                         k(  xr  j                     k(   fd       y )Nc                  j    d  d d ddj                          d dj                      z   S )NzExpected a tensor of dimension z and tensor.size[z] == ry   zbut got : dimension z] = r   r   )r   dim_sizer   r  s   r9   rf   z check_dim_size.<locals>.<lambda>)  sP    1#6GzQVW[V\\^_ .?zfll[cNdMe
fg r;   )rW   rh   r   r   )r  r   r}  r   s   ````r9   check_dim_sizer~  &  s6    	LL

>X 6$ >	gr;   r  r  r_   c                 ,    t        j                  |       S rE   r   )r   r  r  r_   s       r9   meta_quantize_per_tensorr  .  s     E""r;   c                     d } |d|      \  }}	t        j                  t        |      dv d        t        j                   j                  t         j                  t         j
                  t         j                  t         j                  fv fd       t        |      dk(  r||	}}
n%t        |      dk(  r|d   |d   }}
n |d|      \  }
} |d	|      \  }}t        j                  |d u xs |dk7  d
         j                         dk(  r j                  d      nd} j                  d      } j                  d      } j                  d      }t        ||||
d|      }t        ||	||d|      }t        j                         }t         ||	|
|||dd||||||        j                         dk(  r|||g}n||||g}t        j                  | j                   j                  |      S )Nc                      t        j                  t        |      dv  fd       |d   }t        |      dk(  r|n|d   }||fS )Nr4   r  c                      d  dS )Nzavg_pool2d: 4 must either be a single int, or a tuple of two intsr5   r  s   r9   rf   z1meta_avg_pool2d.<locals>.unpack.<locals>.<lambda>B      l4&(\] r;   r   r4   rW   rh   r   r  r  HWs   `   r9   unpackzmeta_avg_pool2d.<locals>.unpack?  G    H]	
 FSQACF!tr;   r  r   r4   r  c                       yNzOavg_pool2d: stride must either be omitted, a single int, or a tuple of two intsr5   r5   r;   r9   rf   z!meta_avg_pool2d.<locals>.<lambda>K  rn   r;   c                  @    d j                   j                          dS )Nz""avg_pool2d" not implemented for 'r>  r?  rA  s   r9   rf   z!meta_avg_pool2d.<locals>.<lambda>O      6u{{7J7J7L6MQQ r;   r   r4   r   r  c                       yNzdivisor must be not zeror5   r5   r;   r9   rf   z!meta_avg_pool2d.<locals>.<lambda>\  rn   r;   rX  rb  r  r   r1   r  )rW   rh   r   r_   r<  uint16uint32rG  r   r   pooling_output_shaperP   r   pool2d_shape_checkr   r   )r   r  r   r  re  count_include_paddivisor_overrider  kHkWdHdWpadHpadWr3  rf  inputHeight
inputWidthrg  rh  r   r   s   `                     r9   meta_avg_pool2dr  5  s    M;/FB	LLFy a 
LLEKKu||U\\RRQ 6{aRB	V	F1IB&)B	7+JD$	LLD 9$4$9*
  %yy{a/UZZ^QF**R.K**R.KBJ'Rr1iPL&z2tRINK//6M



		$ yy{a\;7\;?;;kk||#	 r;   c                     t        | ||||||dd|	|
||||       | j                         }|	}t        |||dz
  |       t        |||dz
  |       t        |||dz
  |       y )Nr4   r1   r  )r  r   r~  )r   
gradOutputr3  r  r  r  r  r  r  rf  r  r  rg  rh  
mem_formatr   nOutputPlanes                    r9   avg_pool2d_backward_shape_checkr    s    " 



		$ 99;DL:tTAX|<:tTAX|<:tTAX{;r;   c                    t        j                  t        |      dk(  xs t        |      dk(  d        |d   }t        |      dk(  r|n|d   }	t        j                  t        |      dk(  xs t        |      dk(  xs t        |      dk(  d        t        |      dk(  r|n|d   }
t        |      dk(  r|	nt        |      dk(  r|
n|d   }t        j                  t        |      dk(  xs t        |      dk(  d        |d   }t        |      dk(  r|n|d   }t        j                  |d u xs |dk7  d        |j                  }|j	                         dk(  r|d	   nd}|d
   }|d   }|d   }t        ||||
d|      }t        ||	||d|      }t        j                  |      }t        || |||	|
|||||||||       t        j                  ||j                  |j                  |      S )Nr4   r  c                       y)NzKavg_pool2d: kernel_size must either be a single int, or a tuple of two intsr5   r5   r;   r9   rf   z*meta_avg_pool2d_backward.<locals>.<lambda>  rn   r;   r   c                       yr  r5   r5   r;   r9   rf   z*meta_avg_pool2d_backward.<locals>.<lambda>  rn   r;   c                       y)NzGavg_pool2d: padding must either be a single int, or a tuple of two intsr5   r5   r;   r9   rf   z*meta_avg_pool2d_backward.<locals>.<lambda>  rn   r;   c                       yr  r5   r5   r;   r9   rf   z*meta_avg_pool2d_backward.<locals>.<lambda>  rn   r;   rX  rb  r  r  r   r  )rW   rh   r   r   r   r  rP   r   r  r   r_   r   )gradOutput_r   r  r   r  re  r  r  r  r  r  r  r  r  
input_sizer3  rf  r  r  rg  rh  r  s                         r9   meta_avg_pool2d_backwardr    s    
LLKA6[!1Q!6] 
QB;1$+a.B	LLFq@CK1,@Fq0@a 6{aVAYB6{a3v;!+;RB	LLG.S\Q.Y 1:Dw<1$4'!*D	LLD 9$4$9*
 J$yy{a/Z^QFR.KR.KBJ'Rr1iPL&z2tRINK,,U3J#



$ ;;kk|| 	 r;   c                     t        j                  t        |      dv d        |d   }t        |      dk(  r|n|d   }t        |      dk(  r|n|d   }	t        j                  | xs t        |      dv d        t        j                   j                  t         j                  t         j
                  t         j                  t         j                  fv fd       |s|n|d   }
|s|nt        |      dk(  r|
n|d   }|s|	nt        |      dk(  r|
n|d   }t        j                  t        |      dv d        |d   }t        |      dk(  r|n|d   }t        |      dk(  r|n|d   }t        j                   j                  d	v d
        t        j                  |d u xs |dk7  d         j                  d      } j                  d      } j                  d      } j                  d      } j                  d      }t        ||||
d|      }t        ||||d|      }t        ||	||d|      }t         ||||	|
|||||ddd||||||dd        j                  dk(  r j                  ||||f      S  j                  |||||f      S )Nr4   r1   c                       yNzFavg_pool3d: kernel_size must be a single int, or a tuple of three intsr5   r5   r;   r9   rf   z!meta_avg_pool3d.<locals>.<lambda>	  rn   r;   r   r4   r  c                       yNzJavg_pool3d: stride must be omitted, a single int, or a tuple of three intsr5   r5   r;   r9   rf   z!meta_avg_pool3d.<locals>.<lambda>  rn   r;   c                  @    d j                   j                          dS )Nz""avg_pool3d" not implemented for 'r>  r?  rA  s   r9   rf   z!meta_avg_pool3d.<locals>.<lambda>  r  r;   c                       yNzBavg_pool3d: padding must be a single int, or a tuple of three intsr5   r5   r;   r9   rf   z!meta_avg_pool3d.<locals>.<lambda>  rn   r;   rX  r|  c                       yNz9non-empty 4D or 5D (batch mode) tensor expected for inputr5   r5   r;   r9   rf   z!meta_avg_pool3d.<locals>.<lambda>%  rn   r;   c                       yr  r5   r5   r;   r9   rf   z!meta_avg_pool3d.<locals>.<lambda>*  rn   r;   rb  r  r  r   zavg_pool3d()T)check_input_sizerX  )rW   rh   r   r_   r<  r  r  rG  r   r   r  pool3d_shape_checkr   )r   r  r   r  re  r  r  kTr  r  dTr  r  padTr  r  r3  nslicesitimeiheightiwidthotimeoheightowidths   `                       r9   meta_avg_pool3dr    s    
LLKF"X 
QB;1$+a.B;1$+a.B	LL
+c&kV+\ 
LLEKKu||U\\RRQ vayBc&kQ&6F1IBc&kQ&6F1IB	LLGT 1:Dw<1$4'!*Dw<1$4'!*D	LL

fK
 
LLD 9$4$9*
 ZZ]FjjnGJJrNEjjnGZZ^F D"aCE"7Bb!YGG!&"dB9EF





			-2 zzQ@AAHIIr;   c                    t        j                  t        |      dv d        |d   }t        |      dk(  r|n|d   }	t        |      dk(  r|n|d   }
t        j                  | xs t        |      dv d        |s|n|d   }|s|	nt        |      dk(  r|n|d   }|s|
nt        |      dk(  r|n|d   }t        j                  t        |      dv d        |d   }t        |      dk(  r|n|d   }t        |      dk(  r|n|d   }t        j                  |j                  dv d	        t        j                  |d u xs |dk7  d
        |j	                  d      }|j	                  d      }|j	                  d      }|j	                  d      }t        ||||d|      }t        ||	||d|      }t        ||
||d|      }t        || |||	|
||||||||||||d       |j                  |j                        S )Nr  c                       yr  r5   r5   r;   r9   rf   z*meta_avg_pool3d_backward.<locals>.<lambda>d  rn   r;   r   r4   r  c                       yr  r5   r5   r;   r9   rf   z*meta_avg_pool3d_backward.<locals>.<lambda>l  rn   r;   c                       yr  r5   r5   r;   r9   rf   z*meta_avg_pool3d_backward.<locals>.<lambda>t  rn   r;   r  c                       yr  r5   r5   r;   r9   rf   z*meta_avg_pool3d_backward.<locals>.<lambda>|  rn   r;   c                       yr  r5   r5   r;   r9   rf   z*meta_avg_pool3d_backward.<locals>.<lambda>  rn   r;   rb  r  r  r   zavg_pool3d_backward())	rW   rh   r   r   r   r  avg_pool3d_backward_shape_checkr   r   )rM  r   r  r   r  re  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  otime_for_shape_checkoheight_for_shape_checkowidth_for_shape_checks                           r9   meta_avg_pool3d_backwardr  V  s    
LLKF"X 
QB;1$+a.B;1$+a.B	LL
+c&kV+\ vayBc&kQ&6F1IBc&kQ&6F1IB	LLGT 1:Dw<1$4'!*Dw<1$4'!*D	LL

fK
 
LLD 9$4$9*
 jjnGJJrNEjjnGZZ^F0D"aS27Bb!YW1&"dB9U#





', ??5;;''r;   c                 ,    t        j                   j                  dk(  xs  j                  dk(   fd        j                  d d t	        |      z   }t        j                         }t        j                  | j                   j                  |      S )Nr1   rX  c                  "    d j                    S )Nz"Expected 3D or 4D tensor, but got rB  r   s   r9   rf   z*meta_adaptive_avg_pool2d.<locals>.<lambda>      4TZZLA r;   r  r  )
rW   rh   r   r   rg   rP   r   r   r_   r   )r   output_sizer  r   s   `   r9   meta_adaptive_avg_pool2dr    s|    	LL		Q($))q.A ::cr?U;%77L//5M ;;jj{{#	 r;   c                      t        j                   j                  dk(  xs  j                  dk(   fd        j                   j                  d d t        |      z         S )NrX  r|  c                  "    d j                    S )Nz"Expected 4D or 5D tensor, but got rB  r   s   r9   rf   z*meta_adaptive_avg_pool3d.<locals>.<lambda>  r  r;   r  )rW   rh   r   r   r   rg   )r   r  s   ` r9   meta_adaptive_avg_pool3dr    sO    	LL		Q($))q.A >>$**Sb/E+,>>??r;   c                      j                   }t        d|      D ].  t        j                   j	                        dkD   fd       0 t        j                  |dk(  xs |dk(  fd       t        j                  j
                   j
                  k(   fd       t        j                  }t              rt        j                  }j                  j                        j                  |      S )	Nr4   r   c                  *    d j                    d dS )Nz{adaptive_avg_pool2d_backward(): Expected grad_output to have non-zero                       size for non-batch dimensions,  with dimension  being emptyrB  )grad_outr   s   r9   rf   z4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>  s&     66>nn5EEUVWUXXdf r;   r1   rX  c                  "    d j                    S )NzBadaptive_avg_pool2d_backward(): Expected 3D or 4D tensor, but got rB  r   s   r9   rf   z4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>  s    TUYU_U_T`a r;   c                  <    dj                    d j                    S Nexpected dtype z! for `grad_output` but got dtype r   )r  r   s   r9   rf   z4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>  s    /$**-Nx~~N^_ r;   r   )r   r   rW   rh   r   r_   r   r	  rd  r   r   rw  )r  r   r   r   r   s   ``  @r9   "meta__adaptive_avg_pool2d_backwardr    s    ==D1d^ 
MM!q f	

 
LL	TQYa 
LL

hnn$_ ++M++>>$**%((}(EEr;   c                 d    t        | d       t        j                  |t        j                        S )Nadaptive_avg_pool3d_backwardr   )!_adaptive_pool_empty_output_checkrW   r   r\  rM  r   s     r9   "meta__adaptive_avg_pool3d_backwardr    s(     &k3QRD0N0NOOr;   rM  c                       j                   }t        d|      D ]/  t        j                   j	                        dkD   fd       1 y )Nr4   r   c                  .      dj                    d dS )Nzc(): Expected grad_output to have non-zero size for non-batch dimensions, but grad_output has sizes r  r  rB  )r$  rM  r   s   r9   rf   z3_adaptive_pool_empty_output_check.<locals>.<lambda>  s/    * --8->->,??OPQsR^` r;   )r   r   rW   rh   r   )rM  r$  r   r   s   `` @r9   r  r    sG    D1d^ 
Q!#	

r;   c                      j                   }t        j                  |dv  fd       t        d|      D ].  t        j                   j	                        dkD   fd       0 t        j                  t        |      dk(  d        d}d}d} j                   dk(  r j	                  d      }|dz  } j	                  |dz
        }|\  }} j                   d	k(  r;|||f} j                  |      }	 j                  |t        j                  
      }
|	|
fS ||||f}t        j                         } j                  |      j                  |      }	 j                  |t        j                  
      j                  |      }
|	|
fS )Nr1   rX  c                  "    d j                    S )Nz:adaptive_max_pool2d(): Expected 3D or 4D tensor, but got: rB  rA  s   r9   rf   z*meta_adaptive_max_pool2d.<locals>.<lambda>      LU[[MZ r;   r4   r   c                  *    dj                    d  dS )Nzjadaptive_max_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has sizes r  r  rB  r   r   s   r9   rf   z*meta_adaptive_max_pool2d.<locals>.<lambda>  %    '',{{m3CA3lT r;   r  c                       y)NzCadaptive_max_pool2d(): internal error: output_size.size() must be 2r5   r5   r;   r9   rf   z*meta_adaptive_max_pool2d.<locals>.<lambda>  rn   r;   rX  r1   r   r   )r   rW   rh   r   r   r   r   r   rP   r   rw  )r   r  r   dimHsizeBsizeDosizeHosizeWr   r   r   r   r   s   `           @r9   meta_adaptive_max_pool2dr    s|    ::D	LLZ 1d^ 
JJqMA	

 
LLKAU
 DEEzzQ

1	JJtax E NFFzzQFF+	ooi(//)5;;/?G|E662	33E:ooi(++-+H//)5;;/?BB' C 
 G|r;   c                 N     j                   }t        j                  |dv  fd       t         d       t        j                  j                   j                  k(   fd       t        j                        }j                  j                        j                  |      S )Nr  c                  "    d j                    S )NzKadaptive_max_pooling2d_backward(): Expected 3D or 4D grad_output, but got: rB  rM  s   r9   rf   z3meta_adaptive_max_pool2d_backward.<locals>.<lambda>!  s    ]^i^o^o]pq r;   adaptive_max_pool2d_backwardc                  <    dj                    d j                    S r  r   )rM  r   s   r9   rf   z3meta_adaptive_max_pool2d_backward.<locals>.<lambda>(  s!    /%++.OP[PaPaObc r;   r   )
r   rW   rh   r  r_   rP   r   r   r   rw  )rM  r   r   r   r   s   ``   r9   !meta_adaptive_max_pool2d_backwardr    s     D	LLq
 &k3QR	LL{(((c
 //6M??5;;'***GGr;   c                      j                   }t        j                  |dv  fd       t        d|      D ].  t        j                   j	                        dkD   fd       0 t        j                  t        |      dk(  d        d}d}d}|dk(  r j	                  d      }|dz  } j	                  |      }|\  }}}|d	k(  r||||f}	n|||||f}	 j                  |	      }
 j                  |	t        j                  
      }|
|fS )Nr  c                  "    d j                    S )Nz:adaptive_max_pool3d(): Expected 4D or 5D tensor, but got: rB  rA  s   r9   rf   z*meta_adaptive_max_pool3d.<locals>.<lambda>5  r  r;   r4   r   c                  *    dj                    d  dS )Nzjadaptive_max_pool3d(): Expected input to have non-zero size for non-batch dimensions, but input has sizes r  r  rB  r  s   r9   rf   z*meta_adaptive_max_pool3d.<locals>.<lambda>:  r  r;   r1   c                       y)NzCadaptive_max_pool3d(): internal error: output_size.size() must be 3r5   r5   r;   r9   rf   z*meta_adaptive_max_pool3d.<locals>.<lambda>B  rn   r;   r|  rX  r   )r   rW   rh   r   r   r   r   r   )r   r  r   dimDr  r  osizeTr  r  r   r   r   r   s   `           @r9   meta_adaptive_max_pool3dr
  /  s    ::D	LLZ 1d^ 
JJqMA	

 
LLKAU
 DEEqy

1	JJtE(FFFqyFFF3	E666:	
//)
$Cooiu{{o;G<r;   c                 P    t        | d       |j                  |j                        S )Nadaptive_max_pool3d_backward)r  r   r   )rM  r   r   s      r9   !meta_adaptive_max_pool3d_backwardr  [  s"     &k3QR??5;;''r;   c                 >    |t        d      | j                  |      S )Nz:cannot repeat_interleave a meta tensor without output_size)rv  r   )repeatsr  s     r9   meta_repeat_interleave_Tensorr  b  s%    WXX[))r;   c                 ~   | j                   j                  st        d| j                          |j                   j                  st        d|j                          t        | j	                  t        | j                               |j	                  t        |j                               t        j                        }|S )Nz!real must be floating point, got z!imag must be floating point, got rM   )r_   r!  r   rU   rw  r   r   rQ   )realimagr,  s      r9   meta_complexr  i  s     ::''@MNN::''@MNN+DJJ78+DJJ786>>F
 Mr;   )
fill_valuer  c                   t        |       dv r1| j                  || j                         ft        j                        S t        j
                  || j                         fd|ft        j                  | j                        S )N)r  mpsr  r   r4   r_   r   )r  r   r   rW   r   ru  r   )r   r   r  s      r9   nonzero_staticr  x  sj     411~~tTXXZ0

~CC""488:I**;;	
 	
r;   c                 
   t        j                  t        j                  d        t        j                  | j                         | j                         fd| j                         ft         j                  | j                        S )Nc                       y)NaY  The register_meta function for torch.nonzero() raises unimplemented by default, as a correct data-independent implementation does not exist. This implementation returns a fake value, assuming all elements of the tensor are non-zero. To enable this registration, please set 'torch.fx.experimental._config.meta_nonzero_assume_all_nonzero' to True.r5   r5   r;   r9   rf   znonzero.<locals>.<lambda>  rn   r;   r4   r  )	rW   _check_not_implementedr  meta_nonzero_assume_all_nonzeroru  r   r   r   r   r   s    r9   nonzeror    sf     
  22	S 	txxz"	
DJJLjj{{	 r;   c           
          t        j                  t              d        g }t              D ]  \  ft        j                  j                  t         j
                  t         j                  t         j                  t         j                  fv d        j                  t         j                  t         j                  fv rȉj                         }t        |      t        j                  j                  z    j                  k   fd       t        j                        D ]`  t        j                  j                      j                  z      k(   fd       |j                  |j                  d             b ]|j                         p|j                          |t        j                  t               j                  k   fd       dd lm} t%         |j&                         t               j                  k  r*j                  d        t               j                  k  r*d}d}D ]  |dk(  rd}|dk(  rd	} n d
}|sg }g }t              D ]*  \  	|j                         |j                         , t              D ]*  \  	|j                         |j                         ,  j)                  |       |g g g t              D ]\  \  }	@rj                   j                  |	          )j                   j                  |	          Ht%        j                        ^ fd}
 j+                  z   z         }ddlm}  | j1                         dk(        r|S  |
       }t3        j4                  |      \  }}t%        |      t%        t        t        |                  k7  r~t3        j6                  |j                  |      }t3        j8                  |      }t3        j6                  |t3        j:                  |            }|j=                  |j?                         |      }|S )Nc                       y)Nz#at least one index must be providedr5   r5   r;   r9   rf   z#meta_index_Tensor.<locals>.<lambda>  rn   r;   c                       y)Nz?tensors used as indices must be long, int, byte or bool tensorsr5   r5   r;   r9   rf   z#meta_index_Tensor.<locals>.<lambda>  rn   r;   c                  "    d j                    S )N)too many indices for tensor of dimension rT  r   s   r9   rf   z#meta_index_Tensor.<locals>.<lambda>  s    G		{S r;   c            	      N    dj                    d  dj                    dz    S )NzThe shape of the mask 
 at index z0 does not match the shape of the indexed tensor rB  )r   r   jr  r   s   r9   rf   z#meta_index_Tensor.<locals>.<lambda>  s<    "8ZPQs SJJN**U_`ade`e_f!h r;   r4   c                  <    dj                    dt                dS )Nr$  z (got rz   )r   r   )r   r   s   r9   rf   z#meta_index_Tensor.<locals>.<lambda>  s!    ;DII;fSQX\NZ[\ r;   r   Fr  Tc                     z   z   }t        | j                               }dgt              z  |t              t        | j                        t              z
   | j	                  ||      S )zI
        This follows restride_src in TensorAdvancedIndexing.cpp
        r   )r   r   r   r   
as_strided)r   r   r  after_shapebefore_shapereplacement_shapes      r9   _restride_srcz(meta_index_Tensor.<locals>._restride_src  so     00;>t{{}%KL#PSQ
 K
L!C

Oc+6F$FG ug..r;   r  ) rW   rh   rC  	enumerater_   r   r  r  r  r   r   r   r   r   r  selecttorch._refsrx  r   r'   r   r   rt  r  r   rP   3compute_elementwise_output_logical_to_physical_perm
apply_permr   invert_permr*  r   )r   r   r,  r  refsstatehas_contiguous_subspacer  transposed_indicesr   r.  r   r  restrided_selfpermrS   
perm_shaper  r+  r,  r   r   r'  r  r-  s   ``                @@@@@@@r9   meta_index_Tensorr<    s   	LLg MN #%Fg& !5LL

EIIuzz5::NNY {{uzz5::66--/K""

Ndii/S uzz* 8A&&A$**QU*;;h
 MM'..A"678 e$MM% /!0 G	LLG		!\
 (4(('23G
g,
"t g,
" E# 'A: aZ} ' #'
 #!'* 	1HAu A"))%0	1 "'* 	1HAu}A"))%0	1 ||D!$ !LK#%( 2
U= ""4::c?3##DJJsO4 $U[[ 12
/ ..(99KG
HCDdjjla'(

 #4(NGGWGD! DzT%D	*++%%cii6
66zB
%%j%2C2CD2IJ
nnSXXZ4Jr;   c                 ,   d }d }d }d } |||      }|
d   r/| j                  |j                               j                  |      }|
d   r/| j                  |j                               j                  |      }|
d   r| j                  |      }|||fS )Nc                 &   t        |       }t        |      }|t        j                  k(  s|t        j                  k(  rt        j                  S |t        j                  k(  s|t        j                  k(  rt        j                  S t        j                  S rE   )r   rW   rd  r+  r   )t1t2fmt1fmt2s       r9   _conv_memory_formatz6meta_convolution_backward.<locals>._conv_memory_format9  sp    $R($R(5&&&$%2E2E*E&&&5)))TU5K5K-K)))&&&r;   r   r   r4   r  )r   r   rw  )grad_output_input_weight_bias_sizes_optr   r  r  r"  r  r  output_maskbackend_grad_inputbackend_grad_weightbackend_grad_biasrC  r   s                   r9   meta_convolution_backwardrL    s      ' (8M1~)33FKKMBEE' F 
 1~*44W\\^DGG' H 
 1~(22>B 35FGGr;   c                   j                  d      }j                  d      }| j                  ||f      } t        j                  j	                         dk(  d        t        j                  j	                         dk(  d        t        j                  j                  d      j                  d      k(  fd       t        j                  j                  d      j                  d      k(  fd       t        j                  | j                  d      |k(  xr | j                  d      |k(  d	        | j                  | j                               S )
Nr4   r  r1   c                       yr  r5   r5   r;   r9   rf   zmeta_addbmm.<locals>.<lambda>W  rn   r;   c                       yr  r5   r5   r;   r9   rf   zmeta_addbmm.<locals>.<lambda>X  rn   r;   r   c                  P    d j                  d       dj                  d       S )Nz8batch1 and batch2 must have same number of batches, got r   r   r   r  r  s   r9   rf   zmeta_addbmm.<locals>.<lambda>[  s.    J6;;WX>JZZ_`f`k`klm`n_op r;   c            
          d j                  d       d j                  d       dj                  d       dj                  d       d	S )Nz#Incompatible matrix sizes for bmm (r4   r>   r  r   rz   r   rQ  s   r9   rf   zmeta_addbmm.<locals>.<lambda>_  sQ    1&++a.1A6;;q>BR S;;q>"!FKKN#316 r;   c                       y)Nz.self tensor does not match matmul output shaper5   r5   r;   r9   rf   zmeta_addbmm.<locals>.<lambda>f  rn   r;   )r   ry  rW   rh   r   r   )r   r  r  r  r  r  r  s    ``    r9   meta_addbmmrT  Q  s    ;;q>D;;q>D;;d|$D	LL"$HI	LL"$HI	LLA&++a.(p 
LLA&++a.(	
 
LL		!51!5@ >>$))+&&r;   c           
          t         j                  j                  | |j                  d      |j                  d      |j                  d      |j                  d      |j                  d            S )Nr_   r   r   r   r   r  )r.   r   r  r^   )r   r6  kwargss      r9   meta_randint_likerW  k  s]    ??""jj!zz(#zz(#::l+jj1 #  r;   )
grad_scale	found_infc       	         n    | |||||fD ])  t        j                  t        t              fd       + y )Nc                       dt                S Nz'exponent must be a tensor list but got r{   ls   r9   rf   z#meta__fused_adam_.<locals>.<lambda>      =d1gYG r;   rW   rh   rq   r   )r   gradsexp_avgsexp_avg_sqsmax_exp_avg_sqsstate_stepslrbeta1beta2weight_decayr   amsgradmaximizerX  rY  r_  s                  @r9   meta__fused_adam_rm  w  s:    & E8[/;O 
q$G	

r;   c       	             | |||||fD ])  t        j                  t        t              fd       + d } ||        ||       ||       ||       ||      fS )Nc                       dt                S r\  r]  r^  s   r9   rf   z"meta__fused_adam.<locals>.<lambda>  r`  r;   c                 R    | D cg c]  }t        j                  |       c}S c c}w rE   r   )tensor_listr  s     r9   empty_like_listz)meta__fused_adam.<locals>.empty_like_list  s!    -89  #999s   $ra  )r   rb  rc  rd  re  rf  rg  rh  ri  rj  r   rk  rl  rX  rY  rr  r_  s                   @r9   meta__fused_adamrs    ss    & E8[/;O 
q$G	

: 	!$( r;   c                     t        j                   j                         dk(  d        t        j                  j                         dk(  d        t        j                   j                  t         j                  t         j
                  fv  fd       t        j                  j                  t         j                  u fd       t        j                   j                  d      j                  d      k(   fd        j                   j                  d      j                  d      ft         j                  	      S )
Nr  c                       y)Nza must be a 2D tensorr5   r5   r;   r9   rf   zmeta__int_mm.<locals>.<lambda>  rn   r;   c                       y)Nzb must be a 2D tensorr5   r5   r;   r9   rf   zmeta__int_mm.<locals>.<lambda>  rn   r;   c                  "    d j                    S )Nz'expected self to be int8 or uint8, got r   )r7   s   r9   rf   zmeta__int_mm.<locals>.<lambda>  s    9!''C r;   c                  "    d j                    S )Nzexpected mat2 to be int8, got r   )r8   s   r9   rf   zmeta__int_mm.<locals>.<lambda>  s    0	: r;   r4   r   c            
          d j                  d       d j                  d       dj                  d       dj                  d       d	S )Nz'Incompatible matrix sizes for _int_mm (r   r>   r4   r   rz   r   r6   s   r9   rf   zmeta__int_mm.<locals>.<lambda>  sM    5affQi[!&&) M66!9+Qqvvayk, r;   r   )	rW   rh   r   r_   r  r<  r   r   r  r6   s   ``r9   meta__int_mmrz    s     
LLA>?	LLA>?	LL	EJJ,,C 
LL	5::: 
LL	q	QVVAY	
 ;;q	166!9-U[[;AAr;   c                 f    t        j                   j                         dk(  d        t        j                   j                  t         j                  u  fd        j                  d      } j                  d      dz  } j                  |dz  ||dz  z  d|dz  ft         j                  	      S )
Nr  c                       yNzw must be a 2D tensorr5   r5   r;   r9   rf   z2meta__convert_weight_to_int4pack.<locals>.<lambda>  rn   r;   c                  "    d j                    S rs  r   ro  s   r9   rf   z2meta__convert_weight_to_int4pack.<locals>.<lambda>      .qwwi8 r;   r   r4             r   )rW   rh   r   r_   r<  r   r   r  r?  inner_k_tilesr+  r  s   `   r9    meta__convert_weight_to_int4packr    s    	LLA>?	LL	5;;8 	
q	A	q	AA;;F-"$%Q		
 kk   r;   c                 J    t        j                   j                         dk(  d        t        j                   j                  t         j                  u  fd        j                  d      } j                  d      } j                  ||dz  ft         j                        S )Nr  c                       yr}  r5   r5   r;   r9   rf   z:meta__convert_weight_to_int4pack_for_cpu.<locals>.<lambda>  rn   r;   c                  "    d j                    S Nzexpected w to be int32, got r   ro  s   r9   rf   z:meta__convert_weight_to_int4pack_for_cpu.<locals>.<lambda>  r  r;   r   r4   r   )rW   rh   r   r_   r  r   r   r<  r  s   `   r9   (meta__convert_weight_to_int4pack_for_cpur    s    	LLA>?	LL	5;;8 	
q	A	q	A;;	
AFkk   r;   c                     t        j                   j                         dk(  d        j                  j                  dk(  rdndt        j                  j                         k(  fd       t        j                   j
                  t         j                  t         j                  t         j                  fv  fd       t        j                  j
                  t         j                  u fd       j                  j                  dk(  rj                  d      nj                  d      d	z  } j                   j                  d      | j
                  
      S )Nr  c                       yNzx must be a 2D tensorr5   r5   r;   r9   rf   z*meta__weight_int4pack_mm.<locals>.<lambda>  rn   r;   r  rX  c                      d  dS )Nzw must be a zD tensorr5   )expected_dims   r9   rf   z*meta__weight_int4pack_mm.<locals>.<lambda>  s    Lh2W r;   c                  "    d j                    S rq  r   rl  s   r9   rf   z*meta__weight_int4pack_mm.<locals>.<lambda>      5aggY? r;   c                  "    d j                    S r  r   ro  s   r9   rf   z*meta__weight_int4pack_mm.<locals>.<lambda>  r  r;   r   r  r   )rW   rh   r   r  r{   r_   r  r  r  r  r   r   )r>   r?  ru  rw  dim_nr  s   ``   @r9   meta__weight_int4pack_mmr    s    	LLA>?**e31L	LLL(*WX	LL	EMM5==%..AA? 
LL	5;;8 ++u4AFF1I!&&)a-E;;qvvay%qww;77r;   c                 (    t        j                   j                         dk(  d        t        j                  j                         dk(  d        t        j                   j                  t         j                  t         j
                  t         j                  fv  fd       t        j                  j                  t         j                  u fd        j                   j                  d      j                  d       j                        S )Nr  c                       yr  r5   r5   r;   r9   rf   z2meta__weight_int4pack_mm_for_cpu.<locals>.<lambda>  rn   r;   c                       yr}  r5   r5   r;   r9   rf   z2meta__weight_int4pack_mm_for_cpu.<locals>.<lambda>  rn   r;   c                  "    d j                    S rq  r   rl  s   r9   rf   z2meta__weight_int4pack_mm_for_cpu.<locals>.<lambda>  r  r;   c                  "    d j                    S rs  r   ro  s   r9   rf   z2meta__weight_int4pack_mm_for_cpu.<locals>.<lambda>  r  r;   r   r   )
rW   rh   r   r_   r  r  r  r<  r   r   rx  s   ``  r9    meta__weight_int4pack_mm_for_cpur        	LLA>?	LLA>?	LL	EMM5==%..AA? 
LL	5;;8 ;;qvvay!&&)177;;;r;   c                 (    t        j                   j                         dk(  d        t        j                  j                         dk(  d        t        j                   j                  t         j                  t         j
                  t         j                  fv  fd       t        j                  j                  t         j                  u fd        j                   j                  d      j                  d       j                        S )Nr  c                       yr  r5   r5   r;   r9   rf   z;_weight_int4pack_mm_with_scales_and_zeros.<locals>.<lambda>  rn   r;   c                       yr}  r5   r5   r;   r9   rf   z;_weight_int4pack_mm_with_scales_and_zeros.<locals>.<lambda>  rn   r;   c                  "    d j                    S rq  r   rl  s   r9   rf   z;_weight_int4pack_mm_with_scales_and_zeros.<locals>.<lambda>  r  r;   c                  "    d j                    S r  r   ro  s   r9   rf   z;_weight_int4pack_mm_with_scales_and_zeros.<locals>.<lambda>  r  r;   r   r   )
rW   rh   r   r_   r  r  r  r  r   r   )r>   r?  ru  qScaleqZeross   ``   r9   )_weight_int4pack_mm_with_scales_and_zerosr    r  r;   r7   r8   c                     | |z   dz
  |z  |z  S r3   r5   r6   s     r9   kai_roundupr    s    UQY1!!r;   c                   	
 | dk(  ry||k(  r(d}d}d}dddd fdfd} ||||||      S |dz  d	k(  rC||z  d	k(  r:d}d}d}dddd		fd
}	
fdd 
	fd	fd |||||||      S y y y )NrX  r  r  r  c                 8    t        ||z  d      }t        | |      S )NrX  r  )r  krsrkr_sr_roundedup4s       r9   kai_k_roundedupz3get_kai_packed_weight_size.<locals>.kai_k_roundedup/  s#     $/rBw#: "1&677r;   c                 f     | ||      }|dz  dk7  rt        d|       ||dz  z   z   z   z  S )Nr  r   zk_internal must be even, got rI  )	r  nrr  r  
k_internalr  kai_num_bytes_biaskai_num_bytes_multiplier_rhskai_num_bytes_sum_rhss	        r9   9kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0z]get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4cxp_qsu4cxs1s05  s`     -QB7
Nq((+H)UVV1_23+, )) r;   c                 >    t        | |      |z  }| ||||      z  S rE   r  )r+  r  r  r  r  num_rowsr  s         r9   7kai_get_rhs_packed_size_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0z[get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_size_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0D  s6     'q"-3 O2r2r;   r  r   c                     ||z  dk7  rt        d| d| d      |	z  dk7  rt        d| d	 d      |z  dk7  rt        d| d d      t        | |      |z  }| |||||      z  S Nr   bl (z) must be divisible by kr (rz   znr (z+) must be divisible by kai_nr_multiple_of (+) must be divisible by kai_bl_multiple_of (r   r  )
r+  r  r  r  r  blr  kai_bl_multiple_of;kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0kai_nr_multiple_ofs
          r9   9kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0z]get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0\  s     G>(4t3NrdRS)TUU++1(rd"MN`Maabc  ++1(rd"MN`Maabc  'q"-3 Q2r2rr;   c                     ||z  dk7  rt        d| d| d      |
z  dk7  rt        d| d
 d      |z  dk7  rt        d| d d       	       } | |      } ||      }|||z  z   z   z  S r  rI  )r  r  r  r  r  num_bytes_multiplier_rhsnum_blocks_per_rownum_bytes_per_blockr  #kai_get_bf16_datatype_size_in_bytesr  kai_num_blocks_per_rowr  kai_num_bytes_per_blockr  s           r9   r  z_get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0s  s     G>(4t3NrdRS)TUU++1(rd"MN`Maabc  ++1(rd"MN`Maabc 
 ,O+P(%;Ar%B"&=0'# (+==+,() r;   c                       y)Nr  r5   r5   r;   r9   r  zGget_kai_packed_weight_size.<locals>.kai_get_bf16_datatype_size_in_bytes  s    r;   c                 V    |z  dk7  rt        d| d d      t        | |      |z  S )Nr   r  r  rz   r  )r  r  r  s     r9   r  z:get_kai_packed_weight_size.<locals>.kai_num_blocks_per_row  sH    ++1(rd"MN`Maabc  #1b)R//r;   c                 H    | z  dk7  rt        d|  d d      | dz  |z   S )Nr   r  r  rz   r  rI  )r  r  r  s     r9   r  z;get_kai_packed_weight_size.<locals>.kai_num_bytes_per_block  sF    ++1(rd"MN`Maabc  a#;;;r;   r5   )n_bitsr  K	groupsizekai_nrkai_krkai_srr  r  r  r  r  r  r  r  r  r  r  r  r  s            @@@@@@@@@@@r9   get_kai_packed_weight_sizer  #  s    {>FFF$%!+,(!"8
 K1fff  ^q Q]a%7FFF$%!!"!"!#. :0< M1fffi ] &8 _ r;   c                     t        j                   j                  t         j                  u  fd       t         j                  j
                  j                         r||k(  r|j                  t         j                  k(  s2||k  re|dz  dk(  r]||z  dk(  rU|j                  t         j                  k(  r8t        d|||      } j                  t        |      t         j                        S  j                         |j                         z   }|||j                         z  } j                  |t         j                        S )Nc                  "    d j                    S rs  r   )weightss   r9   rf   z2meta__dyn_quant_pack_4bit_weight.<locals>.<lambda>  s    .w}}o> r;   r  r   rX  r   )rW   rh   r_   r<  r  kleidiair  r[   r  r  r   r  r   )r  scales_zerosr  
block_sizein_featuresout_featurespacked_weight_sizes   `      r9    meta__dyn_quant_pack_4bit_weightr    s    
LL$> ~~++-	{	"|'9'9U[['H$R1$j(A-""enn4 8|[*
   %7!8 LL <+=+=+??djjl*/u{{CCr;   c                 n    t        j                   j                         dk(  d        t        j                   j                  t         j                  k(  xs$  j                  t         j
                  k(  xr k(   fd        j                  d      } j                  || j                        S )Nr  c                       y)Nzinput must be a 2D tensorr5   r5   r;   r9   rf   z-meta__dyn_quant_matmul_4bit.<locals>.<lambda>  rn   r;   c                  .    dj                    d  d S )NzPexpected input to be f32 or bf16 (bf16 requires block_size == in_features), got z with block_size=z and in_features=r   )r  r  inps   r9   rf   z-meta__dyn_quant_matmul_4bit.<locals>.<lambda>  s*    99+.zl:KK=Z r;   r   r   )rW   rh   r   r_   r  r  r   r   )r  packed_weightsr  r  r  r  s   ` ``  r9   meta__dyn_quant_matmul_4bitr    s     
LLa!DE	LL	emm	# 	GII'EJ+,E	
 	A==L		=::r;   c                 @    t        j                   j                         dk(  d        t        j                   j                  t         j                  t         j
                  t         j                  fv  fd       t        j                  j                         dk(  d        t        j                  j                  t         j                  u fd       t        j                  j                  d       j                  d      k(   fd       t        j                  j                         dk(  xr# j                  d      j                  d      k(  fd	        j                   j                  d      j                  d       j                  
      S )Nr  c                       yr  r5   r5   r;   r9   rf   z*meta__weight_int8pack_mm.<locals>.<lambda>  rn   r;   c                  "    d j                    S rq  r   rl  s   r9   rf   z*meta__weight_int8pack_mm.<locals>.<lambda>  r  r;   c                       yr}  r5   r5   r;   r9   rf   z*meta__weight_int8pack_mm.<locals>.<lambda>  rn   r;   c                  "    d j                    S )Nzexpected w to be int8, got r   ro  s   r9   rf   z*meta__weight_int8pack_mm.<locals>.<lambda>  s    -aggY7 r;   r4   c                  R    d j                  d       dj                  d       dS )Nzexpected w.size(1) (r4   z) == x.size(1) (rz   r   )r?  r>   s   r9   rf   z*meta__weight_int8pack_mm.<locals>.<lambda>  s(    &qvvayk1A!&&)AN r;   r   c                  F    dj                  d       d j                   S )Nz%expected q_scales to be 1D with size r   z, got shape )r   r   )q_scalesr?  s   r9   rf   z*meta__weight_int8pack_mm.<locals>.<lambda>  s"    7q	{,x~~N^_ r;   r   )
rW   rh   r   r_   r  r  r  r  r   r   )r>   r?  r  s   ```r9   meta__weight_int8pack_mmr    s   	LLA>?	LL	EMM5==%..AA? 
LLA>?	LL	5::7 
LL	q	QVVAYN 
LL!=a 0AFF1I =_ ;;qvvay!&&)177;;;r;   c                 n    t        j                   j                         dk\   fd       t        j                  j                         dk\  fd       t        j                   j                  d      j                  d      k(   fd       t        j                  t	        j
                   j                         fd       t        j                  t	        j
                  j                        fd       t        j                  |dk\  d	        t        j                  d
v fd        j                  d      }j                  d      } j                  d d }j                  d d }t        t        j                  ||            }|j                  ||g        j                  |      S )Nr  c                  ,    d j                          dS )Nz1cdist only supports at least 2D tensors, X1 got: r  r   x1s   r9   rf   z$meta_cdist_forward.<locals>.<lambda>      CBFFH:QO r;   c                  ,    d j                          dS )Nz1cdist only supports at least 2D tensors, X2 got: r  r   x2s   r9   rf   z$meta_cdist_forward.<locals>.<lambda>  r  r;   r   c                  P    d j                  d       dj                  d       S )Nz4X1 and X2 must have the same number of columns. X1: r   z X2: r   )r  r  s   r9   rf   z$meta_cdist_forward.<locals>.<lambda>  s,    Frwwr{mSXY[Y`Y`acYdXef r;   c                  "    d j                    S )Nz3cdist only supports floating-point dtypes, X1 got: r   r  s   r9   rf   z$meta_cdist_forward.<locals>.<lambda>      EbhhZP r;   c                  "    d j                    S )Nz3cdist only supports floating-point dtypes, X2 got: r   r  s   r9   rf   z$meta_cdist_forward.<locals>.<lambda>  r  r;   r   c                       y)Nz)cdist only supports non-negative p valuesr5   r5   r;   r9   rf   z$meta_cdist_forward.<locals>.<lambda>  rn   r;   )Nr   r4   r  c                      d  S )Nz(possible modes: None, 0, 1, 2, but was: r5   )compute_modes   r9   rf   z$meta_cdist_forward.<locals>.<lambda>  s    :<.I r;   r  )rW   rh   r   r   rP   is_float_dtyper_   r   r   broadcast_shapesextendr   )	r  r  r  r  r1r2batch_tensor1batch_tensor2r  s	   `` `     r9   meta_cdist_forwardr    sJ   	LL
AO 
LL
AO 
LL
rwwr{"f 
LLRXX&P 
LLRXX&P 
LLaLM	LL'I 
B	BHHSbMMHHSbMM..}mLMLR!<<%%r;   c                 4   |j                   d   }|j                   d   }|j                   d   }|j                   d d }|j                   d d }	t        t        j                  ||	            }
|
j	                         }|j                  ||g       t        j                  |
      }|dk(  s|dk(  s
|dk(  s|dk(  rt        j                  |      S |t        |j                         k7  r|j                  |      }t        j                  |t        j                        S )Nr   r  r   r   )r   r   rW   r  copyr  mathprod
zeros_likery  r   r   )r  r  r  r  cdistc1r  r  r  r   r  tensor1_expand_sizebatch_products                r9   meta_cdist_backwardr    s     
"B	"B	"BHHSbMMHHSbMM 6 6}m TU.335Bx(II23M	Qw"'R1W(:##d288n,YY*+Be.E.EFFr;   c	                     t        j                  j                  t         j                  t         j                  fv fd       t        j                  j                  t         j                  t         j                  fv fd       t        j                  t        j                   j                         fd       j                  d      }	|rt        j                  |	dk\  d        |	dz  }	 j                  |	 j                  d            }
}t        j                  |t        k(  d        t        j                  j                  dk(  fd       t        j                  j                         j                         k(  fd	       fd
d fd}t              dk7  r|j                  j                  d            }j                  j                               }|t        k(  r"j                  |	 j                  d            }nj                  d      }n | |
|      }|t        t        fv s|s!j                  j                  d            }nj                  d      }j                  |	      }j                  d   }|t        k(  rA|rt        j                  |dk\  d        |dz  }j                  | j                  d         }nj                  |j                               }|
|||fS )Nc                  "    d j                    S )Nz(expected indices to be long or int, got r   r   s   r9   rf   z$meta_embedding_bag.<locals>.<lambda>6      :7==/J r;   c                  "    d j                    S )Nz(expected offsets to be long or int, got r   )r  s   r9   rf   z$meta_embedding_bag.<locals>.<lambda>:  r  r;   c                  "    d j                    S )Nz/expected weight to be floating point type, got r   )r  s   r9   rf   z$meta_embedding_bag.<locals>.<lambda>>  s    A&,,P r;   r   r4   c                       yNz1include_last_offset: numBags should be at least 1r5   r5   r;   r9   rf   z$meta_embedding_bag.<locals>.<lambda>E  rn   r;   c                       y)Nz@embedding_bag: per_sample_weights only supported with mode='sum'r5   r5   r;   r9   rf   z$meta_embedding_bag.<locals>.<lambda>N  rn   r;   c                  $    d j                    dS )Nz1expected per_sample_weights to be 1D tensor, got r  rT  )per_sample_weightss   r9   rf   z$meta_embedding_bag.<locals>.<lambda>R  s    GHZH_H_G``ab r;   c                  N    dj                          d j                          dS )Nz%expected per_sample_weights.numel() (z$ to be the same as indices.numel() (rz   r   )r   r  s   r9   rf   z$meta_embedding_bag.<locals>.<lambda>V  s4    78J8P8P8R7S T66=mmo5FaI r;   c                 D     | ||      xr |j                  d      dk(  S Nr   r4   r   )r   r  r  padding_idxis_fast_path_index_selects       r9   is_fast_path_index_select_scalez;meta_embedding_bag.<locals>.is_fast_path_index_select_scale\  s(    %c6;?XELLQROWXDX	
r;   c                     | j                   t        j                  k(  xs | j                   t        j                  k(  xr1 | j	                  d      dk(  xr |j	                  d      dk(  xr |dk  S Nr4   r   )r_   rW   r[   rY   r   )r   r  r  s      r9   r  z5meta_embedding_bag.<locals>.is_fast_path_index_selecta  sb    YY%++%@ejj)@  

1" a A%  a		
r;   c                 2    | | |||      S  | ||      S rE   r5   )r   r  r  r  r  r  s       r9   is_fast_pathz(meta_embedding_bag.<locals>.is_fast_pathi  s)    23v{SS,S&+FFr;   r  c                       yr  r5   r5   r;   r9   rf   z$meta_embedding_bag.<locals>.<lambda>  rn   r;   )rW   rh   r_   r   r  rP   r  r   r   MODE_SUMr   r   r  MODE_MAX	MODE_MEANr   )r  r   r  scale_grad_by_freqr  sparser  include_last_offsetr  num_bagsr  r"  
offset2bagbag_sizemax_indicesfast_path_sumnumBagsr  r  s   ```   `          @@r9   meta_embedding_bagr0  (  s}    
LL%**eii00J 
LL%**eii00J 
LLV\\*P
 ||AHMG	
 	AhA7F%HV	
 	##q(b	
 	$$&'--/9	



G 7u$&&w||A7
$$W\\^48!++Hfkk!nEK!++A.K$V-?UIx(( **7<<?;J **1-J$$X.--"8"qLO 1!++GV\\!_EK!++HMMO<K:x44r;   c                     t        | ||g| \  }}}}t        |      dk(  r|j                  |j                               }||||fS Nr  )r0  r  r   r   )r  r   r  rR   r  r+  r,  r-  s           r9   meta_embedding_bag_forward_onlyr3    sX    0B1#'1-FJ+ 7u$$$W\\^4:x44r;   c                     |r|S | j                   j                  s| j                   j                  r| j                   S |rt        j                  S | j                   S rE   )r_   r!  r  rW   r   )r   r_   promote_int_to_longs      r9   _get_reduction_dtyper6    sD    {{$$(>(>{{	zz;;r;   r   c                    t        | |d      }t        j                  | j                  |      }t	        | ||      }| j                  ||      S )NT)r5  r   )r6  rP   r  r   r  r   )r   r  r  r_   rD  r  s         r9   meta_nansumr8    sI     (u$OLT2D+E4AL??<|?<<r;   c           	          t        j                  | j                  t        t	        | j                                           }| j                  |      S rE   )rP   r  r   rg   r   r   r   )r   r  s     r9   meta_medianr:    s<    77U5-.L ??<((r;   c                    t        |       dk(  rt        j                  d       t        j                  | j                  |f      }t        | ||      }| j                  |      | j                  |t        j                        fS )Nr	  zmedian CUDA with indices outputr   )	r  rP   alert_not_deterministicr  r   r  r   rW   r   )r   r   r  r  s       r9   meta_median_mode_dimr=    sp     5V#%%&GH


u{{SF
3C+E3@L%EJJ7 r;   c                     | S rE   r5   r   s    r9   meta_logical_not_r?    r  r;   c                    t        j                  t        |      | j                         k\  d        t	        |      D ]"  \  t        j                  dk\  fd       $ t        |      | j                         z
  }d|z  t        | j                        z   }t        t        |            D cg c]  }||   ||   z   }}| j                  |      S c c}w )Nc                       y)NzZNumber of dimensions of repeat dims can not be smaller than number of dimensions of tensorr5   r5   r;   r9   rf   zmeta_repeat.<locals>.<lambda>  rn   r;   r   c                      d d  S )Nz"Repeats cannot be negative, found r&  r5   )r   reps   r9   rf   zmeta_repeat.<locals>.<lambda>  s    8ZsK r;   r_  )	rW   rh   r   r   r/  rg   r   r   r   )r   r  num_new_dimensionspadded_sizer   target_sizerC  s       ` @r9   meta_repeatrG    s    	LLG
"l G$ 
31HK	

 W
2++eDJJ.??K8=c'l8KL1;q>GAJ.LKL>>+&& Ms   1Cc                     | S rE   r5   r   s    r9   
meta_zero_rI    r  r;   c                 z    t        |t        j                        r t        | j                  |j                         | S rE   )rq   rW   r   rj   r   r   r   s     r9   meta_binop_inplacerL    s)     %&

EKK8Kr;   c                 .    t         |t        j                        \  }t        j                  t        j                   j                         fd       t        |t        j                        r t         j                  |j                          S )Nr  )	cast_fromcast_toc                  (    d  dj                    S )Nzresult type z* can't be cast to the desired output type r   )rT   r   s   r9   rf   z"meta_div_inplace.<locals>.<lambda>   s"    <. )::,( r;   )r   r   r  rW   rh   rP   can_safe_cast_tor_   rq   r   rj   r   )r   r   rS   rT   s   `  @r9   meta_div_inplacerR    sn    (;HHOA|
 
LLtzzJ	
 %&

EKK8Kr;   c                     d }d }d } ||       r ||      rt        d       ||       r ||      st        d      t        |t        j                        r t	        | j
                  |j
                         | S )a*  
    Some checks for inplace ops.
    Checks for promotion rules for some dtypes.
    int.add/sub_(float) and bool.add/sub_(others) are rejected.
    Promoting in these in-place operations would require reallocating
    and copying over elements, hence not allowed.
    Checks for alpha param.
    c                     t        | t              rt        j                  | j                        S t        | t
              S rE   )rq   r    rP   r  r_   r   rt   s    r9   is_integericz.meta_binop_inplace_alpha.<locals>.is_integeric  s.    c:&))#))44c7++r;   c                     t        | t              rt        j                  | j                        S t        | t
              S rE   )rq   r    rP   r  r_   r   rU  s    r9   
is_floaticz,meta_binop_inplace_alpha.<locals>.is_floatic#  s.    c:&''		22c9--r;   c                     t        | t              rt        j                  | j                        S t        | t
              S rE   )rq   r    rP   is_boolean_dtyper_   r   rU  s    r9   is_booleanicz.meta_binop_inplace_alpha.<locals>.is_booleanic)  s.    c:&))#))44c8,,r;   z]Promotion of int.add/sub_(float) in in-place ops are not possible due to element size change.z_Promotion of bool.add/sub_(others) in in-place ops are not possible due to element size change.)rv  rq   rW   r   rj   r   )r   r   r  rV  rX  r[  s         r9   meta_binop_inplace_alphar\    sz    $,.- Dj/k
 	

 D,u"5m
 	
 %&

EKK8Kr;   c                 :    t        | |t        j                        S Nr  rU   r   rQ   r   r   r  s      r9   meta_binop_alphara  @  s     e$C$K$K r;   c                 8    t        | t        j                        S r^  r_  )r   rV  s     r9   
meta_roundrc  L  s    <DD r;   c                 l    t        j                  t        j                  j                         fd       t        t         j                        r8t        j                  t        j                  j                         fd       y t        j                  t        t               fd       y )Nc                  &      dj                    S )Nz7: Expected input tensor to have an integral dtype. Got r   )r-  r   s   r9   rf   z#shift_dtype_check.<locals>.<lambda>V  s    7)RSWS]S]R^_ r;   c                  &      dj                    S )Nz6: Expected shift value to have an integral dtype. Got r   r-  r  s   r9   rf   z#shift_dtype_check.<locals>.<lambda>[  s    wiUVYV_V_U`a r;   c                        d S )Nz): Expected shift value to be an int. Got r5   rg  s   r9   rf   z#shift_dtype_check.<locals>.<lambda>`  s    wiHN r;   )rW   rh   rP   r  r_   rq   r   r   )r-  r   r  s   ```r9   shift_dtype_checkri  S  sp    	LLtzz*_ #u||$""399-a	

 	sG$N	
r;   c                 T    t        d| |       t        | |t        j                        S )Nrshiftr  ri  rU   r   rQ   rK  s     r9   meta_rshiftsrm  d  )    he,e$C$K$K r;   c                 T    t        d| |       t        | |t        j                        S )Nlshiftr  rl  rK  s     r9   meta_lshiftsrq  l  rn  r;   c                 ,    t        j                  |       S rE   r   r   s    r9   	meta_zerors  t      D!!r;   c                     t        t        j                        r*t        j                  j	                         dk(  fd       | S )Nr   c                  ,    d j                          dS NzAfill_ only supports 0-dimension value tensor but got tensor with  dimensions.r   r  s   r9   rf   zmeta_fill_.<locals>.<lambda>~      WX[X_X_XaWbbno r;   )rq   rW   r   rh   r   r   r  s    `r9   
meta_fill_r{  y  s4    #u||$GGINo	
 Kr;   c                     t        t        j                        r*t        j                  j	                         dk(  fd       t        j
                  |       S )Nr   c                  ,    d j                          dS rw  r   r  s   r9   rf   zmeta_fill.<locals>.<lambda>  ry  r;   )rq   rW   r   rh   r   r   rz  s    `r9   	meta_fillr~    s@    #u||$GGINo	
 D!!r;   c                     | S rE   r5   r   s    r9   
meta_relu_r    r  r;   c                 :    t        | |t        j                        S r^  r_  r`  s      r9   meta__add_relur    s     e$C$K$K r;   c                 ,    t        j                  |       S rE   r   r   noiselowerr7  r  r   s         r9   meta_rrelu_with_noiser    s    
 D!!r;   c                 V    t        j                  |       t        j                  |      fS rE   r   r  s         r9    meta_rrelu_with_noise_functionalr    s%     D!5#3#3E#:::r;   c                     | S rE   r5   )r   r  r7  r  r   s        r9   meta_rrelu_with_noise_r    s	     Kr;   c                 ,    t        j                  |       S rE   r   r   r   r   
accumulates       r9   meta_index_putr    rt  r;   c                 F    t        | j                  |j                         | S rE   rj   r   )r   r   values      r9   meta_masked_fill_r    s    DJJ

3Kr;   c                     | j                  | j                               j                  t        j                  |             }|S r   )r   r   rw  rP   r   )r   r   r  masked_scales       r9   meta__masked_scaler    s<    >>$))+.1111$7 2 L r;   c                      t        j                  |j                  t         j                  t         j                  fv d        t        j                   j                  j                  k(   fd        S )Nc                       y)NzMask must be bool or uint8r5   r5   r;   r9   rf   z&meta_masked_scatter_.<locals>.<lambda>  rn   r;   c                  <    d j                    dj                    S )NzEmasked_scatter: expected self and source to have same dtypes but got r   r   )r   r  s   r9   rf   z&meta_masked_scatter_.<locals>.<lambda>  s"     **U6<<.: r;   )rW   rh   r_   rC  r<  )r   r   r  s   ` `r9   meta_masked_scatter_r    sU    	LL

uzz5;;//1U 
LL

fll"	:
 Kr;   c                     t        | |      \  } }t        j                  | t        j                        }t	        |||      S r   )r'   rW   r   r   r  )r   r   r  r  s       r9   meta_masked_scatterr    s;     "$-JD$d%2I2IJFf55r;   c                 $    | j                  |      S rE   r  )r   r   r  s      r9   meta_masked_scatter_backwardr    s    >>%  r;   c                     | S rE   r5   r  s       r9   meta_index_put_r    r  r;   c           	          ddl m}m} t        j                   j                         dk(  d        t        j                  j                         dk(  d        t        j                   j                  j                  k(   fd        j                         }j                         |d   |d   |d   }d   }	||	ft        j                   | |d          |d               fd	       |r j                  t        j                  k(  xs  j                  t        j                  k(  xr |t        j                  k(  }
t        j                  | j                  k(  xs |
d
        j                        j                  |      }nj                        }|sYWt        j                  j                         dk(  d        t        j                   |j                               fd       |S )Nr   )r  r  r1   c                       yr  r5   r5   r;   r9   rf   z)common_meta_baddbmm_bmm.<locals>.<lambda>  rn   r;   c                       yr  r5   r5   r;   r9   rf   z)common_meta_baddbmm_bmm.<locals>.<lambda>  rn   r;   c                  <    d j                    dj                    S )Nzexpected scalar type z but found r   rQ  s   r9   rf   z)common_meta_baddbmm_bmm.<locals>.<lambda>  s    '~[O r;   r  r4   c            	      .    d d d d    d d    d	S r  r5   r  s   r9   rf   z)common_meta_baddbmm_bmm.<locals>.<lambda>  s5    RSURV
l<?*;2l1o=NbR r;   c                       y)Nzfout_dtype only supported for torch.float32 output with float16/bfloat16 inputs or same as input dtypesr5   r5   r;   r9   rf   z)common_meta_baddbmm_bmm.<locals>.<lambda>   rn   r;   c                       y)Nzself must be a 3D tensorr5   r5   r;   r9   rf   z)common_meta_baddbmm_bmm.<locals>.<lambda>  rn   r;   c                  0    d  dj                          S )Nz*Expected an input tensor shape with shape z but got shape: r   )r  self_baddbmms   r9   rf   z)common_meta_baddbmm_bmm.<locals>.<lambda>  s!    @M]^j^o^o^q]rs r;   )rt  r  r  rW   rh   r   r_   r   r  r  r  r   rw  )r  r  is_bmmr  r  r  r  r  res_rowsres_colssupported_out_dtyper  r  r  r  r  s   `` `        @@@@r9   common_meta_baddbmm_bmmr    s   E	LL"$HI	LL"$HI	LL$O
 ;;=L;;=L	aB#AAHAHx*K	LL|A+VLOEU-VW	R
 LLEMM)KV\\U^^-K)5==( 	 	%<)<|	
 !!+.11)< !!+.l.\%%'1,.PQ<$$&4s	

 Mr;   c                     t        | |d      S )NTr  )r   r  s     r9   meta_bmmr    s     #4t44r;   c                      t        | |d|      S )NT)r  r  )r   r  r  s      r9   meta_bmm_dtyper    s     #4tyIIr;   c                     | |z  S rE   r5   r=   s     r9   div_rtnr    s     6Mr;   c                     t        | |z   |z   ||dz
  z  z
  dz
  |r|dz
  ndz   |      dz   }|r|dz
  |z  | |z   k\  r|dz  }|S r   )r  )	inputSize
kernelSizer+  r,  r   r  re  
outputSizes           r9   pooling_output_shape_pad_lrr  #  s     	 *q.)* 	
 'vzA/ 	
 		  Nf$	E(99!OJr;   c           	          t        j                  |dk7  d        t        j                  dk\  fd       t        j                  dz
  z  dz   dz  k  fd       t        | ||      S )Nr   c                       y)Nzstride should not be zeror5   r5   r;   r9   rf   z&pooling_output_shape.<locals>.<lambda>?  rn   r;   c                      d  S )Nz'pad must be non-negative, but got pad: r5   pads   r9   rf   z&pooling_output_shape.<locals>.<lambda>@  s    %LSE#R r;   r4   r  c                      d d d  S )NzApad should be at most half of effective kernel size, but got pad=z, kernel_size=z and dilation=r5   )r  r  r  s   r9   rf   z&pooling_output_shape.<locals>.<lambda>C  s'    OPSu U%,nXJ@ r;   )rW   rh   r  )r  r  r  r   r  re  s    `` ` r9   r  r  >  ss    	LL1AB	LLRS	LLa8+a/A55	
 ':sC9 r;   c                    	
 ddl m}m}  j                         }	t	        j
                  dkD  xr dkD  fd       t	        j
                  dkD  xr dkD  fd       t	        j
                  dkD  xr dkD  fd        | j                  d      dk7   j                  d      dk7        }|t        j                  k(  r:t	        j
                  |dk(  xr  || j                  d	      dk7         fd
       n`t	        j
                   ||d	k(  xr  | j                  d      dk7  |      |dk(  xr  || j                  d	      dk7               fd       t	        j
                   |dz  k\  dz  k\        fd       t	        j
                   |dk\  dk\        
	fd       y )Nr   r  c                      d  d S )Nz5kernel size should be greater than zero, but got kH: , kW: r5   )r  r  s   r9   rf   z$pool2d_shape_check.<locals>.<lambda>e  s    Gt6RTQUV r;   c                      d  d S )Nz0stride should be greater than zero, but got dH: , dW: r5   )r  r  s   r9   rf   z$pool2d_shape_check.<locals>.<lambda>i  s    B2$fRDQ r;   c                      d  d S )Nz9dilation should be greater than zero, but got dilationH: , dilationW: r5   )	dilationH	dilationWs   r9   rf   z$pool2d_shape_check.<locals>.<lambda>m  s    KI;Vcdmcno r;   r4   r  rX  r1   c                  *    d j                          S )NzExpected 4D (batch mode) tensor expected for input with channels_last layout with optional 0 dim batch size for input, but got: r   rA  s   r9   rf   z$pool2d_shape_check.<locals>.<lambda>u  s     CCH::<.R r;   c                  *    d j                          S )NzYExpected 3D or 4D (batch mode) tensor with optional 0 dim batch size for input, but got: r   rA  s   r9   rf   z$pool2d_shape_check.<locals>.<lambda>~  s    opupzpzp|o}~ r;   c                       d d d d  S )NzKpad should be smaller than or equal to half of kernel size, but got padW = z	, padH = z, kW = z, kH = r5   )r  r  r  r  s   r9   rf   z$pool2d_shape_check.<locals>.<lambda>  s&     ygbT> r;   c                  .    d d  d d d d dS NzGiven input size: (r>   z). Calculated output size: (z). Output size is too smallr5   )r  r  rf  r  rg  rh  s   r9   rf   z$pool2d_shape_check.<locals>.<lambda>  s:    %k]!K=* N$$0><.+ O## r;   )rt  r  r  r   rW   rh   r   rd  )r   r  r  r  r  r  r  r  r  rf  r  r  rg  rh  r   r  r  r   
valid_dimsr  s   ``````````````     @r9   r  r  M  s   " F99;DL	LL
Q26V 
LL
Q26Q 
LLA')a-o
 A!+UZZ]a-?@J+++AIA'*ejjmq.@AR	
 		Egejjmq&8*E	Egj%**Q-12DE 	
 
LLa4qD1	> 
LLq ,!"34	# 	#r;   r  r  r  r  r  r  r  pTpHpW	dilationTr  r  r  r  r  r  r  r  r  c           
      V   	
  j                   }t        j                  dkD  xr dkD  xr dkD  fd       t        j                  dkD  xr dkD  xr dkD  fd       t        j                  dkD  xr dkD  xr dkD  fd       t        j                  |dv  fd       t        |      D ]:  |dk(  rdk(  rt        j                   j	                        dkD   fd       < dd	lm} |r/t        j                   |k\  k\  k\        fd
       t        j                   |dz  k\  dz  
k\  dz  	k\        	
fd       t        j                   |dk\  dk\  dk\        fd       y )Nr   c                      d d  d S )Nz5kernel size should be greater than zero, but got kT: z, kH: r  r5   )r  r  r  s   r9   rf   z$pool3d_shape_check.<locals>.<lambda>  s#    $fRDrd, r;   c                      d d  d S )Nz0stride should be greater than zero, but got dT: z, dH: r  r5   )r  r  r  s   r9   rf   z$pool3d_shape_check.<locals>.<lambda>  s     >rd&FSURVW r;   c                      d d  d S )Nz9dilation should be greater than zero, but got dilationT: z, dilationH: r  r5   )r  r  r  s   r9   rf   z$pool3d_shape_check.<locals>.<lambda>  s$    #M)M)V r;   r  c                  &      dj                    S )Nz/: Expected 4D or 5D tensor for input, but got: rB  )r-  r   s   r9   rf   z$pool3d_shape_check.<locals>.<lambda>  s    7)J5;;-X r;   r|  c                  L      dj                    dj                         dS )NzZ: Expected input's non-batch dimensions to have positive length, but input has a shape of z and non-batch dimension z has length zero!)r   r   )r-  r   r   s   r9   rf   z$pool3d_shape_check.<locals>.<lambda>  s3    ) --2[[M+EJJqM?:KM r;   r$  c                  .    d d  d d d d dS )Nzinput image (T: r  ra  z ) smaller than kernel size (kT:  kH:  kW: rz   r5   )r  r  r  r  r  r  s   r9   rf   z$pool3d_shape_check.<locals>.<lambda>  s9    "5'gYd6( C$$&4uRDbT< r;   r  c                  ,    d d d  d d d S )NzHpad should be smaller than or equal to half of kernel size, but got kT: r  r  z padT: z padW: z padH: r5   )r  r  r  r  r  r  s   r9   rf   z$pool3d_shape_check.<locals>.<lambda>  s6    $eB4uRDt72$gbTK r;   r4   c                  :    d d d  d d d d d dS r  r5   )r  r  r  r  r  r  r  s   r9   rf   z$pool3d_shape_check.<locals>.<lambda>  sI    !'!E7!G9AfX F((/y%'!F8 L'( r;   )r   rW   rh   r   r   rt  r  )r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r-  r  r   r  r   s   `````````````````````   @r9   r  r    s   0 ::D	LL
Q$26$b1f	
 
LL
Q$26$b1f	
 
LLA9)a-9IM	
 
LLX
 4[ 
19aJJqMA	
	
 >ERKB"= 	
 
LLQ"b1flBFbL9	
 	
 
LL
FaKA6	
 	
r;   c                 j   | j                   }t        | |||||||	|
||||||||||||       t        |||dz
  |       t        |||dz
  |       t        |||dz
  |       t        |||dz
  |       t        |||dz
  |       t        |||dz
  |       t        |||dz
  |       t        |||dz
  |       y )NrX  r1   r  r4   r   r  r~  )r   rM  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r-  r   s                           r9   max_pool3d_backward_shape_checkr    s    2 ::D








+0 ;dQh8;dQh6;dQh8;dQh77D$(G47D$(E27D$(G47D$(F3r;   c                     | j                   }t        | ||||||||	|
|ddd|||||||d       t        |||dz
  |       t        |||dz
  |       t        |||dz
  |       t        |||dz
  |       y )Nr4   TrX  r1   r  r  )r   rM  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r-  r   s                       r9   r  r  +  s    * ::D








			-2 ;dQh8;dQh6;dQh8;dQh7r;   c                    d } |d|      \  }}t        j                  t        |      dv d        t        |      dk(  r||}
}	n |d|      \  }	}
 |d|      \  }} |d|      \  }}| j                  d	      }| j                  d
      }| j                  d      }t	        j
                  |       }|t         j                  k(  r)t        j                  | j                         dk(  d        nR|t         j                  k(  r(t        j                  | j                         dv d        nt        j                  dd        t        ||||	||      }t        ||||
||      }t        | |||	|
||||||||||       |||fS )Nc                      t        j                  t        |      dv  fd       |d   }t        |      dk(  r|n|d   }||fS )Nr  c                      d  dS )Nzmax_pool2d: r  r5   r  s   r9   rf   zEmax_pool2d_checks_and_compute_shape.<locals>.unpack.<locals>.<lambda>m  r  r;   r   r4   r  r  s   `   r9   r  z3max_pool2d_checks_and_compute_shape.<locals>.unpackj  r  r;   r  r  c                       y)NzOmax_pool2d: stride must either be omitted, a single int, or a tuple of two intsr5   r5   r;   r9   rf   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>w  rn   r;   r   r   r  r  r  r  r   rX  c                       y)NzMnon-empty 4D (batch mode) tensor expected for input with channels_last layoutr5   r5   r;   r9   rf   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>  rn   r;   r  c                       y)Nz9non-empty 3D or 4D (batch mode) tensor expected for inputr5   r5   r;   r9   rf   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>  rn   r;   Fc                       y)NzAUnsupported memory format. Supports only ChannelsLast, Contiguousr5   r5   r;   r9   rf   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>  rn   r;   )rW   rh   r   r   rP   r   rd  r   r   r  r  )r   r  r   r  r  re  r  r  r  r  r  r  r  r  r  rf  r  r  r   rg  rh  s                        r9   rc  rc  a  s    M;/FB	LLFy a 6{aRB&)B	7+JD$!*h7Iy**R.K**R.KBJ//6M+++IIK1c	
 
%11	1IIK6!O	

 	W	

 (Rr9iXL&z2tRIVK



$ k11r;   c                 |    t        |||||      \  }t        j                  j                   j                  k(   fd       |j                  fd}	 |	         |	|       t        j                        }
t        j                  j                  j                  j                  |
      S )Nc                  <    dj                    d j                    S )NzExpected dtype z  for `gradOutput` but got dtype r   r  s   r9   rf   z7meta_max_pool2d_with_indices_backward.<locals>.<lambda>  s     /$**-MkN_N_M`a r;   c                 l    t        | dz
         t        | dz
         t        | dz
         y )Nr1   r  r4   )r~  )r  r  r   rg  rh  s    r9   _check_dim_sizez>meta_max_pool2d_with_indices_backward.<locals>._check_dim_size  s9    q$q,7q$q,7q$q+6r;   r  )
rc  rW   rh   r_   r   rP   r   r   r   r   )rM  r   r  r   r  r  re  r   rf  r  r   r  r   rg  rh  s   ``         @@@@r9   %meta_max_pool2d_with_indices_backwardr    s      	,k67Hi		
 
LL

k'''a
 L99D7
 K G//5M;;

jj{{#	 r;   c                    t        | |||||      \  }}}| j                         dk(  r| j                  d      nd}	t        j                  |       }
| j                         dk(  r|||g}n|	|||g}t        j                  || j                  | j                  |
      t        j                  |t
        j                  | j                  |
      fS ra  )
rc  r   r   rP   r   rW   r   r_   r   r   rd  s               r9   meta_max_pool2d_with_indicesr    s     	,{FGXy		
  %yy{a/UZZ^QF//6Myy{a\;7\;?++<<'		
 	++<<'		
 r;   c                 t   
 t        j                   j                  dv  fd        j                  }t        |dz
  |      D ].  
t        j                   j	                  
      dkD  
 fd       0 t        j                  t              dk(  d        t        j                  t        |      dk(  d         j	                  d	      } j	                  d
       j	                  d      |dk(  r j	                  d      }nd}t        j                   j                  j                  k(  d        t        j                  j                  dk(  fd       j	                  d      }j	                  d      }j	                  d      
t        j                  ||k\  d        t        j                  ||k(  d        t        j                  
dk(  
fd       t        j                  |d   d   z   dz
  k  fd       t        j                  |d   d   z   dz
  k  fd        j                         dk(  r|||d   |d   g}	n||d   |d   g}	t        j                  |	 j                   j                        t        j                  |	t         j                   j                        fS )Nr  c                  "    d j                    S )Nz:fractional_max_pool2d: Expected 3D or 4D tensor, but got: rT  r   s   r9   rf   z,meta_fractional_max_pool2d.<locals>.<lambda>  s    LTYYKX r;   r1   r   c                  2    dj                          d  dS )Nz_fractional_max_pool2d: Expected input to have non-zero  size for non-batch dimensions, but got r  z emptyr   )r   r   s   r9   rf   z,meta_fractional_max_pool2d.<locals>.<lambda>  s'     77;yy{mCSTUSVV\^ r;   r  c                       y)NzNfractional_max_pool2d: kernel_size musteither be a single int or tuple of Intsr5   r5   r;   r9   rf   z,meta_fractional_max_pool2d.<locals>.<lambda>  rn   r;   c                       y)NzOfractional_max_pool2d: output_size must either be a single int or tuple of Intsr5   r5   r;   r9   rf   z,meta_fractional_max_pool2d.<locals>.<lambda>  rn   r;   r  r  r   rX  r4   c                       y)Nz6Expect _random_samples to have the same dtype as inputr5   r5   r;   r9   rf   z,meta_fractional_max_pool2d.<locals>.<lambda>&  rn   r;   c                  "    d j                    S )Nz1Expect _random samples to have 3 dimensions got, rT  )random_sampless   r9   rf   z,meta_fractional_max_pool2d.<locals>.<lambda>*  s    CNDWDWCXY r;   c                       y)Nz=Expect _random_samples.size(0) no less then input batch size.r5   r5   r;   r9   rf   z,meta_fractional_max_pool2d.<locals>.<lambda>2  rn   r;   c                       y)Nz<Expect _random_samples.size(1) equals to input channel size.r5   r5   r;   r9   rf   z,meta_fractional_max_pool2d.<locals>.<lambda>6  rn   r;   c                      d  dS )Nz/Expect _random_samples.size(2) equals to 2 got .r5   )r   s   r9   rf   z,meta_fractional_max_pool2d.<locals>.<lambda>8  s    #RSTRUUV!W r;   c                      dd    d  S )Nz%fractional_max_pool2d: kernel height r   z' is too large relative to input height r5   )input_heightr  s   r9   rf   z,meta_fractional_max_pool2d.<locals>.<lambda><  s    7A7GGno{n|} r;   c                      dd    d  S )Nz$fractional_max_pool2d: kernel width r4   z& is too large relative to input width r5   )input_widthr  s   r9   rf   z,meta_fractional_max_pool2d.<locals>.<lambda>@  s    6{1~6FFlmxlyz r;   r  )rW   rh   r   r   r   r   r_   r   r   r   r   )r   r  r  r  r   input_channelsinput_batchr+  cr   r   r  r   s   `` `      @@@r9   meta_fractional_max_pool2dr    ss   	LL		VX 99D4!8T" 
IIaL1^	

 
LLKA	2
 
LLKA	2 YYr]N99R=L))B-Kqyiil	LL

n***H 
LLq Y
 	AAAAAA	LL	[O 
LL	^N 
LLaWX	LLAQ'!+|;} 
LLAQ'!+{:z
 xxzQ^[^[^LAA? 	**;;	

 	++;;	
 r;   c                    t        j                  t        |      dv d        |d   }t        |      dk(  r|n|d   }t        |      dk(  r|n|d   }t        j                  | xs t        |      dv d        |s|n|d   }	|s|nt        |      dk(  r|	n|d   }
|s|nt        |      dk(  r|	n|d   }t        j                  t        |      dv d        |d   }t        |      dk(  r|n|d   }t        |      dk(  r|n|d   }t        j                  t        |      dv d        |d   }t        |      dk(  r|n|d   }t        |      dk(  r|n|d   }t        j                  | j                  d	v d
        | j                  dk(  r| j	                  d      nd}| j	                  d      }| j	                  d      }| j	                  d      }| j	                  d      }t        ||||	||      }t        ||||
||      }t        ||||||      }t        | |||||	|
|||||||||||||d       | j                  dk(  xr& t        j                  |       t         j                  k(  }| j                  dk(  r||||f}n|||||f}| j                  |      }| j                  |t         j                        }|r@|j                  t         j                        }|j                  t         j                        }||fS )Nr  c                       yNzMmax_pool3d: kernel_size must either be a single int, or a tuple of three intsr5   r5   r;   r9   rf   z.meta_max_pool3d_with_indices.<locals>.<lambda>b  rn   r;   r   r4   r  c                       yNzQmax_pool3d: stride must either be omitted, a single int, or a tuple of three intsr5   r5   r;   r9   rf   z.meta_max_pool3d_with_indices.<locals>.<lambda>j  rn   r;   c                       yNzImax_pool3d: padding must either be a single int, or a tuple of three intsr5   r5   r;   r9   rf   z.meta_max_pool3d_with_indices.<locals>.<lambda>r  rn   r;   c                       yNzJmax_pool3d: dilation must be either a single int, or a tuple of three intsr5   r5   r;   r9   rf   z.meta_max_pool3d_with_indices.<locals>.<lambda>z  rn   r;   r  c                       yr  r5   r5   r;   r9   rf   z.meta_max_pool3d_with_indices.<locals>.<lambda>  rn   r;   r|  rb  r  r  r   zmax_pool3d_with_indices()rX  r   r   )rW   rh   r   r   r   r  r  rP   r   r+  r   r   rw  )r   r  r   r  r  re  r  r  r  r  r  r  r  r  r  r  r  r  r3  r  r  r  r  r  r  r  rd  r   r   r   s                                 r9   meta_max_pool3d_with_indicesr  V  s    
LLKF"_ 
QB;1$+a.B;1$+a.B	LL
+c&kV+c vayBc&kQ&6F1IBc&kQ&6F1IB	LLG[ 
B7|q gajB7|q gajB	LLH\ I ]a/	Xa[I ]a/	Xa[I	LL

fK
  %zzQUZZ^AFjjnGJJrNEjjnGZZ^F BIyIE"7BB	9MG!&"b"iKF








#+4 	

aXE77>%BXBXX  zzQeWf5	WeWf=	
//)
$Cooiu{{o;Gff5#9#9f:**5+A+A*B<r;   c                    t        j                  t        |      dv d        |d   }t        |      dk(  r|n|d   }	t        |      dk(  r|n|d   }
t        j                  | xs t        |      dv d        |s|n|d   }|s|	nt        |      dk(  r|n|d   }|s|
nt        |      dk(  r|n|d   }t        j                  t        |      dv d        |d   }t        |      dk(  r|n|d   }t        |      dk(  r|n|d   }t        j                  t        |      dv d        |d   }t        |      dk(  r|n|d   }t        |      dk(  r|n|d   }t        j                  |j                  d	v d
        |j	                  d      }|j	                  d      }|j	                  d      }|j	                  d      }| j	                  d      }| j	                  d      }| j	                  d      }t        || ||||	|
|||||||||||||||d       |j                  dk(  xr& t        j                  |      t         j                  k(  }|j                  |j                        }|r |j                  t         j                        }|S )Nr  c                       yr  r5   r5   r;   r9   rf   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>  rn   r;   r   r4   r  c                       yr	  r5   r5   r;   r9   rf   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>  rn   r;   c                       yr  r5   r5   r;   r9   rf   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>  rn   r;   c                       yr  r5   r5   r;   r9   rf   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>  rn   r;   r  c                       yr  r5   r5   r;   r9   rf   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>  rn   r;   rb  r  r  r   z"max_pool3d_with_indices_backward()r|  r   )rW   rh   r   r   r   r  rP   r   r+  r   r   rw  )rM  r   r  r   r  r  re  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rd  rP  s                                r9   %meta_max_pool3d_with_indices_backwardr    s    
LLKF"_ 
QB;1$+a.B;1$+a.B	LL
+c&kV+c vayBc&kQ&6F1IBc&kQ&6F1IB	LLG[ 
B7|q gajB7|q gajB	LLH\ I ]a/	Xa[I ]a/	Xa[I	LL

fK
 jjnGJJrNEjjnGZZ^FR Er"Gb!F#








,/8 	

aXE77>%BXBXX  -J]]1G1G]H
r;   gridc                 z    t        j                   j                  j                  k(   fd       t        j                   j                  t         j                  k(  xr j                  t         j                  k(   fd       t        j                   j
                  d   j
                  d   k(   fd       t        j                  j
                  d    j                  dz
  k(   fd       t        d j                        D ],  t        j                   j
                     dkD   fd       . y )	Nc                  <    dj                    d j                    S )NzNgrid_sampler(): expected input and grid to be on same device, but input is on z and grid is on rU  r  r   s   r9   rf   z+check_grid_sampler_common.<locals>.<lambda>  s'    \\N"24;;-A r;   c                  <    dj                    d j                    S )NzTgrid_sampler(): expected input and grid to have torch.strided layout, but input has z and grid has )r   r  s   r9   rf   z+check_grid_sampler_common.<locals>.<lambda>%  s&    nT[[MC r;   r   c                  <    dj                    d j                    S )NzZgrid_sampler(): expected grid and input to have same batch size, but got input with sizes  and grid with sizes rB  r  s   r9   rf   z+check_grid_sampler_common.<locals>.<lambda>,  s'      %},A$**O r;   r   r  c                  B    dj                   dz
   d j                   S )Nz+grid_sampler(): expected grid to have size r  z, in last dimension, but got grid with sizes )r   r   r  s   r9   rf   z+check_grid_sampler_common.<locals>.<lambda>3  s,    9%**q.9I J226**? r;   c                  *    dj                    d  dS )NzYgrid_sampler(): expected input to have non-empty spatial dimensions, but input has sizes r  r  rB  r  s   r9   rf   z+check_grid_sampler_common.<locals>.<lambda><  r  r;   )rW   rh   r   r   r  r   r   r   )r   r  r   s   ``@r9   check_grid_sampler_commonr!    s    	LL#	
 
LL%F$++*F	
 
LLA$**Q-'	
 
LL

2%**q.(	
 1ejj! 
KKNQ	

r;   c                       e Zd ZdZdZdZy)GridSamplerInterpolationr   r4   r  N)r|   
__module____qualname__BILINEARNEARESTBICUBICr5   r;   r9   r#  r#  C  s    HGGr;   r#  interpolation_modec                     t        j                   j                  dk(  xr  j                  j                  k(   fd       t        j                   j                  dk(  xr |t        j                  j
                  k(   d        y )Nr|  c                  <    dj                    d j                    S )Nzdgrid_sampler(): expected 5D input and grid with same number of dimensions, but got input with sizes r  rB  r  s   r9   rf   z'check_grid_sampler_3d.<locals>.<lambda>L  s&    449KK=#DJJ<1 r;   c                       y)Nz<grid_sampler(): bicubic interpolation only supports 4D inputr5   r5   r;   r9   rf   z'check_grid_sampler_3d.<locals>.<lambda>W  rn   r;   )rW   rh   r   r#  r(  r  )r   r  r)  s   `` r9   check_grid_sampler_3dr-  I  sp    	LL

a3EJJ$))3	
 
LLJJ!O M"&>&F&F&L&LL	
 	Or;   c                     |d   }|r&t        j                  |t         j                        }nd }t        j                  |t         j                        }	||	fS Nr   r   )rW   r  r   r   
rM  r   r  r)  padding_modealign_cornersrH  input_requires_gradrP  	grad_grids
             r9   grid_sampler_2d_backward_metar5  [  sQ     &a.%%e5;R;RS

  U5L5LMI	""r;   c                     t        | |       t        | ||       | j                  d   }| j                  d   }|j                  d   }|j                  d   }|j                  d   }	| j                  |||||	f      S )Nr   r4   r  r1   )r!  r-  r   r   )
r   r  r)  r1  r2  r  Cout_Dout_Hout_Ws
             r9   grid_sampler_3dr;  n  sv     eT*%'9:AAAAJJqMEJJqMEJJqME??Aq%677r;   r4  c                     t        ||       t        |||       |d   }|r&t        j                  |t        j                        }nd }t        j
                  |t        j                        }	||	fS r/  )r!  r-  rW   r  r\  r   r0  s
             r9   grid_sampler_3d_backwardr=    sm     eT*%'9:%a.%%!?!?

 
  U5S5STIy  r;   c                     |j                  d      }|st        j                  |      }||d<   t        j                  | g|i |S )Nr_   )r^   rP   	get_dtyperW   r   )r   r  rR   rV  r_   s        r9   fullr@    sC    JJwE
+F7O;;t-d-f--r;   c                 N   |t         j                  k(  rt        j                  |d u d        t        j                  d|| j                  n|||| j
                  n||      }| j                  r>|j                  | j                         | j                         | j                                n/|j                  | j                         | j                         d       |j                  d       |S t        j                  j                  | |||||      }|j!                  d       |S )Nc                       y)Nz9memory format option is only supported by strided tensorsr5   r5   r;   r9   rf   zzeros_like.<locals>.<lambda>  rn   r;   r   r-  Tr  )rW   
sparse_coorh   r   r_   r   	is_sparsesparse_resize_and_clear_r   
sparse_dim	dense_dimr   _coalesced_r.   r   r  fill_)r   r_   r   r   r   r   r  s          r9   r  r    s     !!!T!O	

 kk %$**5"(.4;;f!
 >>((		T__.0@ ((dhhj!D

//
!
!# " C IIaLJr;   r   c                    |t        j                         }|t        j                         }|t         j                  }t        j                  | ||||      S r/  rW   r   get_default_devicer  r   r   r_   r   r   r   r   s         r9   	meta_onesrN    T     }'')~))+~;;E&J r;   c                    |t        j                         }|t        j                         }|t         j                  }t        j                  | ||||      S r/  rK  rM  s         r9   
meta_zerosrQ    rO  r;   c                     t        |       S rE   _scatter_meta_output)r   r   r   r   s       r9   meta_select_scatterrU        %%r;   c                     t        |       S rE   rS  )r   r   r   r~   r}   steps         r9   meta_slice_scatterrY    rV  r;   c                     t        |       S rE   rS  )r   r   offsetr  r  s        r9   meta_diagonal_scatterr\    rV  r;   c                     ddl m}  ||       s(t        j                  |       dk(  r| j	                         S t        j                  |       S )Nr   rr  r4   )rt  rs  rW   ru  r   rP   clone_preserve_strides)r   rs  s     r9   rT  rT    s?    K
 !&5+L+LT+RVW+Wzz|''--r;   dim_post_exprwrap_scalarc           	          |dk  r|st        d| d      d}| }|dz
  }| |k  s| |kD  rt        d| d| d|  d      | dk  r| |z  } | S )	Nr   zdim_post_expr=z <= 0 but wrap_scalar is Falser4   z4Dimension out of range (expected to be in range of [ry   z], but got rz   )r   
IndexError)r   r_  r`  r  r  s        r9   r   r     s      /MN  .C
!
C
SyC#IB3%r#kZ]Y^^_`
 	
 Qw}Jr;   c                 J    | j                         dk(  rdS | j                  |   S r  r|  )r  r   s     r9   ensure_nonempty_sizerd  *  s!    11.!''#,.r;   c                 :    t         j                         d      }t        j                         d      }t        j                  ||k(  d        t	        |      D ];  k7  s	t        j                  t              t               k   fd       = y )Nr4   c                       y)NzDIndex tensor must have the same number of dimensions as input tensorr5   r5   r;   r9   rf   z$gather_shape_check.<locals>.<lambda>4  rn   r;   c                  N    d dj                    dj                    d  z   S )Nz!Size does not match at dimension z expected index  to be no larger than self  apart from dimension rB  )r   r   r   r   s   r9   rf   z$gather_shape_check.<locals>.<lambda>:  s7    ;A3>Nu{{m\/

|;QRUQVWX r;   )r  r   rW   rh   r   rd  )r   r   r   	self_dims
index_dimsr   s   ```  @r9   gather_shape_checkrl  /  s    DHHJ"IUYY[!$J	LLZV 9 8LL$UA.2FtQ2OOXr;   c                 p   ddl m} t        || j                               } |j	                         dk(        }|s`t        j                  j                  t
        j                  k(  xs j                  t
        j                  k(  fd       t        | |       | j                  j                        S )Nr   r  c                  "    d j                    S )Nz8gather(): Expected dtype int32/int64 for index, but got r   r   s   r9   rf   zmeta_gather.<locals>.<lambda>H  s    Nu{{m\ r;   )rt  r  r   r   r   rW   rh   r_   r   r  rl  r   r   )r   r   r   sparse_gradr  wrapped_dimis_index_emptys     `    r9   meta_gatherrr  ?  s    D dhhj1K#EKKMQ$67NKK5::%A		)A\	
 	4e4>>%++&&r;   c                     |r6| dk(  ry| dk(  ry| dk(  ry| dk(  ry| d	k(  ry
t        j                  dd        y | dk(  ry| dk(  ryt        j                  dd        y )NrH  
REDUCE_ADDr  REDUCE_MULTIPLYrf  REDUCE_MEANamaxREDUCE_MAXIMUMaminREDUCE_MINIMUMFc                       y)Nz=reduce argument must be either sum, prod, mean, amax or amin.r5   r5   r;   r9   rf   z#get_operator_enum.<locals>.<lambda>]  rn   r;   addmultiplyc                       y)Nz/reduce argument must be either add or multiply.r5   r5   r;   r9   rf   z#get_operator_enum.<locals>.<lambda>e  rn   r;   r  )reduce_use_new_optionss     r9   get_operator_enumr  O  s{    e$ ##S	
 	e
"$UUVr;   c                 P    ddl m}  ||j                         dk7        rSt        j                  |j
                  t        j                  k(  xs |j
                  t        j                  k(   fd       |1t        j                  |j
                  |j
                  k(   fd       y y )Nr   )r  c                        dS )Nz((): Expected dtype int32/int64 for indexr5   method_names   r9   rf   z,scatter_gather_dtype_check.<locals>.<lambda>p  s    {m#KL r;   c                        dS )Nz0(): Expected self.dtype to be equal to src.dtyper5   r  s   r9   rf   z,scatter_gather_dtype_check.<locals>.<lambda>v  s    {m#ST r;   )rt  r  r   rW   rh   r_   r   r  )r  r   r   src_optr  s   `    r9   scatter_gather_dtype_checkr  j  sv    CU[[]a'(KK5::%A		)AL	

 JJ'--'T	
 r;   c                     t        | d      S r3   )r  r   s    r9   ensure_nonempty_dimr  z  s    sA;r;   c                     ddl m}  |j                         dk(        ry t        j                  t         j                               t        j                               k(  d        t         j                               }t        |      D ]>  }|k(  r	t        |      }t         |      }t        j                  ||k   fd       @ t        j                  t         j                               t        j                               k(  d        t        |      D ]7  }t        |      }t        |      }	t        j                  ||	k  fd       9 y y )Nr   r  c                       y)NzCIndex tensor must have the same number of dimensions as self tensorr5   r5   r;   r9   rf   z%scatter_shape_check.<locals>.<lambda>  rn   r;   c                  H    dj                    dj                    d  z   S )NExpected index rh  ri  rB  )r   r   r   s   r9   rf   z%scatter_shape_check.<locals>.<lambda>  s,    oekk]2Mdjj\Z&se,- r;   c                       y)NzBIndex tensor must have the same number of dimensions as src tensorr5   r5   r;   r9   rf   z%scatter_shape_check.<locals>.<lambda>  rn   r;   c                  <    d j                    dj                    S )Nr  z to be no larger than src rB  )r   r  s   r9   rf   z%scatter_shape_check.<locals>.<lambda>  s!    /%++6PQXQ^Q^P_` r;   )	rt  r  r   rW   rh   r  r   r   rd  )
r   r   r   r  r  rj  r   index_d_sizeself_d_size
src_d_sizes
   ````      r9   scatter_shape_checkr    s-   Dekkmq()	LLDHHJ'+>uyy{+KKU
 $DHHJ/I 9 	
8+E15*43K'-	
	
 
+/B7;;=/QQX	
 y! 	A/q9L-gq9JLL
*`	 r;   c                     t        || j                               }t        d| ||       t        | |||       |t	        ||       y y )Nscatter)r   r   r  r  r  )r   r   r   r   r  r  rp  s          r9   scatter_meta_implr    sE     dhhj1Ky$s;k5#6'?3 r;   c                 V    t        | |||d       | j                  | j                        S Nr|  r  r   r   r   r   r   r   s       r9   meta_scatter_addr    s%    dCU3>>$**%%r;   c                 $    t        | |||d       | S r  r  r  s       r9   meta_scatter_add_r    s    dCU3Kr;   c                     t        |t        j                        r|nd }t        | ||||       | j	                  | j
                        S rE   )rq   rW   r   r  r   r   r   r   r   src_or_valuer  r   s         r9   meta_scatterr    s;     %\5<<@,dCdCV4>>$**%%r;   c                 `    t        |t        j                        r|nd }t        | ||||       | S rE   )rq   rW   r   r  r  s         r9   meta_scatter_r    s-     %\5<<@,dCdCV4Kr;   queryr   r  	dropout_p	is_causalreturn_debug_maskc           	      .   | j                  d      }| j                  d      }| j                  d      }	| j                  d      }
|j                  d      }t        j                  |       }t        j                  |||	ft        j                  | j
                        }|ra|
dkD  rdnd}t        j                  |	|z        }|dk  rd}n|dk  rd}t        j                  |||	|f| j                  | j
                        }n,t        j                  d| j                  | j
                        }t        j                  j                  rt        j                  j                         st        |       d	k(  rMt        j                  d
t        j                  d      }t        j                  d
t        j                  d      }nLt        j                  dt        j                  d      }t        j                  d
t        j                  d      }||d d |	||||f	S )Nr   r4   r  r1   r  @         r  r5   r   )r   rW   r   r   r[   r   r  ceilr_   r  r   r	  r  r  r   rG  )r  r   r  r  r  r  r  r   r   max_seqlen_batch_qhead_dimmax_seqlen_batch_k	attention	logsumexpblocksize_cmax_seqlen_k
debug_maskseedr[  s                      r9   (meta__scaled_dot_product_flash_attentionr    s    AJ

1IAzz!}H!  'I	Y 23kk||I %]cyy!3k!AB$L3&L[[$6E++<<

 [[%++ellK
 }}UZZ446+e:LPU:U{{2UZZ?Ruzz&A{{Aell6BRu||FC 	
 
r;   	q_descale	k_descale	v_descalec
           	          | j                   t        j                  k(  r| j                  t        j                        } t        | ||||||	      S rE   )r_   rW   r  rw  r  r  )
r  r   r  r  r  r  r  r  r  r  s
             r9   2meta__scaled_dot_product_flash_attention_quantizedr    sJ     {{e)))(3 r;   	res_shape.c                 B    t         j                        |k(  rt        j                         }|S t	        g d fdd      }dgt        |      z  }d}t        |      D ]  }|||<   |||   z  } t        j                  || j                   j                        }|S )N)r   r4   r  r1   c                 *    j                         |    S rE   r  )idxr  s    r9   rf   z,alloc_with_matching_layout.<locals>.<lambda>=  s    %,,.*= r;   Tr   r   r4   r  )
rg   r   rW   r   r  r   reversedru  r_   r   )r  r  r  	dim_orderr  r   r  s   `      r9   alloc_with_matching_layoutr  5  s     U[[Y&u% J =t
	 #I&I& 	%C!GCLin$F	% !!wekk%,,
 Jr;   	attn_biascompute_log_sumexpc	           	         | j                  d      }	| j                  d      }
| j                  d      }|j                  d      }|j                  d      }|	|
||f}t        | |      }t        j                  |	|
|dft        j                  | j
                        }t        j                  dt        j                  d      }t        j                  dt        j                  d      }||d d ||||d f	S )Nr   r4   r  r   r  r5   r   )r   r  rW   r   r[   r   r   )r  r   r  r  r  r  r  r  r  r(  r  S_QS_KVD_Vr  r  
logsum_expr  r[  s                      r9   (meta__scaled_dot_product_cudnn_attentionr  K  s     	

1A

1A
**Q-C88A;D
**R.CAsC I
$UI
6C	
AsAkk||J ;;rF;D[[5::f=F 	
 
r;   c           	         d\  }}	}
| j                         dk(  r| j                         \  }}	}
}n5| j                         dk(  r| j                         \  }	}
}d}nt        d      |j                  d      }|j                  d      }t        | j                        }||d<   t        | t        |            }t        j                  ||	|
ft        j                  | j                        }t        j                  d	t        j                  d
      }t        j                  d	t        j                  d
      }||d d |
|||d f	S )N)r   r   r   rX  r1   r4   zquery must be 3D or 4Dr  r   r  r5   r   )r   r   rv  r   r   r  rg   rW   r   r[   r   r   )r  r   r  r  r  r  r  r  r(  H_Qr  rS   r  r  r   r  r  r  r[  s                      r9   5meta__scaled_dot_product_fused_attention_overrideabler  w  s    KAsCyy{a3Q		jjlS!34488B<D
**R.C U[[!IIbM
$UE),<
=C	
Ckk||J ;;rF;D[[5::f=F 	
 
r;   )r  r  grad_input_maskr  	cum_seq_q	cum_seq_kmax_qmax_kphilox_seedphilox_offsetc                    t        j                  |      }t        j                  |      }t        j                  |      }|t        j                  |      nd }||||fS rE   r   )r  r  r   r  r  r  r   r  r  r  r  r  r  r  r  r  r  grad_qgrad_krq  grad_attn_biass                        r9   >meta__scaled_dot_product_fused_attention_overrideable_backwardr    sZ    * e$Fc"Fe$F4=4IU%%i0tN66>11r;   c                     t        j                  |      }t        j                  |      }t        j                  |      }|||fS rE   r   )r  r  r   r  r   r  r  r  r  r  r  r  r  r  r  r  r  rq  s                     r9   'meta__scaled_dot_product_flash_backwardr    sA    , e$Fc"Fe$F66!!r;   	attn_maskc                 >   | j                  d      }| j                  d      }| j                  d      }	t        j                  |       }
t        j                  ||	|ft	        j
                  | j                        | j                        j                  dd      }|
|fS )Nr   r4   r  r  )	r   rW   r   r   rP   get_computation_dtyper_   r   rM  )r  r   r  r  r  r  r  r   r   r  r  r  s               r9   0meta__scaled_dot_product_flash_attention_for_cpur    s     AJ

1IA  'I	

 ))%++6|| i1o  	 r;   c
                 n   t        j                  |j                         d|j                  |j                        }
t        j                  |j                         d|j                  |j                        }t        j                  |j                         d|j                  |j                        }|
||fS )Nr   r  r4   r1   r  )rW   empty_permutedr   r_   r   )r  r  r   r  r   r  r  r  r  r  r  r  rq  s                r9   9meta__scaled_dot_product_flash_attention_for_cpu_backwardr    s    & !!

kk||	F !!
iizz	F !!

kk||	F 66!!r;   dropout_mask
enable_gqac	                      d }	 |	       \   |	|      \  }
} |	|      \  }}j                   \  }|j                   \  }} fd} |       S )Nc                 l   | j                         dk(  r| j                  d      dfS | j                         dkD  rxd}t        | j                         dz
        D ]  }|| j                  |   z  } | j	                  || j                  d      | j                  d      | j                  d            dfS | d	fS )
Nr1   r   TrX  r4   r  r  r   F)r   r  r   r   viewr   )r>   r   r   s      r9   	ensure_4dzBmeta__scaled_dot_product_attention_math_for_mps.<locals>.ensure_4d:  s    557a<;;q>4''UUWq[J1557Q;' )aggaj(
)66*affRj!&&*affRjI4OOe8Or;   c                     j                  f      } j                  f      }
r	j                         dk(  r&| j                  d      } |j                  d      }| |fS t        	j                  d d       t        | j                  dd       z   }t        	j                  d d       t        |j                  dd       z   }| j                  |      } |j                  |      }| |fS )Nr1   r   r  r4   rX  )r   r   squeezer   r   r  )r   attnr   
attn_shaper   max_seq_lengthnum_headq_q_sizer  
unsqueezedvalue_head_sizes       r9   sdpa_general_mpszImeta__scaled_dot_product_attention_math_for_mps.<locals>.sdpa_general_mpsL  s    llJ&/JK||Z6>JKyy{akk!n||A Dy	 !Sb!12T#))Aa.5II	!%++cr"23d4::a?6KK
hhy)yy,Dyr;   rB  )r  r   r  r  r  r  r  r  r  r  k_rS   v_r  r   r  r  r  r  r  r  s   `             @@@@@@@r9   /meta__scaled_dot_product_attention_math_for_mpsr  .  sl    	 u%NB
cNEBeEB&(hh#J&!,.HH)Aq./   r;   c                 @   | j                  dd      } |j                  dd      }|j                  dd      }| j                  d      }| j                  d      }	| j                  d      }
|j                  d      }t        j                  ||	|
|| j                  | j
                        }t        j                  j                  r&t        j                  j                         r	 |r|	nd}n|rt        j                  |	dz        dz  nd}t        j                  ||
|ft        j                  | j
                        }|j                  dd      }t        j                  dt        j                  d	      }t        j                  dt        j                  d	      }||||fS )
Nr4   r  r   r  r   r  r  r5   r   )rM  r   rW   r   r_   r   r  r   r	  r  r  r  r[   r   )r  r   r  r  r  r  r  r  r(  r  r   Kvr  logsumexp_dimr  r  r[  s                    r9   ,meta__scaled_dot_product_efficient_attentionr  _  sD    OOAq!E
--1
COOAq!E

1A

1A

2I	BB
++aIrU\\
RC}}UZZ446	 0Q2D		!b&)B.!	
I}%kk||J --1
C ;;rF;D[[5::f=F
D&((r;   c                    |j                  d      }|j                  d      }|j                  d      }|j                  d      }|j                  d      }|j                  d      }t        j                  ||||fd|j                  |j                        }t        j                  ||||fd|j                  |j                        }t        j                  ||||fd|j                  |j                        }d }|~|
d   ry|j                  d      }|dz  dk(  r|n
|dz   |dz  z
  }t        |j                               }||d<   t        j                  ||j                  |j                        }|d	d |f   }||||fS )
Nr   r4   r  r1   r  r  r   r  .)r   rW   r  r_   r   r   r   )r  r  r   r  r  r   r  r  r  r  r  r  r  r   r   r  r  
head_dim_vr  r  r  rq  	grad_biaslastDimlastDimAligned	new_sizess                             r9   +meta__scaled_dot_product_efficient_backwardr    s{   ( AJ

1IJJqMEzz!}HAJHHQKE!!	Yx0kk||	F !!	Yx0iizz	F !!	Yz2kk||	F I!3..$$+bLA$57R<'TV,;V)*	&	"KKY__Y5E5E
	 c8G8m,	669,,r;   c                     t        j                  |      }t        j                  |      }t        j                  |      }|||fS rE   r   )r  r  r   r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  rq  s                      r9   'meta__scaled_dot_product_cudnn_backwardr    sA    . e$Fc"Fe$F66!!r;   window_size_leftwindow_size_right	seqused_kalibi_slopesblock_tablerF  c                    || j                  d      n|j                         dz
  }|| j                  d      n|}||j                  d      n|}| j                  d      }| j                  d      }t        j                  |       }|4t        j                  |||ft        j
                  | j                        }nC| j                  d      }t        j                  ||ft        j
                  | j                        }|	ra|dkD  rdnd}t        j                  ||z        }|dk  rd}n|dk  rd}t        j                  ||||f| j                  | j                        }n,t        j                  d| j                  | j                        }d	\  }}t        j                  j                  rkt        j                  j                         rMt        j                  d
t        j                  d      }t        j                  d
t        j                  d      }nLt        j                  dt        j                  d      }t        j                  d
t        j                  d      }|||||fS )Nr   r4   r  r   r  r  r  r  NNr5   r   r  )r   r   rW   r   r   r[   r   r  r  r_   r  r   r	  r  r   rG  )r  r   r  r  r  r  r  r  r  r  r  r  r  r	  r
  r  rF  r   r  r  r   r  r  r  total_qr  r  r  r  r[  s                                 r9   meta__flash_attention_forwardr    s   8 #,"3A9JQ9NJ*3*;A(1(9!u

2Izz"~H   'IKK$67++<<
	 **Q-KK ELL
	 %]cyy!3k!AB$L3&L[[$6E++<<

 [[%++ellK
 LD&}}UZZ446{{2UZZ?Ruzz&A{{Aell6BRu||FC r;   c                 F    t        |||||||||	|
||||||      \  }}}}}|S rE   )r  )r   r  r   r  r  r  r  r  r  r  r  r  r  r  r	  r
  r  rF  rS   r  s                       r9   0meta__flash_attention_forward_no_dropout_inplacer  7  sQ    * :!Ay!Q$ r;   c                     | j                   t        j                  k(  r| j                  t        j                        } t        | |||||||||	|||||      S rE   )r_   rW   r  rw  r  r  )r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  r
  s                     r9   'meta__flash_attention_forward_quantizedr  a  sb    * {{e)))(( r;   c                     t        j                  |      }t        j                  |      }t        j                  |      }|||fS rE   r   )r  r  r   r  r   r  r  r  r  r  r  r  r  r  r  r  r  
grad_querygrad_key
grad_values                       r9   meta__flash_attention_backwardr    sA    0 !!%(J$H!!%(Jx++r;   cu_seqlens_qcu_seqlens_kmax_seqlen_qr  custom_mask_typecausal_diagonalseqlen_kwindow_sizec                    | j                  d      }| j                  d      }|j                  d      }| j                  d      }|j                  d      }t        j                  ||||| j                  | j                        }||j                  d      dz
  n|}|}||t        d      |}||n|}|
rt        j                  |dz        dz  nd}t        j                  |||ft        j                  | j                        }t        j                  dt        j                  d	      }t        j                  dt        j                  d	      }||||||fS )
Nr   r4   r  r   r  z;max_seqlen_q must not be None when cu_seqlens_q is providedr  r5   r   )
r   rW   r   r_   r   r   r  r  r[   r   )r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r(  r  r  r   r  r  logsumexp_batch_dimactual_max_seqlen_qactual_max_seqlen_kr  r  r  r[  s                               r9   !meta__efficient_attention_forwardr$    sE   , 	

1A

1AA

2I	BB
++aIrU\\
RC7C7O,++A.2VW M  +*6*B,4F		%*+b0A  	i7kk||J ;;rF;D[[5::f=F
D&*=?RRRr;   bias_requires_gradnum_splits_keyshared_storage_dqdkdvc                    |rt        j                  |j                  d   |j                  d   k(  d        t        j                  |j                  d   |j                  d   k(  d        t        j                  g |j                  dd d|j                  d   |j                  d   |j                  |j
                        }|j                  d	d      }|j                  d	d      }|j                  d	d
      }n?t        j                  |      }t        j                  |      }t        j                  |      }|z|j                  d      }|dz  dk(  r|n
|dz   |dz  z
  }t        |j                               }||d<   t        j                  ||j                  |j
                        }|dd |f   }n!t        j                  d|j
                        }||||fS )Nr4   c                       y)Nz,seqlen must match for `shared_storage_dqdkdvr5   r5   r;   r9   rf   z4meta__efficient_attention_backward.<locals>.<lambda>  rn   r;   r1   c                       y)Nz3embedding dim must match for `shared_storage_dqdkdvr5   r5   r;   r9   rf   z4meta__efficient_attention_backward.<locals>.<lambda>  rn   r;   r   r  r   r  r  r  r  .r5   rU  )
rW   rh   r   r   r_   r   r0  r   r   r   )r  r  r   r  r  r  r  r  r  r  r  r  r  r  r%  r  r&  r'  chunkr  r  r  r  r  r  r   s                             r9   "meta__efficient_attention_backwardr,    s   2 KKNciil*B	
 	KKNciil*I	
 Eekk!BEEEKKOEU[[_E++<<

 \\"a(
<<A&\\"a(
%%e,
##C(%%e,
))B-$+bLA$57R<'TV,;V%	&	"KK	DKKP	c8G8m,	KK5<<8	xY66r;   scale_ascale_bscale_resultuse_fast_accumc                 2    d }t        j                   j                         dk(  xr j                         dk(   fd       t        j                   | j                        xr  |j                         fd       t	               dk(  st	               dk(  st	               dk(  rd }	d	 }
d
 }t	               dk7  rt        j                   |	 j                               xs  |        fd       t        j                   |
j                               xs  |      fd       t        j                   j                  d      dz  dk(   fd       t        j                  j                  d      dz  dk(  xr j                  d      dz  dk(  fd        j                  \  j                  d      j                  t         j                  k(  xr j                  t         j                  k(  xs< j                  t         j                  k(  xr j                  t         j                  k(  }j                         dk(  rfj                         dk(  rSt        j                  j                  t         j                  k(  xr j                  t         j                  k(  d        n|rj                  t         j                  k(  rd}dz  n$d} j                  t         j                  k(  rdz  d}t        |      }t        |d      dz  }|t        |      z  |z  |t        |      z  |z  j                         k(  r_j                         k(  rLt        j                  j                         d        t        j                  j                         d        nt        j                  dfd       nht        j                  j                  t         j                  k(  xr j                  t         j                  k(  d        t        j                  j                         dk(  xr j                         dk(  fd       j                  d      k(  ruj                  d      dk(  raj                  d      dk(  rMj                  d      k(  r9t        j                  j                         xr j                         d        nNj                  d      k(  rUj                  d      j                  d      cxk(  rt        d      k(  r"n nj                  d      t        d      k(  rnj                  d      k(  rKj                  d      j                  d      cxk(  rt        d      k(  rn nj                  d      k(  rnj                  d      t        d      k(  rKj                  d      j                  d      cxk(  rt        d      k(  rn nj                  d      k(  rnt        j                  dfd       ||n j                  }t        j                   j                  d      j                  d      | j                         S )Nc                     | t         j                  t         j                  t         j                  t         j                  t         j
                  fv S rE   rW   r  float8_e5m2float8_e4m3fnuzfloat8_e5m2fnuzfloat4_e2m1fn_x2r   s    r9   is_fp8_or_fp4_typez2_check_scaled_mm_sizes.<locals>.is_fp8_or_fp4_type(  A    !!!!""
 
 	
r;   r  c                  L    dj                          d j                          S Nz%Inputs must be 2D but got self.dim()=z and mat2.dim()=r   r  r   s   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>3  '    7
|CSTXT\T\T^S_` r;   c                  <    dj                    d j                    S Nz?Expected both inputs to be fp8 or fp4 types but got self.dtype=z and mat2.dtype=r   r<  s   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>7  &    QRVR\R\Q]]mnrnxnxmyz r;   r	  r  r  c                 ,    | d   | d   kD  xr | d   dk(  S r  r5   r  s    r9   is_row_majorz,_check_scaled_mm_sizes.<locals>.is_row_major@  "    !9vay(;VAY!^;r;   c                 &    | d   dk(  xr | d   dkD  S r  r5   r  s    r9   is_col_majorz,_check_scaled_mm_sizes.<locals>.is_col_majorC      !9>3fQi!m3r;   c                 V    | j                  d      dk(  xs | j                  d      dk(  S r  r   	tensor_2ds    r9   has_zero_dimz,_check_scaled_mm_sizes.<locals>.has_zero_dimF  )    >>!$)CY^^A->!-CCr;   c                  *    d j                          S Nz#self must be row_major, got stride r  r   s   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>L      =dkkm_M r;   c                  *    d j                          S Nz#mat2 must be col_major, got stride r  r  s   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>P  rN  r;   r4   r  r   c                  ,    d j                  d       S NzBExpected self.size(1) to be divisible by 16, but got self.size(1)=r4   r   r   s   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>T      \]a]f]fgh]i\jk r;   c                  "    d j                    S Nz?Expected both dimensions of mat2 to be divisible by 16 but got rB  rQ  s   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>X      YZ^ZdZdYef r;   c                       y)NzNFor tensorwise scaling, both scale_a and scale_b must be float (fp32) tensors.r5   r5   r;   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>o  rn   r;   r  r  rX  c                       y)Nzscale_a must be contiguousr5   r5   r;   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>  rn   r;   c                       y)Nzscale_b must be contiguousr5   r5   r;   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>  rn   r;   Fc            	      Z    d  dj                          d dj                          d	S )NzTInvalid blockwise scaling configuration. For blockwise scaling, scale_a should have  elements, got z, scale_b should have r  r  )expected_a_sizeexpected_b_sizer-  r.  s   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>  sH    FFUEVVefmfsfsfuev w//>.?w}}N__`b r;   c                       y)NzKFor rowwise scaling, both scale_a and scale_b must be float (fp32) tensors.r5   r5   r;   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>  rn   r;   c                  L    d j                         dj                         S )NzLFor non-tensorwise scaling, scale tensors must be 2D, but got scale_a.dim()=z and scale_b.dim()=r   r-  r.  s   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>  s,    gY`YdYdYfXhh|nunynyn{m}~ r;   c                       y)Nz@Both scale_a and scale_b must be contiguous for rowwise scaling.r5   r5   r;   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>  rn   r;   c                     d d d dt         d       d	dt         d       dt        d       d dt         d       d	z   dt         d       d d	t        d       dt         d       d	z   dt         d       d d
j                  d       dj                  d       dj                  d       dj                  d       dz   S )N}Invalid scaling configuration. For tensorwise scaling, both scales should be scalar. For rowwise scaling, scale_a should be (, 1), scale_b should be (1, >). For (BlockWise1x128, BlockWise128x128), scale_a should be (ry   r  ), scale_b should be (<). For (BlockWise1x128, BlockWise1x128), scale_a should be (z>). For (BlockWise128x128, BlockWise1x128), scale_a should be (). Got scale_a.size()=(r   r4   ) and scale_b.size()=(rz   r:   r   )_kr  r+  r-  r.  s   r9   rf   z(_check_scaled_mm_sizes.<locals>.<lambda>  sL   CCD#Eabcad eVVWUXXZ[cdfhk[lZmmpr 0S0A/B"XaQTEUDV WTTUSVVXYabdfiYjXkknp	p 0S0A/B"QC HVV^_`beVfUggijrsuwzj{i||AA 0S0A/B"QC H//6||A.?r',,q/AR S//6||A.?r',,q/ARRSU
U r;   r  )rW   rh   r   r_   r  r   r   r   float8_e8m0fnur  r   r  r7  r:   r  r   r   )r   r  r-  r.  r  r/  r  r0  r8  rB  rE  rJ  is_blockwise_scalingblock_size_kblock_size_mnnum_k_blockspadded_num_k_blocks
_out_dtyperm  r]  r^  r  r+  s   ````              @@@@@r9   _check_scaled_mm_sizesru    sB   
 
LL
a+DHHJ!O` 
LL4::&I+=djj+Iz 	DV#t%t%	<	4	D t%LLT[[]+A|D/AM LLT[[]+A|D/AM LL		!r!Q&k LL		!r!Q&A499Q<"+<+Af 

2IIaL !5!55 :MMU%9%99
 !4!44 9MMU%8%88 	 ==?aGMMOq$8LL.Q7==EMM3Qh " }} 3 33  "!V!::!7!77aBM#B5L"*<";a"? M ::=PP  M ::=PP 
 ?2MMO6))+8 ))+8
  LL.Q7==EMM3Qe
 LL"9w{{}'9~ Q1$LLOq(LLOq(LLOq( ))+G0E0E0G^
 Q1$LLOw||AK(2s:KKLLOx3'77 Q1$LLOw||AK(2s:KKLLOq( Q8As#33LLOw||AK(2s:KKLLOq(  " (3J;;tyy|TYYq\DKKXXr;   c           
      &    t        | |||||||      S rE   )ru  )r   r  r-  r.  r  r/  r  r0  s           r9   meta_scaled_mmrw    s"     "dGWdL)^ r;   scale_recipe_ascale_recipe_b	swizzle_a	swizzle_bc           
      N   	 !"#$%& d }d }t        j                   j                         dk(  xr j                         dk(   fd       t        j                   | j                        xr  |j                         fd        j                  d     j                  d   j                  d   ! | j                        r |j                        rd}|z  |D cg c]  }t        |       }}|D cg c]  }t        |       }}rD cg c]  }t        |       c}nt        j                  g	r	D cg c]  }t        |       c}	nt        j                  g	t               dk(  st               d	k(  st               d
k(  rd }d }d }t               d
k7  rt        j                   | j                               xs  |        fd       t        j                   |j                               xs  |      fd       t        j                   j                  d      dz  dk(   fd       t        j                  j                  d      dz  dk(  xr j                  d      dz  dk(  fd       dt        t
           dt        t
           fd}dt        t
           dt        t
           fd}dt        t
           dt        t
           fd}dt        t
           dt        t
           fd}dt        t
           dt        t
           fd}dt        t
           dt        t
           fd}dt        t
           dt        t
           fd}dt        t
           dt        t
           fd} |||      rt        j                  d   j                         dk(  xr  d   j                  t         j                  k(  d        t        j                  d   j                         dk(  xr  d   j                  t         j                  k(  d        
n0 |||      rFt        j                  d   j                  d       k(  xr d   j                  d      dk(   fd       t        j                  d   j                          k(  xr  d   j                  t         j                  k(   fd        t        j                  d   j                         !k(  xr  d   j                  t         j                  k(  !fd!       t        j                  d   j                  d      dk(  fd"       t        j                  d   j                  d      dk(  fd#       n |||      rqd   j                  t         j                  k(  xr  d   j                  t         j                  k(  }d   %%j                  d    k(  xrk %j                  d   d$z  k(  xrT %j                  d      dk(  xr> %j                  d       k(  xs( %j                  d   dk(  xr %j                  d      dk(  }d   &&j                  d   !k(  xrk &j                  d   d$z  k(  xrT &j                  d      dk(  xr> &j                  d      !k(  xs( &j                  d   dk(  xr &j                  d      dk(  }t        j                  |xr |xr | !%&fd%       ne |||      rd   j                  t         j                  k(  xr  d   j                  t         j                  k(  }t        d$z  d&      d   %%j                  d   k(  xrk %j                  d    d$z  k(  xrT %j                  d      dk(  xr> %j                  d      k(  xs( %j                  d   dk(  xr %j                  d      dk(  }d   &&j                  d   !k(  xrk &j                  d   d$z  k(  xrT &j                  d      dk(  xr> &j                  d      !k(  xs( &j                  d   dk(  xr &j                  d      dk(  }t        j                  |xr |xr | !%&fd'       n |||      rd   j                  t         j                  k(  xr  d   j                  t         j                  k(  }t        d$z  d&      d   %%j                  d    k(  xrk %j                  d   d$z  k(  xrT %j                  d      dk(  xr> %j                  d       k(  xs( %j                  d   dk(  xr %j                  d      dk(  }d   &&j                  d   k(  xrk &j                  d   !d$z  k(  xrT &j                  d      dk(  xr> &j                  d      k(  xs( &j                  d   dk(  xr &j                  d      dk(  }t        j                  |xr |xr | !%&fd(       nO |||      rt         j                   j"                  rct%         j                  d   d)       j                  d   z  "t%         j                  d   d)       j                  d   z  #t        j                  $nt         j                  d   d$      t        t%         j                  d   d)      d&      z  "t        j                  d   d$      t        t%         j                  d   d)      d&      z  #t        j&                  $t        j                  d   j                         "k(  xrn d   j                  t         j(                  k(  xrL d   j                         #k(  xr4 d   j                  t         j(                  k(  xr d   $k(  xr 	d   $k(  "#$	fd*       n |||      rt         d$      t        t%        d      d&      z  "t        !d$      t        t%        d      d&      z  #t        j&                  $t        j                  d   j                         "k(  xrn d   j                  t         j*                  k(  xrL d   j                         #k(  xr4 d   j                  t         j*                  k(  xr d   $k(  xr 	d   $k(  "#$	fd+       n |||      rnt         d$      t        t%        d      d&      z  "t        !d$      t        t%        d      d&      z  #t        j&                  $t        j                  d   j                         "k(  xr d   j                  t         j*                  k(  xr d   j                         dk(  xr d   j                  t         j                  k(  xr d   j                         #k(  xrn d   j                  t         j*                  k(  xrL d   j                         dk(  xr4 d   j                  t         j                  k(  xr d   $k(  xr 	d   $k(  "#$	fd,       nt        j                  d- !fd.       ||n j                  }t        j,                   !| j.                  /      S c c}w c c}w c c}w c c}w )0Nc                     | t         j                  t         j                  t         j                  t         j                  t         j
                  fv S rE   r3  r   s    r9   r8  z5_check_scaled_mm_sizes_v2.<locals>.is_fp8_or_fp4_type  r9  r;   c                 (    | t         j                  k(  S rE   )rW   r7  r   s    r9   is_fp4_typez._check_scaled_mm_sizes_v2.<locals>.is_fp4_type  s    ....r;   r  c                  L    dj                          d j                          S r;  r   r<  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>  r=  r;   c                  <    dj                    d j                    S r?  r   r<  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>  r@  r;   r   r4   r	  r  r  c                 ,    | d   | d   kD  xr | d   dk(  S r  r5   r  s    r9   rB  z/_check_scaled_mm_sizes_v2.<locals>.is_row_major6  rC  r;   c                 &    | d   dk(  xr | d   dkD  S r  r5   r  s    r9   rE  z/_check_scaled_mm_sizes_v2.<locals>.is_col_major9  rF  r;   c                 V    | j                  d      dk(  xs | j                  d      dk(  S r  r   rH  s    r9   rJ  z/_check_scaled_mm_sizes_v2.<locals>.has_zero_dim<  rK  r;   c                  *    d j                          S rM  r  r   s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>B  rN  r;   c                  *    d j                          S rP  r  rQ  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>F  rN  r;   r  c                  ,    d j                  d       S rS  r   r   s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>J  rT  r;   c                  "    d j                    S rV  rB  rQ  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>N  rW  r;   recipe_arecipe_bc                     t        |       dk(  xr> t        |      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S r   )r   r)   
TensorWiser  r  s     r9   is_tensorwisez0_check_scaled_mm_sizes_v2.<locals>.is_tensorwiseQ  sT    H" :MQ&:QK;#9#99: QK;#9#99	r;   c                     t        |       dk(  xr> t        |      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S r   )r   r)   RowWiser  s     r9   
is_rowwisez-_check_scaled_mm_sizes_v2.<locals>.is_rowwiseY  sT    H" 7MQ&7QK;#6#667 QK;#6#66	r;   c                     t        |       dk(  xr> t        |      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S r   )r   r)   BlockWise1x32r  s     r9   is_mxz(_check_scaled_mm_sizes_v2.<locals>.is_mxa  sT    H" =MQ&=QK;#<#<<= QK;#<#<<	r;   c                     t        |       dk(  xr> t        |      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S r   )r   r)   BlockWise1x16r  s     r9   is_nv_single_levelz5_check_scaled_mm_sizes_v2.<locals>.is_nv_single_leveli  sV     H" =MQ&=QK;#<#<<= QK;#<#<<	r;   c                     t        |       dk(  xrn t        |      dk(  xr^ | d   t        j                  k(  xrF | d   t        j                  k(  xr. |d   t        j                  k(  xr |d   t        j                  k(  S )Nr  r   r4   )r   r)   r  r  r  s     r9   is_nvz(_check_scaled_mm_sizes_v2.<locals>.is_nvs  s    H" :MQ&:QK;#<#<<: QK;#9#99: QK;#<#<<	:
 QK;#9#99r;   c                     t        |       dk(  xr> t        |      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S r   )r   r)   BlockWise1x128r  s     r9   is_1x128_1x128z1_check_scaled_mm_sizes_v2.<locals>.is_1x128_1x128}  sT    H" >MQ&>QK;#=#==> QK;#=#==	r;   c                     t        |       dk(  xr> t        |      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S r   )r   r)   r  BlockWise128x128r  s     r9   is_1x128_128x128z3_check_scaled_mm_sizes_v2.<locals>.is_1x128_128x128  sW    H" @MQ&@QK;#=#==@ QK;#?#??	r;   c                     t        |       dk(  xr> t        |      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S r   )r   r)   r  r  r  s     r9   is_128x128_1x128z3_check_scaled_mm_sizes_v2.<locals>.is_128x128_1x128  sT    H" >MQ&>QK;#?#??> QK;#=#==	r;   c                       y)Nz!scale_a must have 1 Float elementr5   r5   r;   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>  rn   r;   c                       y)Nz!scale_b must have 1 Float elementr5   r5   r;   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>  rn   r;   c                  H    d  dt        d   j                                S )Nzscale_a must have shape [z
, 1], got r   )r   r   r  r-  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>  s)    /s*T'!*//BS=T<UV r;   c                  6    d  dd   j                          S )Nzscale_a must have  Float elements, got r   r  r  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>  '    (+@AQAQAS@TU r;   c                  6    d  dd   j                          S )Nzscale_b must have r  r   r  )r  r.  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>  r  r;   c                  2    d d   j                  d       S )Nz,expected scale_a.stride(1) to be 1, but got r   r4   r  )r-  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>       FwqzGXGXYZG[F\] r;   c                  2    d d   j                  d       S )Nz,expected scale_b.stride(1) to be 1, but got r   r4   r  )r.  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>  r  r;   r  c                      d d dz   dj                    d dj                   d d dz   dj                    d dj                   dS )Nz>For 1x128 x 1x128 blockwise scaling, scale a must have shape [ry   r  ] (got: ) and stride [1, )scale b must have shape [rz   r   r   )r  r  r  sasbs   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>  s~    001s"Q#XJhrxxjPabcaddlmomvmvlw x001s"Q#XJhrxxjPabcaddlmomvmvlwwxz r;   rX  c                      d ddz   dj                    d dj                   d d dz   dj                    d dj                   dS )Nz]For 128x128 x 1x128 blockwise scaling, L4 = {round_up(K / 128, 4)}, scale a must have shape [ry   r  r  r  r  rz   r  r  L4r  r  r  r  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>  s~    002t2a3hZxzQbcebffnoqoxoxny z001s"Q#XJhrxxjPabcaddlmomvmvlwwxz r;   c                      d d dz   dj                    d dj                   d ddz   dj                    d dj                   dS )Nz]For 1x128 x 128x128 blockwise scaling, L4 = {round_up(K / 128, 4)}, scale a must have shape [ry   r  r  r  r  rz   r  r  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>  s~    001s"Q#XJhrxxjPabcaddlmomvmvlw x002t2a3hZxzQbcebffnoqoxoxnyyz| r;   r  c                      d  dd   j                          d dd   j                          dt        j                   dd   j                   dd   j                   d d	d    dd    d
S )Nz!for MX scaling scale_a must have  (got: r   ) and scale_b must have z). Scales must have types z (for self: 	, mat_b: z) Must have swizzle type  (got self: rz   )r   rW   rn  r_   expected_scale_a_elemsexpected_scale_b_elemsexpected_swizzler-  r.  rz  r{  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>/  s    78N7OwW^_`WaWgWgWiVj k--C,DGGTUJL\L\L^K_ `""'"6"6!7|GAJDTDTCUU^_fgh_i_o_o^p q..>-?|IVWL>Ybclmncobppqs r;   c                      d  dd   j                          d dd   j                          d dd    dd    dS )	Nz.for single-level NV scaling scale_a must have r  r   r  ). Must have swizzle type r  r  rz   r  r  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>A  sw    DE[D\\cdklmdndtdtdvcw x--C,DGGTUJL\L\L^K_ `$$4#5\)A,yYbcdYeXffgi r;   c                      d  dd   j                          d dd   j                          d dd    dd    dS )	Nz!for NV scaling scale_a must have r  r   r  r  r  r  rz   r  r  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>V  sv    78N7OwW^_`WaWgWgWiVj k--C,DGGTUJL\L\L^K_ `$$4#5\)A,yYbcdYeXffgi r;   Fc                  b   d d d dt         d       d	dt         d       dt        d       d dt         d       d	z   dt         d       d d	d
   j                  d
       dd
   j                  d       dd
   j                  d
       dd
   j                  d       dz   S )Nrd  re  rf  ry   r  rg  rh  ri  rj  r   r4   rk  rz   rl  )r  r  r  r-  r.  s   r9   rf   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>_  s   ??@cA]^_]` aRRSQTTVW_`acfWgVhhkm ,HQ,<+=RC@P?Q RPPQsRTU]^_adUeTffik	k ,HQ,<+=Rs C++21:??1+=*>bQRAS@T U++21:??1+=*>bQRAS@TTUWW r;   r  )rW   rh   r   r_   r   r)   r*   
NO_SWIZZLEr  r   r   r   _check_valuer   r  r@   r  r   r:   SWIZZLE_32_4_4rn  r  r   r   )'r   r  r-  rx  r.  ry  r  r  rz  r{  r0  r8  r  K_packed_multipliersirB  rE  rJ  r  r  r  r  r  r  r  r  types_ok
scale_a_ok
scale_b_okrt  r  r  r  r  r  r  r  r  r  s'   ``` `   ``                    @@@@@@@@@r9   _check_scaled_mm_sizes_v2r    s   
/ 
LL
a+DHHJ!O` 
LL4::&I+=djj+Iz 	

1A

1A

1A 4::;tzz#:	  0>?"k"o?N?0>?"k"o?N?/89[_9	 ""
	 /89[_9	 ""
	
 	DV#t%t%	<	4	D t%LLT[[]+A|D/AM LLT[[]+A|D/AM LL		!r!Q&k LL		!r!Q&A499Q<"+<+Af
	D$5 	kAR 		k!2 	d;>O 		D- 	k9J 		;'	373D		D- 	k9J 		T+%6 	${BS 		tK'8 	DDU 		tK'8 	DDU 	 8 
  "a'MGAJ,<,<,M; 
  "a'MGAJ,<,<,M; 7
"a'CGAJOOA,>!,C 
  "a'MGAJ,<,<,M 
  "a'MGAJ,<,<,M 
!!!$)] 
!!!$)] NN;
 
  EMM1Wgaj6F6F%--6W  Bq  THHQK18+TIIaLA%T YYq\Q&R288A;!+;+Q		!PQ@Q	  Bq  THHQK18+TIIaLA%T YYq\Q&R288A;!+;+Q		!PQ@Q	  LL6Z6J nn= 
  EMM1Wgaj6F6F%--6W  !c'1%BBr! UHHQK18+UIIaLA%U YYq\R'SBHHQK1,<,R1QRAR	  Bq  THHQK18+TIIaLA%T YYq\Q&R288A;!+;+Q		!PQ@Q	  LL6Z6J  nn= 
  EMM1Wgaj6F6F%--6W  !c'1%BBq  THHQK18+TIIaLA%T YYq\Q&R288A;!+;+Q		!PQ@Q	  Br! UHHQK18+UIIaLA%U YYq\R'SBHHQK1,<,R1QRAR	  LL6Z6J  >>2}}  )1$**Q-)DtzzRS})T&)1$**Q-)DtzzRS})T&#.#9#9 )1$**Q-)ETZZ]B/I *& *2$**Q-)ETZZ]B/I *& $/#=#= LL
  "&<< 5AJ$$(<(<<5AJ$$&*@@5 AJ$$(<(<<5 aL$44	5
 aL$44   ?%-a%5!RRS8T%T"%-a%5!RRS8T%T"*99LL
  "&<< 5AJ$$(;(;;5AJ$$&*@@5 AJ$$(;(;;5 aL$44	5
 aL$44  >>2%-a%5!RRS8T%T"%-a%5!RRS8T%T"*99LL
  "&<< 	5AJ$$(;(;;	5AJ$$&!+	5 AJ$$5	5 AJ$$&*@@		5
 AJ$$(;(;;	5 AJ$$&!+	5 AJ$$5	5 aL$44	5 aL$44 $ LL
 (3J;;q!:dkkBB[
 @? : :s   0zz"zz"rD  contraction_dimsc                 .    t        | |||||||	|||      S )N)r  r  rz  r{  r0  )r  )r   r  r-  rx  rz  r.  ry  r{  r  rD  r  r0  s               r9   meta_scaled_mm_v2r  p  s2     %% r;   c                 Z    t        | ||||d       | j                  | j                        S NT)r  r  r   r   r   r   r  r  s         r9   meta_scatter_reduce_twor    s)     dCVTJ>>$**%%r;   c                 (    t        | ||||d       | S r  r  r  s         r9   meta_scatter_reduce__twor    s    dCVTJKr;   c                t    t        j                  d j                         cxk  xr dk  nc  fd        j                         dk(  r0t        j                  |t         j                   j
                        S t        j                   j                  d      |t         j                   j
                        S )Nr   r  c                  *    d j                          S )NzAThe probability distributions dimensions must be 1 or 2, but got r   rA  s   r9   rf   z"meta_multinomial.<locals>.<lambda>  s    STYT]T]T_S`a r;   r4   r  )rW   rh   r   r   r   r   r   )r   num_samplesreplacementr   s   `   r9   meta_multinomialr    s|     
LL	EIIK1a yy{a{{;ejjNN;;

1{%**U\\ r;   c                 "    d}| D ]  }||z  }	 |S r3   r5   )vsr  vs      r9   multiply_integersr    s$    	A 	QHr;   c                 L    t        j                  t              k(  fd       dz   t        j                  t               k(   fd       t        j                  t        d  dd  D              xr t        d D               fd        d d \  }}||gS )Nc                  &    d  dt               S )Nz%It is expected output_size equals to , but got size r  )num_spatial_dimsr  s   r9   rf   z'upsample_common_check.<locals>.<lambda>  s    78H7IY\]hYiXjk r;   r  c                  &    d  dt               S )Nz$It is expected input_size equals to r  r  )expected_input_dimsr  s   r9   rf   z'upsample_common_check.<locals>.<lambda>  s    67J6K?[^_i[jZkl r;   c              3   &   K   | ]	  }|d kD    ywr   Nr5   rs   r  s     r9   ru   z(upsample_common_check.<locals>.<genexpr>  s     *aAE*   c              3   &   K   | ]	  }|d kD    ywr  r5   r  s     r9   ru   z(upsample_common_check.<locals>.<genexpr>  s     2NQ1q52Nr  c                      d  d S )NzDInput and output sizes should be greater than 0, but got input size z and output size r5   )r  r  s   r9   rf   z'upsample_common_check.<locals>.<lambda>  s      \!2;-A r;   )rW   rh   r   r`  )r  r  r  r3  channelsr  s   ```  @r9   upsample_common_checkr    s    	LLK,,k +Q.	LLJ..l
 
LL*:ab>**Ns2N+2N/N	A ""1~FHH+{++r;   c                 4    t        j                   j                         dk7  xs t         j	                         dd         fd       t         j	                         |d      } j                  |      j                  t        j                               S )Nr   r4   c                  *    d j                          S )Nz>Non-empty 3D data tensor expected but got a tensor with sizes r   rA  s   r9   rf   z$upsample_nearest1d.<locals>.<lambda>      PQVQ[Q[Q]P^_ r;   r  r   
rW   rh   r   r  r   r  r   rw  rP   r   )r   r  scalesfull_output_sizes   `   r9   upsample_nearest1dr         
LLA/

QR0@A_ -

kA ??+,//11%8 0  r;   c                     t        j                   j                         dk7  xs t         j	                         dd         fd       t         j	                         |d      } j                  |      }t        j                         } j                  \  }}}} j                  j                  dk(  r|dk  rt         j                  }|j                  |      }|S )	Nr   r4   c                  *    d j                          S Nz>Non-empty 4D data tensor expected but got a tensor with sizes r   rA  s   r9   rf   z$upsample_nearest2d.<locals>.<lambda>  r  r;   r  r  r	  rX  r   )rW   rh   r   r  r   r  r   rP   r   r   r   r{   r   
contiguous)	r   r  scales_hscales_wr  r  r   rS   
n_channelss	   `        r9   upsample_nearest2dr    s     
LLA/

QR0@A_ -

kA __-.F //6M  ++Az1a||F"zA~//];FMr;   r  r  r  r  c                 X    t        ||d      t        j                   j                  dk(   fd       t	        d      D ]2  t        j                   j                           k(   fd       4  j                  |      j                  t        j                               S )Nr  r  rX  c                  "    d j                    S NzFExpected grad_output to be a tensor of dimension 4 but got: dimension rT  r   s   r9   rf   z-upsample_nearest2d_backward.<locals>.<lambda>      XYdYiYiXjk r;   c            
      D    d d     d dj                         S )NzCExpected grad_output to have the same shape as output; output.size() = z but got grad_output.size(r   r  rM  r   s   r9   rf   z-upsample_nearest2d_backward.<locals>.<lambda>  s>      !s$'7':&;,QCtK4D4DQ4G3HJ r;   r   )
r  rW   rh   r   r   r   r   rw  rP   r   )rM  r  r  r  r  r  r   s   `    @@r9   upsample_nearest2d_backwardr	    s     -K! 
LLAk 1X 
Q#3A#66	

   ,//11+> 0  r;   c                 4    t        j                   j                         dk7  xs t         j	                         dd         fd       t         j	                         |d      } j                  |      j                  t        j                               S )Nr   r4   c                  *    d j                          S )Nz>Non-empty 5D data tensor expected but got a tensor with sizes r   rA  s   r9   rf   z$upsample_nearest3d.<locals>.<lambda>  r  r;   r1   r  r   r  )r   r  scales_dr  r  r  s   `     r9   upsample_nearest3dr
	    r  r;   c                    t        j                  |       t        j                  | t         j                        }}||t        |t              st        dt        |             t        |t              st        dt        |             |j                  }|j                         }	t        ||      }t        ||      }|j                  ||	       |j                  ||	       t        ||       t        ||       ||fS ||fS )Nr   zvalues must be TensorLike, got z indices must be TensorLike, got )r  r  )rW   r   r   rq   r    r   r{   r   r   r"   r   r$   )
r   stabler   
descendingr   r   r  r   r   
out_strides
             r9   	meta_sortr	  "  s     D!5#3#3D#LqAg1&*- #B4<.!QRR':. #CDM?!STT GG	XXZ
"695#GY79j1Iz2F3G4wa4Kr;   c           	          t        j                   j                  dk(   fd       t        j                   j                  j                  k(   fd        j	                  d      t        j                  j                  dk(  fd       t        j                  j                         k(  fd       t        j                  j                  j                  k(  fd       t        j                  j                  dk(  fd        j	                  d	      z  z  t        j                  j                         k(   fd
       t        j                  t         fdfD              d        y )Nr  c                  "     j                    dS Nz != 2rT  )input_gatess   r9   rf   z%rnn_cell_checkSizes.<locals>.<lambda>G      ;3C3C2DE0J r;   c                  :    j                    d j                    S Nr  rB  )hidden_gatesr	  s   r9   rf   z%rnn_cell_checkSizes.<locals>.<lambda>J  s     ;$$%T,*<*<)=> r;   r4   c                  "     j                    dS )Nz != 1rT  )
input_biass   r9   rf   z%rnn_cell_checkSizes.<locals>.<lambda>N  s    joo5Fe3L r;   c                  .    j                          d  S r	  r  )
gates_sizer	  s   r9   rf   z%rnn_cell_checkSizes.<locals>.<lambda>Q  s    z'')*$zl; r;   c                  :    j                    d j                    S r	  rB  )hidden_biasr	  s   r9   rf   z%rnn_cell_checkSizes.<locals>.<lambda>U  s     z''([->->,?@ r;   c                  "     j                    dS r	  rT  )prev_hiddens   r9   rf   z%rnn_cell_checkSizes.<locals>.<lambda>W  r	  r;   r   c            
      `    j                          dj                  d       d d d  d
S )Nr  r   z * z // z (aka rz   )r   r   )expected_prev_hidden_numelfactorr	  r	  r	  s   r9   rf   z%rnn_cell_checkSizes.<locals>.<lambda>[  sB    ;$$&'tK,<,<Q,?+@J<tTZS[[ab|a}}~ r;   c              3   P   K   | ]  }|j                   j                   k(    y wrE   rU  )rs   r>   r	  s     r9   ru   z&rnn_cell_checkSizes.<locals>.<genexpr>^  s(      
 HH***
s   #&c                       y)Nz%expected all inputs to be same devicer5   r5   r;   r9   rf   z%rnn_cell_checkSizes.<locals>.<lambda>b  rn   r;   )rW   rh   r   r   r   r   r`  )r	  r	  r	  r	  r"	  r	  r!	  r	  s   ``````@@r9   rnn_cell_checkSizesr%	  ?  s@    
LL!!Q&(JK	LL\///> !!!$JZ__)+LM*,;	
 	 1 11@	
 
LL!!Q&(JK!,!1!1!!4z!AV!K	LL99 
LL 
"J[I
 	
 	8r;   c                 
   t        | |||d|       t        j                  | t        j                        }t        j                  |t        j                        }t        j                  |t        j                        }|||fS )NrX  r   )r%	  rW   r   r   )r	  r	  cxr	  r	  	workspacehycys           r9   _thnn_fused_lstm_cell_metar+	  f  sk     \:{ArR  E<S<STI			"E,C,C	DB			"E,C,C	DBIr;   c                 b   t        |      dk7  }|r t        |      }|d   }| j                  d   }nB|
r| j                  d   n| j                  d   }|
r| j                  d   n| j                  d   }d}|rdnd}|dk7  r|n|}|r|||z  g}n|
r||||z  gn||||z  g}| j                  |      }|	|z  ||g}|"t        j                  d| j
                        }n|j                  |      }|j                  |	|z  ||g      }|rdnd}| j                  |t        j                        }|||||fS )Nr   r4   r   r  rU  r   )r   r   r   rW   r   r   r<  )r   r  weight_stride0
weight_bufhxr'	  r  hidden_size	proj_size
num_layersbatch_firstdropouttrainbidirectionalrH  dropout_stateis_input_packed
seq_length
mini_batchbatch_sizes_sumnum_directionsout_sizer   r  
cell_shaper*	  r)	  reserve_shapereserves                                r9   
_cudnn_rnnrA	  u  sS   & +&!+O%
 ^
++a.'2U[[^A
'2U[[^A
'QQN%NyH$h&?@	  X%>?j(^*CD 	
 __Y'F~-z;GJ	z[[5<<0\\*%	zN2JI	JB AAMoom5;;o?G2r7J..r;   c                     d}|D ])  }|j                         dkD  s||j                         z  }+ | j                  |f      }t        | |||||||d|||	|
|||      S r  )r   r   rA	  )r   r  r-	  r/	  r'	  r  r0	  r2	  r3	  r4	  r5	  r6	  rH  r7	  total_weight_elemsr?  r.	  s                    r9   
miopen_rnnrD	    s    &  ,779q=!'')+, "4!67J

	! r;   c                 (   |r| j                   d   n| j                   d   }|r| j                   d   n| j                   d   }|
}|r|||gn|||g}| j                  |      }|"t        j                  d| j                        }n|j                  |j                         }|"t        j                  d| j                        }n|j                  |j                         }t        j                  d| j                  t        j
                        }||||fS )Nr4   r   rU  r   )r   r   rW   r   r   r<  )r   w0w1w2w3hx_cx_r   rH  r  r0	  r2	  
has_biasesr6	  r3	  r5	  r9	  r:	  output_chanelsr   r  r)	  r*	  r(	  s                           r9   mkldnn_rnn_layerrN	    s    & $/QEKKNJ#.QEKKNJ N  
Z0*n5 
 __Y'F
{[[5<<0]]399%
{[[5<<0]]399%Aell%++FI2r9$$r;   c                     | j                   dk(  r%t        j                  dk(  xs dk(  fd       y t        j                  | j                        dk7  fd       y )Nr   r   c                       d  S )Nz4: Expected reduction dim -1 or 0 for scalar but got r5   r   r-  s   r9   rf   z'zero_numel_check_dims.<locals>.<lambda>  s    wiSTWSXY r;   c                       d  dS )Nz: Expected reduction dim z to have non-zero size.r5   rQ	  s   r9   rf   z'zero_numel_check_dims.<locals>.<lambda>  s    wi8=TU r;   )r   rW   r   r   )r   r   r-  s    ``r9   zero_numel_check_dimsrS	    sR    yyA~1H!r	Y	

 	IIcNaU	
r;   c                      |(t        ||j                               }t        ||        y t        j                  |j                         dk7   fd       y )Nr   c                        dS )Nz@: Expected reduction dim to be specified for input.numel() == 0.r5   r  s   r9   rf   z%check_argmax_argmin.<locals>.<lambda>  s    tf\] r;   )r   r   rS	  rW   rh   r   )r  r   r   s   `  r9   check_argmax_argminrV	    sC    
S$((*-dC.JJLA]	
r;   c                     t        d| |       t        j                  | j                  ||fnd       }t	        | ||      }| j                  |t        j                        S )Nargmaxr   )rV	  rP   r  r   r  r   rW   r   )r   r   r  r  r   s        r9   argmax_argmin_metarY	    sQ    $,

coSF4PD$T49E>>%u{{>33r;   c                 |    |t         j                  k(  rt         j                  }t        j                  d||||      S )Nr5   r-  )rW   jaggedr  r   )r  r_   r   r   r   s        r9   scalar_tensorr\	  #  s5    
 ;;
%v* r;   c                    t        || j                         d      }| j                         dk(  rdn| j                  |      }t        j                  |dk\         t        j                  ||k  d        t        | j                        }t        |      dkD  r|||<   | j                  |      | j                  |t        j                        fS )NTr`  r   r4   c                       y)Nzk not in range for dimensionr5   r5   r;   r9   rf   ztopk_meta.<locals>.<lambda>5  rn   r;   r   )
r   r   r   rW   rh   r   r   r   r   r   )r   r  r   largestr  	sliceSizetopKSizes          r9   	topk_metarc	  /  s     dhhjd
;CXXZ1_$))C.I	LLa	LLi!GHDJJH
8}q>>(#T^^HEKK^%PPPr;   c                     ||t        d      |j                         }| j                         }	t        j                  ||	j                  |	j
                  |	j                        S )Nz;segment_reduce(): Either lengths or offsets must be defined)r_   r   r   )r   r  rW   r   r_   r   r   )
r  r  rM  r  r  r  r  r  data_contiggrad_contigs
             r9   meta__segment_reduce_backwardrg	  =  sh    
 7?I
 	
 //#K//#K!!!!	 r;   c                    ddl m} t        | j                         d      | j                         dkD  r| j	                        nd}t        j                   ||dk\  ||k        fd       t        | j                  d  | j                  dz   d  z         }|r%| j                         dkD  r|j                  d       | j                  |      | j                  |t
        j                        fS )Nr   r$  Tr^	  r4   c                      d  S )Nz9kthvalue(): selected number k out of range for dimension r5   r   s   r9   rf   zkthvalue_meta.<locals>.<lambda>Y  s    KC5Q r;   r   )rt  r  r   r   r   rW   rh   r   r   r~  r   r   )r   r  r   r  r  dimSizer   s     `    r9   kthvalue_metark	  P  s     >
dhhjd
;C $
QdiinAG	LLQW%Q
 DS!DJJsQwy$99:E488:>S!>>% $..ekk."JJJr;   c                    | | n|}t        j                  |j                         dk(  d        |j                         }| (t        j                  | j                         |k(  d        |(t        j                  |j                         |k(  d        t        j                  |j                         |k(  d        t        j                  |j                         |k(  d        t        j                  |j                         dk(  d        t        j                  |j	                         |d   |d	   z  d
z  k(  d        y )Nr  c                       yN r5   r5   r;   r9   rf   z(checkLSTMBackwardSizes.<locals>.<lambda>h  rn   r;   c                       yrn	  r5   r5   r;   r9   rf   z(checkLSTMBackwardSizes.<locals>.<lambda>k  rn   r;   c                       yrn	  r5   r5   r;   r9   rf   z(checkLSTMBackwardSizes.<locals>.<lambda>m  rn   r;   c                       yrn	  r5   r5   r;   r9   rf   z(checkLSTMBackwardSizes.<locals>.<lambda>n  rn   r;   c                       yrn	  r5   r5   r;   r9   rf   z(checkLSTMBackwardSizes.<locals>.<lambda>o  rn   r;   c                       yrn	  r5   r5   r;   r9   rf   z(checkLSTMBackwardSizes.<locals>.<lambda>p  rn   r;   r   r4   rX  c                       yrn	  r5   r5   r;   r9   rf   z(checkLSTMBackwardSizes.<locals>.<lambda>q  rn   r;   )rW   rh   r   r   r   )grad_hygrad_cyr'	  r*	  r(	  defined_gradexp_sizes          r9   checkLSTMBackwardSizesrz	  f  s    %17wL	LL!!#q(*5  "HW\\^x/<W\\^x/<	LLh&
3	LLh&
3	LLA%z2	LL"hqkHQK&?!&CCZPr;   c                     | |yt        | ||||       t        j                  |t              }t        j                  |t              }|r|j	                  dd      nd }|||fS )NNNNr   r   F)r  )rz	  rW   r   legacy_contiguous_memory_formatrH  )	rv	  rw	  r'	  r*	  r(	  has_bias
grad_gatesgrad_cxr   s	            r9   #_thnn_fused_lstm_cell_backward_implr	  u  sl    7?7GRY?!!!@J r1PQG4<
q%0$Iw	))r;   c                    d }d }d }|d   r|j                  | j                               }|d   s|d   rQ|j                  |j                  d      | j                  d      f      }|j                  |j                  d            }|||fS )Nr   r4   r  r   r   r   )rE  rD  rF  rH  rP  grad_weightr   s          r9   linear_backwardr	    s    JKI1~!++FKKM:
1~Q",,l.?.?.CV[[QS_-UV **<+<+<R+@A	Y//r;   c                 N    t        j                  t         j                        dkD   fd       t        j                   j                  d   z  z  dk(   fd        fd} j                  d   z  z  } j                  d   z  } j                  d   z  }g  j                  d d |||} |       }|t         j                  u s|t         j
                  u rt        |      }nt        |      }t        j                  || j                   j                  	      S )
Nr  c                  "    d j                    S )N'Invalid input shape for pixel_shuffle: rB  r   s   r9   rf   z$meta_pixel_shuffle.<locals>.<lambda>  s    9$**F r;   r  r   c                  (    d j                    d S )Nr	  z with upscale_factor = rB  )r   upscale_factors   r9   rf   z$meta_pixel_shuffle.<locals>.<lambda>  s    9$**E\]k\lm r;   c                      t              } | t        j                  u s| t        j                  u rt	              dk(  r| S t        j
                  S r2  )r   rW   rd  r+  r  r   )r  r   s    r9   pick_memory_formatz.meta_pixel_shuffle.<locals>.pick_memory_format  sH     $D)5&&&#1G1G*G$5(J&&&r;   r  r   r  )rW   rh   r   r   rd  r+  r   r   ru  r_   r   )	r   r	  r	  r7  HrWrr   r  r   s	   ``       r9   meta_pixel_shuffler	    s   	LLDJJ!F 
LL

2.>9:a?m
'" 	

2>N:;A	B.	(B	B.	(B-$**Sb/-1-b-"-I 
C
e!!!SE,B,B%B4Y?1)<;djj r;   c                 >   | j                  | j                  t        j                        }|j                  |j                  t        j                        }|j                  |j                  t        j                        }|j                  |j                  t        j                        }|j                  |j                  t        j                        }d}||k(  rd|z  }n|j                  d   }|j                  |gt        j                        }|||||||fS )Nr   r1   rX  r   )r   r   rW   r[   )r   weight0weight1weight2weight3rJ	  cx_tmpr  hy_cy_grad_output_r_optgrad_hy_r_optgrad_cy_r_optr   r  r0	  r2	  rL	  r5	  r6	  rH  r3	  r(	  diff_xdiff_hxdiff_cxdiff_w1diff_w2	_GRU_MODE	bias_sizediff_bs                                  r9   mkldnn_rnn_layer_backwardr	    s    4 __U[[_<FmmCIIU[[m9Gv||5;;?GU[[AGU[[AG IyO	MM!$		{%++>F7GVVWgEEr;   )	out_int32rightc                    t        j                  | |rt         j                  nt         j                  t         j                        S )Nr_   r   )rW   r   r  r   r   r   
boundariesr	  r	  s       r9   meta_bucketizer	    s2     &ekkEKK-- r;   r	  r	  r	  c                r    |j                  d|rt        j                        S t        j                        S )Nr5   r   )r   rW   r  r   r	  s       r9   meta_bucketize_scalarr	    s>     
&ekk    ,1KK    r;   c                     dt               dk(  r't        j                   j                          fd       t               dk(  r% j                         rt	        j
                  d       t        j                  t        t              fd       t        j                  dkD  fd       t        j                  t        t              fd	       t        j                  t        t              fd
       t        j                  k\  fd       t        j                   j                   j                        S )Nzhistc()r  c                  $    d j                    dS )Nz%"histogram_cpu" not implemented for 'r>  r   rA  s   r9   rf   zmeta_histc.<locals>.<lambda>   s    =ekk]!L r;   r	  z%_histc_cuda with floating point inputc                  $     dt                S )Nz#: argument 'bins' must be int, not r]  binsr-  s   r9   rf   zmeta_histc.<locals>.<lambda>   s    7)>tDzlK r;   r   c                       d  S )Nz: bins must be > 0, but got r5   r	  s   r9   rf   zmeta_histc.<locals>.<lambda>   s    gY.J4&#Q r;   c                  $      dt               S )Nz%: argument 'min' must be Number, not r]  )r-  r  s   r9   rf   zmeta_histc.<locals>.<lambda>       7)@cL r;   c                  $      dt               S )Nz%: argument 'max' must be Number, not r]  )r-  r  s   r9   rf   zmeta_histc.<locals>.<lambda>   r	  r;   c                        dS )Nz: max must be larger than minr5   )r-  s   r9   rf   zmeta_histc.<locals>.<lambda>   s    y0M%N r;   r   )r  rW   rh   r!  rP   r<  rq   r   r   r   r   r_   )r   r	  r  r  r-  s   ````@r9   
meta_histcr	     s     G5U"##%L	
 5V#(?(?(A%%&MN	LL4!K 
LLQR	LL3L 
LL3L 
LLNO;;tELLDDr;   c                 B    t         j                         |d      }t        j                   j	                         dk7  xs# t        d  j                         dd  D               fd        j                  |      j                  t        j                               S )Nr  r  r   c              3   &   K   | ]	  }|d kD    ywr  r5   )rs   r   s     r9   ru   z,meta_upsample_bimode2d_aa.<locals>.<genexpr>2   s     !Ht$(!Hr  r4   c                  *    d j                          S r  r   rA  s   r9   rf   z+meta_upsample_bimode2d_aa.<locals>.<lambda>3   r  r;   r   )
r  r   rW   rh   r   r`  r   rw  rP   r   )r   r  r2  r  r  r  s   `     r9   meta_upsample_bimode2d_aar	      s     -

kA 
LLHc!Huzz|AB7G!HH_ ??+,//11%8 0  r;   c                 T    t        ||d      t        j                   j                  dk(   fd       t	        d      D ]0  t        j                   j
                        k(   fd       2  j                  |      j                  t        j                               S )Nr  r  rX  c                  "    d j                    S r	  rT  r   s   r9   rf   z4meta_upsample_bimode2d_aa_backward.<locals>.<lambda>M   r	  r;   c            
      D    d d     d dj                         S )NzD
Expected grad_output to have the same shape as output; output.size(r	  z
but got grad_output_size(r   r	  s   r9   rf   z4meta_upsample_bimode2d_aa_backward.<locals>.<lambda>R   s@     DDE3dK[\]K^J_ `D!1!1!!4 59 r;   r   )
r  rW   rh   r   r   r   r   rw  rP   r   )rM  r  r  r2  r  r  r  r   s   `     @@r9   "meta_upsample_bimode2d_aa_backwardr	  :   s     -K! 
LLAk 1X 
a $4Q$779	

   ,//11+> 0  r;   c                 P   t        j                  |j                         dk(  d        t        j                  |j                         dk(  d        t        j                  |j                  j                  d        t        j                  |j                  j                  d        y )Nr4   c                       y)Nz%found_inf must be a 1-element tensor.r5   r5   r;   r9   rf   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>_   rn   r;   c                       y)Nz%inv_scale must be a 1-element tensor.r5   r5   r;   r9   rf   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>b   rn   r;   c                       y)Nz!found_inf must be a float tensor.r5   r5   r;   r9   rf   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>f   rn   r;   c                       y)Nz!inv_scale must be a float tensor.r5   r5   r;   r9   rf   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>j   rn   r;   )rW   rh   r   r_   r!  )r   rY  	inv_scales      r9   *_amp_foreach_non_finite_check_and_unscale_r	  \   s|    	LLQ O 
LLQ O 
LL))3 
LL))3r;   c                 ,    t        j                  |       S rE   r   )r   nanposinfneginfs       r9   
nan_to_numr	  o   r   r;   c                    | j                   t        j                  t        j                  t        j                  t        j
                  hv rt        d| j                    d      | j                  }t        ||      }t        ||      }||k(  r| S t        | j                               }t        | j                               }||   ||   c||<   ||<   ||   ||   c||<   ||<   | j                  ||       | S )Nz>torch.transpose_: in-place transposition is not supported for z layout)r   rW   r  
sparse_cscr  
sparse_bscr   r   r   r   r   r   r   )r   dim0r  ndimsr   r   s         r9   r  r  u   s    {{	  LT[[MY`a
 	
 IIE$&D$&Dt|		D$++- F!'vd|F4L&,!$ZdDJT
T6"Kr;   c                    | j                   }| j                  r<| j                         }| j                         }|dk  r|dk(  s4t	        d| d| d      | j                         dkD  rt	        d| d      t        | d|dk  rd      S d      S )	Nr  r   zEt_ expects a tensor with <= 2 sparse and 0 dense dimensions, but got z sparse and z dense dimensionsz6t_ expects a tensor with <= 2 dimensions, but self is r  r4   )r   rD  rF  rG  r   r   r  )r   r	  rF  rG  s       r9   t_r	     s    IIE~~__&
NN$	aIN %,l9+=NP 
 88:> HqQ  dAEAIq55155r;   )r	  r	  sidesorterc                ^    t        j                  t         j                        dk  xs  j                  d d j                  d d k(   fd       t        j                  d u xs  j                  j                  k(   fd       t        j                  |dk7  xs | d        |rt         j                  nt         j
                  }t        t         j                        r&t        j                  |t         j                        S t        j                  d| j                  	      S )
Nr4   r   c                  `    dt        j                         dt         j                         S )Nztorch.searchsorted(): boundaries tensor should be 1 dimension or the first N-1 dimensions of boundaries tensor and input value tensor must match, but we got boundaries tensor z and input value tensor r^  )r   sorted_sequences   r9   rf   z#meta_searchsorted.<locals>.<lambda>   s8    3378M8M3N2O P""&tzz"2!35 r;   c                  l    dt         j                         dt        j                         S g  S )Nz[torch.searchsorted(): boundary and sorter must have the same size, but got boundary tensor z and got sorter tensor r^  )r	  r	  s   r9   rf   z#meta_searchsorted.<locals>.<lambda>   sO    ##'(=(=#>"??V%+%7tFLL!@B  >@@B r;   r   c                       y)Nzetorch.searchsorted(): side and right can't be set to opposites, got side of left while right was Truer5   r5   r;   r9   rf   z#meta_searchsorted.<locals>.<lambda>   rn   r;   r	  r5   r  )rW   rh   r   r   r  r   rq   r   r   r   r   r   )r	  r   r	  r	  r	  r	  r_   s   ``   ` r9   meta_searchsortedr	     s     
LLO!!"a' 	9  "%CR8	
	 
LL$?///6<<?	
 
LL#e)	$ %EKK%++E$%U-D-D
 	
 {{2U?3I3IJJr;   c                      t        j                   t         j                  t         j                  t         j                  fv fd       y )Nc                      d  S )Nz/Unsupported input type encountered for isin(): r5   r   s   r9   rf   z3_check_for_unsupported_isin_dtype.<locals>.<lambda>   s    A%I r;   )rW   rh   rC  
complex128	complex64r   s   `r9   !_check_for_unsupported_isin_dtyper	     s/    	LLejj%"2"2EOODDIr;   c                 J    | j                  || j                  d      f      }|S )Nr   r	  )rM  r   num_weightsr  r'  r	  s         r9   meta_embedding_dense_backwardr	     s*     ''k6F6Fr6J(KLKr;   c                 j    |	rt         j                  | ||||||||
|
      S t        | ||||||||
|
      S rE   )r.   _embedding_bag_sparse_backward!meta_embedding_bag_dense_backward)r  r   r  r+  r,  maximum_indicesr	  r'  r  r(  r  r  s               r9   meta_embedding_bag_backwardr	     se     22
 	
 1
 	
r;   c
                 N    t        j                   j                  t         j                  t         j                  t         j
                  t         j                  fv  fd       |t        k(  rt        j                  |d u        j                  | j                  d      f      }
|
S )Nc                  "    d j                    S )Nz$Unsupported input type encountered: r   )r  s   r9   rf   z3meta_embedding_bag_dense_backward.<locals>.<lambda>'!  s    6tzzlC r;   r4   )
rW   rh   r_   r  r  r  float64r%  r   r   )r  r   r+  r,  r	  r	  r'  r  r  r  index_grad_weights   `          r9   r	  r	  !  sv     
LL

u}}ennemmU]]SSC x_D01TYYq\'BCr;   c                    | j                  d      }t        j                  |t        k(  d        t        j                  | j	                         dk(         t        j                  |j	                         dk(         |j                  d      }t        j                  |j	                         dk(         t        j                  |j                  d      |k(         | j                  |f      }	|	S )Nr4   c                       y)NzHembedding_bag_backward: per_sample_weights only supported for mode='sum'r5   r5   r;   r9   rf   z@meta_embedding_bag_per_sample_weights_backward.<locals>.<lambda><!  rn   r;   r  r   )r   rW   rh   r$  r   r   )
r  r  r   r  r+  r  r  embedding_featuresr  r  s
             r9   .meta_embedding_bag_per_sample_weights_backwardr	  /!  s     1	LLZ 
LLq!	LL!#$,,q/K	LL"#	LLQ#556^^[N+FMr;   )assume_uniqueinvertc                   t        j                  t        | t              xs t        |t              d        t        | t              s!t        j                  | |j
                        } t        |t              s!t        j                  || j
                        }t        | j                         t        |j                         t        j                  | t         j                        S )Nc                       y)Nz<At least one of elements and test_elements must be a Tensor.r5   r5   r;   r9   rf   zmeta_isin.<locals>.<lambda>L!  rn   r;   rU  r   )
rW   rh   rq   r   r  r   r	  r_   r   rC  )elementstest_elementsr	  r	  s       r9   	meta_isinr	  G!  s     
LL8V$I
=&(IN h'<<1E1EFmV,]8??K%hnn5%m&9&9:HEJJ77r;   r+  c                     t        j                  | dk\  d        t        |t        j                        \  }}t        j
                  ||      S )Nr   c                       y)Nz,polygamma(n, x) does not support negative n.r5   r5   r;   r9   rf   z meta_polygamma.<locals>.<lambda>\!  rn   r;   r  r   )rW   rh   r   r   r  r   )r+  r   rS   rT   s       r9   meta_polygammar	  Y!  sF     
LLaOP(;HHOA| D55r;   c                     t        d      )Nz.Tensor.item() cannot be called on meta tensors)rv  r   s    r9   meta_local_scalar_denser	  d!  s    
G
HHr;   c                 ,    t        j                  |       S rE   r   r   s    r9   silur	  i!  r   r;   c                 l    t        | t        j                        \  }}t        j                  | |      S r  )r   r   r  rW   r   )r   rS   rT   s      r9   sigmoidr	  o!  s3     );HHOA| D55r;   c                 R   | j                         dk(  }|j                         dk(  }|r|r4|j                  d      | j                  d      |j                  d      g}n"t        j                  |j                  d      |j                  d      k(  d        | j                  d      |j                  d      g}n|r[t        j                  |j                  d      | j                  d      k(  d        | j                  d      |j                  d      g}njt        j                  | j                  d      |j                  d      k(  d        | j                  d      | j                  d      |j                  d      g}|xs | j                  }t        j
                  j                  rZd|j                  z  }|d   |z   dz
  |z  |z  }||k(  r|d   |z  |dg}	n|dg}	t        j                  ||	|| j                  	      }
|
S t        j                  ||| j                  	      }
|
S )
Nr  r   r4   c                       yNz matrix batch sizes have to matchr5   r5   r;   r9   rf   z2_create_grouped_mm_output_tensor.<locals>.<lambda>!  rn   r;   r   c                       yr
  r5   r5   r;   r9   rf   z2_create_grouped_mm_output_tensor.<locals>.<lambda>!  rn   r;   c                       y)Nzbatched dimension has to matchr5   r5   r;   r9   rf   z2_create_grouped_mm_output_tensor.<locals>.<lambda>!  rn   r;   r  r  )r   r   rW   rh   r_   r  r	  itemsizeru  r   r   )r  r  offsr  
mat1_is_2d
mat2_is_2dr=	  	alignmentsize_paddedr	  r   s              r9    _create_grouped_mm_output_tensorr
  y!  s   qJqJ		!diilDIIaLAHLL		!		!,.X 		!diim4HLL		!		!,.X 		!diil3H LL		!		!,.V 		!diilDIIbMBH'TZZI}}),,,	|i/!3	AIM#"1+3[!DJ%q)J!!j	$++

 J kk()DKKHJr;   mat_amat_br
  c	                     t        j                  d u d u k(  d        d uxr d u}	|	rt         j                  }
t         j                  j                  rYt         j
                  j                         r;dt         j
                  j                  d      j                  v rt         j                  }
t        j                   j                  |
k(  xr j                  |
k(   fd       nTt        j                   j                  t         j                  k(  xr j                  t         j                  k(   fd       t        j                   j                         dv xr j                         dv  fd        j                         dk(  }j                         dk(  }|r|s8t        j                   j                  d	      j                  d
      k(  d        |	rDd }d }t        j                   |        fd       t        j                   |      fd       d } |d         |d       t        j                  j                  t         j                  k(  xr j                  t         j                  k(  xs< j                  t         j                  k(  xr j                  t         j                  k(  fd       j                  t         j                  k(  xr j                  t         j                  k(  dfd	}|r|rj                   d   nd} |d d|        |dd|       t        j                  |d u d        |s|r}t        j                  d u fd       xt        j                  j                         dk(  fd       t        j                  j                  t         j"                  k(  fd       nt        j                  d u d        t        j                  |d u d        t        j                  |d u xs |t         j                  k(  d        t%         |      S ) Nc                       y)Nz,Either both scale factors are given, or noner5   r5   r;   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>!  rn   r;   gfx94r   c                  >    d j                    dj                    dS )Nz5Expected inputs of E4M3 FP8 type but got mat_a.dtype= and mat_b.dtype=r  r   r
  r
  s   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>!  s%    KEKK=Xijojujuivvwx r;   c                  >    d j                    dj                    dS )Nz1Expected inputs of BF16 type but got mat_a.dtype=r
  r  r   r
  s   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>!  s%    G}Tefkfqfqerrst r;   )r  r1   c                  L    d j                          dj                          S )Nz3Multiplicands must be 2D or 3D but got mat_a.dim()=z and mat_b.dim()=r   r
  s   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>!  s'    Eeiik]Rcdidmdmdocpq r;   r  r   r  c                       y)Nz3contraction dimension of mat_a and mat_b must matchr5   r5   r;   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>!  rn   r;   c                 F    | j                         }|d   dkD  xr |d   dk(  S Nr  r4   r   r  mat
mat_strides     r9   rB  z-_meta_grouped_mm_common.<locals>.is_row_major!  s*    Jb>A%=*R.A*==r;   c                 F    | j                         }|d   dk(  xr |d   dkD  S r
  r  r
  s     r9   rE  z-_meta_grouped_mm_common.<locals>.is_col_major!  s*    Jb>Q&=:b>A+==r;   c                  0    d j                         dd   S )NzNExpected mat_a tensor to be row major in the last two dimensions, got strides r  r  )r
  s   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>!  s#    dejeqeqestvtwexdyz r;   c                  0    d j                         dd   S )NzQExpected mat_b tensor to be column major in the last two dimensions, got strides r  r  )r
  s   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>!  s#    ghmhththvwywzh{g|} r;   c                     j                         dz
  dj                         z  }j                         dz
     dk(  rG   t        dj                  dz
           k\  r%t        j                     |z  dk(   fd       y    dk(  rJdz
     t        dj                           k\  r(t        j                  dz
     |z  dk(   fd       y t        j                  dfd       y )Nr4   r  r   c                  "    d d  d     dS )Nr   stride along % dim to be multiple of 16 bytes, got r  r5   end_dimmat_namer
  s   r9   rf   zF_meta_grouped_mm_common.<locals>.check_valid_strides.<locals>.<lambda>!  s)    )H:^G9Dijtu|j}i~~  A r;   c                  .    d d dz
   d dz
      dS )Nr  r!
  r4   r"
  r  r5   r#
  s   r9   rf   zF_meta_grouped_mm_common.<locals>.check_valid_strides.<locals>.<lambda>!  sK    )H:^GaK=Hmnx  zA  DE  zE  oF  nG  GH  I r;   Fc                  *    d d j                    dS )NzInvalid strides/sizes, got z for strides and z for sizes.rB  r
  s   r9   rf   zF_meta_grouped_mm_common.<locals>.check_valid_strides.<locals>.<lambda> "  s!    5j\ARSVS\S\R]]hi r;   )r   element_sizer   r  r   rW   rh   )r%
  r
  r	
  r$
  r
  s   `` @@r9   check_valid_stridesz4_meta_grouped_mm_common.<locals>.check_valid_strides!  s    '')a-#**,,	ZZ\
gk"a'Jw,?3syy1%D
 -
 LL7#i/14 A  A%*Wq[*ASsyy!F
 +
 LL7Q;')3q8 I
 LLir;   r
  r
  c                  >    d j                    dj                    dS )NzhFor FP8 scales must both be float32, or for MXFP8 both scales must be float8_e8m0fnu. Got scale_a.dtype=z and scale_b.dtype=r  r   ra  s   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>"  sT    ~  @G  @M  @M  N  Na  bi  bo  bo  ap  pq  r r;   r4   c                    	
 j                         dk(  rt        j                  j                          fd       r;t        j                  j                         j                         k(   fd       y t        j                  j                         dk(   fd       t        j                  j                  d   j                     z  k(   fd       y t        j                  j                  d      dk(   fd	       t        j                  j                  d   j                  d   k(   fd
       rt        j                  j                  j                  dz
  k(   fd       j                  \  }}d}t        ||z  d      	t        |d      
t        j                  j                  d   k(  xr j                  d   	
z  k(  	
fd       y t        j                  j                         dk(   fd       t        j                  j                  d   j                  dz      k(   fd       y )Nr  c                      d  dS )Nr  z to be contiguous.r5   
scale_names   r9   rf   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>"  s    i
|3EF r;   c                  B    d d j                    dj                    S )NzKFor MXFP8, scale must have same number of dimensions as target tensor, but  has mat.ndim= and scale.ndim=rT  r
  r  r.
  s   r9   rf   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>!"  sZ    "mnxmy  zH  IL  IQ  IQ  HR  Rb  ch  cm  cm  bn  !o r;   r4   c                  2    d d j                          dS )Nr  z to be 1D tensor, but got 	D tensor.r   r  r.
  s   r9   rf   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>&"  #    )J<7QRWR[R[R]Q^^g h r;   r   c                  V    d d j                      z   dj                   d    dS )Nr  z	 to have r\  r   z
 elements.rB  )r
  r  scale_multiplierr.
  
scaled_dims   r9   rf   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>*"  sW    )J<y:AVYiAi@jjyz  {F  {F  GH  {I  zJ  JT  !U r;   r   c                      d  dS )Nr  z( to be contiguous in the last dimension.r5   r-
  s   r9   rf   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>/"  s    i
|3[\ r;   c                  P    d d j                   d    dj                   d    dS )Nr  z batch dimension to be r   , got r  rB  r2
  s   r9   rf   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>3"  s6    i
|3J399UV<.X^_d_j_jkl_m^nnop r;   c                  B    d d j                    dj                    S )Nz0For MXFP8, 3d tensor should have 2d scales, but r0
  r1
  rT  r2
  s   r9   rf   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>:"  sF    "RS]R^^lmpmumulv  wG  HM  HR  HR  GS  !T r;   r  rX  r  c                  N    dj                    d  dz   dj                    S )NzFor MXFP8, expected mat.shape=z to have scale shape of (,z), but got rB  )G	blocked_K	blocked_Nr
  r  s   r9   rf   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>D"  sQ    "@Kdefdgghiru~i~h  @K  LQ  LW  LW  KX  !Y r;   c                  2    d d j                          dS )Nr  z to be 2D tensor, but got r4
  r   r5
  s   r9   rf   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>I"  r6
  r;   c                  V    d d j                   dz       dj                   d    dS )Nr  z non-batch dimension to be r4   r<
  r  rB  )r
  r  r.
  r9
  s   r9   rf   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>M"  sT    )J<7RSVS\S\]^ak]kSlRmmstytt  AB  uC  tD  DE  !F r;   )r   rW   rh   r  r   r   r   r@   )r.
  r  r
  r9
  r8
  r  r  r  r@
  rA
  rB
  is_mxfp8s   `````   @@@r9   check_scalez,_meta_grouped_mm_common.<locals>.check_scale"  s   wwyA~'')F LL		swwy0 o
 LL		q(h LLA#))J*?BR*RR U
 LL$)\ KKNciil2p LL

chhl2 T "iiGAq!!#J (Z ;I (C 0ILLA!+WA)iBW0W Y
 LL		q(h LLA#))A
N*CC Fr;   r-  r.  c                       y)Nz:Scale result tensor provided, but it is not supported yet.r5   r5   r;   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>X"  rn   r;   c                  N    d j                          dj                          dS )Nz/Offsets tensor not provided, but is needed for zD/zD multiplicand layouts.r   r
  s   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>^"  s*    Eeiik]RTUZU^U^U`Taaxy r;   c                  ,    d j                          dS )Nz.Offsets tensor must be 1D, but got offs.dim()=r  r   r
  s   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>c"  s    HTUV r;   c                  $    d j                    dS )Nz7Offsets tensor must be integer (int32) tensor, but got r  r   rJ
  s   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>g"  s    QRVR\R\Q]]^_ r;   c                       y)NzJOffsets tensor provided, but is not needed for 3D/3D multiplicand layouts.r5   r5   r;   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>l"  rn   r;   c                       y)Nz2Bias tensor provided, but it is not supported yet.r5   r5   r;   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>q"  rn   r;   c                       y)Nz4If output dtype provided, it must be torch.bfloat16.r5   r5   r;   r9   rf   z)_meta_grouped_mm_common.<locals>.<lambda>v"  rn   r;   r_  )rW   rh   r  r  r   r	  r  get_device_propertiesgcnArchNamer5  r_   r  r   r   r  rn  r   r  r
  )r
  r
  r-  r.  r
  r  r/  r  r0  scaled	fp8_dtypemat_a_is_2dmat_b_is_2drB  rE  r)
  rF
  r8
  rE
  s   `````             @r9   _meta_grouped_mm_commonrU
  !  s    
LL	Dgo.> D 8WD%8F ''	MM

'')5::;;A>JJJ--IKK9$A	)Ax	

 	KK5>>)KekkU^^.Kt	

 
LL		v7%))+"7q
 ))+"K))+"KkJJrNejjn,I	

 	>	> 	z	
 	}	

0 ''w2]]emm+N0N !5!55 :MMU%9%99 r	
 MMU111 6!5!55 	
:	z "-++DJJqMST 	 	Iwq2BCIwq2BCD P	

 ky	
 LL
aV LL

ekk)_
 	DL`	

 
LLD
 
LLT8Y%..8F
 ,E5$	JJr;   c           
      (    t        | |d d ||d |      S )N)r-  r.  r
  r  r/  r  )rU
  )r
  r
  r
  r  r  s        r9   meta_grouped_mmrW
  |"  s)     #	 	r;   c	                 R    |xs t         j                  }t        | ||||||||	      S )N)r-  r.  r
  r  r/  r  r0  )rW   r  rU
  )	r
  r
  r-  r.  r
  r  r/  r  r0  s	            r9   meta_scaled_grouped_mmrY
  "  s:     +U^^I"!%
 
r;   c                 6   t        |      t        d      k(  r/| D ]*  }t        j                  |j                         dkD  d        , g }| D ]K  }||n|j                  }|j
                  rt        |      }|j                  |j                  d|             M |S )Ninfr   c                       y)Nz:_foreach_norm cannot compute infinity norm on empty tensorr5   r5   r;   r9   rf   z#meta_foreach_norm.<locals>.<lambda>"  rn   r;   r5   r   )	r[   rW   rh   r   r_   r  r   r  r   )tensorsordr_   r  resultsr  s         r9   meta_foreach_normr`
  "  s    SzU5\! 	ALL	AT	
 G 9".EAGG	0;Iq{{2Y{78	9
 Nr;   r>   half_to_floatc                 P   |rE| j                   t        j                  t        j                  fvrt	        d| j                    d      t        j                  | t
        j                  j                        \  }}|s|n|}t        j                  | |t        j                        }|S )Nz%half_to_float is True but x.dtype is z, expected half or bfloat16r  r	  )r_   rW   rY   r  r   rP   r   r   rQ   r   r   )r>   r   ra
  computation_dtyperT   r  s         r9   softmaxrd
  "  s     775::u~~66 7y@[\  ',&>&>	uDDLL'#| (5<:KL


1L@W@W
XCJr;   c           	        	
 t        j                  t              dz  dk(  fd       | j                  t              
t              dz  }
|z
  	t        j                  
|k\  
fd       t	        d D              r| }t        	
      D ]t  d
z
  dz
  z     dk  r*|j                      |j                        z         }dz      dk  sL|j                  d|j                     dz      z         }v |j                         S t        d 	       }t        |      D ]^  t              dz   dz  z
  	z         z   dz      z   }t        j                  |dk\  	fd       |j                  |       ` t        j                  || j                  | j                  | j                  t        |             S )	Nr  r   c                       dt                S )Nz1Length of pad must be even but instead it equals r  r  s   r9   rf   z'_constant_pad_nd_meta.<locals>.<lambda>"  s    CCH:N r;   c                  (    dt               d  dS )Nz`Length of pad should be no more than twice the number of dimensions of the input. Pad length is z while the input has rx  r  )l_inpr  s   r9   rf   z'_constant_pad_nd_meta.<locals>.<lambda>"  s"     225c(;P' r;   c              3   ^   K   | ]%  }t        |t        j                        xr |d k   ' ywr  )rq   rP   IntWithoutSymInt)rs   r  s     r9   ru   z(_constant_pad_nd_meta.<locals>.<genexpr>"  s)     
I:a//0;Q!V;
Is   +-r4   c            	      F    d z       d    ddz       d z    d	S )NzThe input size z, plus negative padding r   r4   zG resulted in a negative output size, which is invalid. Check dimension z of your input.r5   )r   r#  l_diffr  pad_idxs   r9   rf   z'_constant_pad_nd_meta.<locals>.<lambda>"  sG    ok&1*&=%>>V7|nE#gk"2!3 4117!OM r;   )r_   r   r   r   )rW   rh   r   r   r`  r   narrowr   r   r  r   r_   r   r   r   )r   r  r  l_padc_input	new_shapenew_dimr   r#  rl
  rh
  rm
  s    `     @@@@@r9   _constant_pad_nd_metars
  "  s    
LLC1N
 ++KEHMEU]F	LL	 
IS
IIvu% 	TA519q=)G7|a!..G}gmmA&6W&E 7Q;!#!..Aw}}Q/?#gPQkBR/RS	T }}[&)*I5\ 	"c(q1uk*fqj)CL83w{;KKqLM	
 	!	" ;;kk||))+E2 r;   r  r'  r(  c                    | j                         dk7  rt        d| j                          d      t        j                  j                  t        j
                  t        j                  fv fd       | j                  }j                  }j                  dk(  r|d   f}n$j                  dk(  r|d   |d   f}n
g ||d   }| j                  }| j                  ||      S )Nr  z'weight' must be 2-D, got z-Dc                  $    d j                    dS )NzhExpected tensor for argument #1 'indices' to have one of the following scalar types: Long, Int; but got r  r   r  s   r9   rf   zembedding.<locals>.<lambda>#  s    007hH r;   r   r4   r   )
r   r   rW   rh   r_   r   r  r   r   r   )	r  r   r  r'  r(  weight_shapeindices_shaper   r  s	    `       r9   	embeddingrx
  #  s     zz|q9&**,rJKK	LL%**ekk22	
 <<LMMM||q&21o%7			"1%|A7	5m5\!_5	IIY77r;   max_lengthspadding_valuec                    t        |      dk7  rt        dt        |       d      t        |      dk7  rt        dt        |       d      |d   j                  d   dz
  }|d   }||g| j                  dd  }| j                  |      S )Nr4   z&Only one jagged dim is supported, got z offsetsz max_lengthsr   )r   r   r   r   )r   r  ry
  rz
  r(  r  r  s          r9   $meta__jagged_to_padded_dense_forwardr|
  (#  s     7|q4S\N(K
 	
 ;14S5E4FlS
 	
 	
aAAAq,6<<+,LL))r;   c                 B    t        |       t               d               }|S )Nc                 8    t        | t        j                        S r^  rU   r   r  rl  s    r9   _fz)_create_unary_float_meta_func.<locals>._f@#  s      =JJ
 	
r;   rL   r%   funcr
  s     r9   _create_unary_float_meta_funcr
  ?#  *    4]
  

 Ir;   c                    | j                   s|j                   s|j                   rt        d      | j                         dk(  r,| j                  | j                        | j                  d      fS | j                  d      }| j                  d      }|j                  d      }| j                  |||      }|
r1|r| j                  |||      }||fS | j                  ||||      }||fS | j                  d      }||fS )NzP_native_multi_head_attention fake implementation does not support nested tensorsr   r4   )r   r   r   r   r   r   )r  r   r  r   r  r   r   r   r   r   need_weightsaverage_attn_weightsr   r(  T
output_dimr  attn_weightss                     r9    native_multi_head_attention_faker
  K#  s      #--5??!^
 	
 {{},eooa.@AA

1A

1A !!!$J__Q:.F !??1a3L L!!	 !??1h1=L L!! q)L!!r;   c                 B    t        |       t               d               }|S )Nc                 :    t        | |t        j                        S r^  r
  r=   s     r9   r
  z*_create_binary_float_meta_func.<locals>._f}#  s      q!@!M!M
 	
r;   r
  r
  s     r9   _create_binary_float_meta_funcr
  |#  r
  r;   c                      t                fd       } j                   d}||_         t        t        t        |            |      }|S )Nc                 `     | g|i |}t        | j                  |j                         | S rE   r  )r   rR   rV  r   rG   s       r9   _fnz#_register_inplace_meta.<locals>._fn#  s.    '''

CII6r;   rS   )r   r|   rL   getattrr.   )rG   r
  inplace_names   `  r9   _register_inplace_metar
  #  sO    
2Y 
 kk]!$LCL
4-l3
4S
9CJr;   c                 f    t        j                   j                  j                  k(   fd        g}t        t              rQj
                  dk7  r1t        j                   j                  j                  k(   fd       |j                         t        |dt        j                  iS )Nc                  <    dj                    d j                    S )Nr  z for `end`, but got dtype r   )r}   r~   s   r9   rf   zlerp.<locals>.<lambda>#  s    /%++.HT r;   r   c                  <    d j                    dj                    S )Nr  z for `weight`, but got dtype r   )r~   r  s   r9   rf   zlerp.<locals>.<lambda>#  s!    /%++6STZT`T`Sab r;   rM   )
rW   rh   r_   rq   r    r   r  rU   r   rQ   )r~   r}   r  rR   s   ``` r9   lerpr
  #  s     
LLsyy T 3<D&*%;;!LLv||+b 	F	=EE r;   )r  c                <    t        | ||t        j                        S r^  r_  r   tensor1tensor2r  s       r9   addcmulr
  #  s!     w0O0W0W r;   c                    t        j                  t        j                  |j                        xr t        j                  |j                         d        t        | ||t        j                        S )Nc                       y)N)zFInteger division with addcdiv is no longer supported, and in a future zErelease addcdiv will perform a true division of tensor1 and tensor2. z4The historic addcdiv behavior can be implemented as zA(input + value * torch.trunc(tensor1 / tensor2)).to(input.dtype) zfor integer inputs and as z6(input + value * tensor1 / tensor2) for float inputs. z?The future addcdiv behavior is just the latter implementation: z4(input + value * tensor1 / tensor2), for all dtypes.r5   r5   r;   r9   rf   zaddcdiv.<locals>.<lambda>#  rn   r;   r  )rW   rh   rP   r  r_   rU   r   rQ   r
  s       r9   addcdivr
  #  sb     
LL""7==1 6&&w}}5	
		
  w0O0W0W r;   c                     i } dD ]  }t         |   }|D ]  }|| vs||   | |<    ! | j                         D ]  \  }}t        |t        j                  j
                        r,t        |t              st        dt        |              |j                  t        j                  j                  j                        |       t        j                  j                  |j                         d      r|t         d   v st        | d      |j                   r|j                         dv rd|j                         v rt"        j%                  ||       d|j                         v rt&        j%                  ||       Hd	|j                         v rt(        j%                  ||       rd
|j                         v rt*        j%                  ||       t,        j%                  ||        y )N)r   post_autogradpre_autogradz$op_overload must be OpOverload, got CompositeImplicitAutogradr   z is a CompositeImplicitAutograd op, we shouldn't register meta function for it. Instead, we should let the decomposition run and write meta kernels for the base operators.>   aten::cloneaten::copy_aten::rot90aten::_to_copyaten::empty_stridedaten::constant_pad_ndaten::as_strided_scatterzmkldnn::zmkl::zonednn::zquantized::)r   itemsrq   rW   _opsHigherOrderOperatorr   r   r{   py_impl_CDispatchKeyr0   %_dispatch_has_kernel_for_dispatch_keyr  rv  is_view2_meta_lib_dont_use_me_use_register_meta_for_mkldnnimpl/_meta_lib_dont_use_me_use_register_meta_for_mkl2_meta_lib_dont_use_me_use_register_meta_for_onednn5_meta_lib_dont_use_me_use_register_meta_for_quantized'_meta_lib_dont_use_me_use_register_meta)activate_meta_tabletypregistryopoop_overloadrG   s         r9   activate_metar
  #  s    9 9-c2 	9C--+3C=#C(	99 /446 9NR
 k5::#A#AB+z2 6tK7H6IJ  	7EHH00556r:8899 ;
 8@@""m $; ; 
    	 [--//BGGUWXK,,..?DD[RTU{//11BGGUWX+"2"2"44EJJ 8<<["Ms9Nr;   rE   r  )F)        g      ?r|	  )NNNFr   r4   r   )Tr  )r,  )r5  T)FF)TT)r  )FTN)TFF)TF)r  )g      ?N)rA   r  )r5   r  r_  F)r5   r  FTN)Fr   FNFr   )NF)r   F)g      ?gUUUUUU?FN)NNNNN)r   NNr4   )r   r   r4   )NNF)r
  FFN)Nr
  FFN)r
  FNN)Nr
  FNNF)r
  FN)FN)NNNNNNN)FNNNN)NNNF)NNNNF)Nr   FNN)NNNN)r   TT)NNr   N)d   r   r   )r   )r  N)r   FF)r
  )NTTN(  r  collections.abcr   r   enumr   	functoolsr   typingr   typing_extensionsr   rW   torch._prims_commonr  rP   r	   r
   r   torch._decompr   r   r   r   
torch._opsr   torch._primsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    torch._prims_common.wrappersr!   r"   r#   r$   r%   r1  r&   r'   torch.fx.experimentalr(   r  torch.nn.functionalr)   r*   torch.utilsr+   rI   r,   r-   r  r.   libraryLibraryr
  r   r$  r&  r%  r:   r@   rL   rU   ra   rj   linspacelogspacer  r   taker  r   r   _standard_gammar   _transformer_encoder_layer_fwdr  rC  r[   r   r   r   cummaxcumminr   r   r   r   r  _fft_c2cr  r  r  _fft_r2cr(  randpermgenerator_outr,  r   r1  randintr:  r7  low_outr=  randr?  _philox_key_splitrI  _philox_key_fold_inrN  rb  rd  rh  rj  rk  _fft_c2rrp  r  r|  r  
unsqueeze_r  _sparse_semi_structured_linearr  r_   r  _sparse_semi_structured_mmr  _sparse_semi_structured_addmmr  _cslt_sparse_mmr  index_reducer  index_reduce_r  index_selectr  segment_reducer  r  	unary_outr  r   r  r  r  r  r  r  r  _assert_asyncr  msgr  _printr  _make_dep_tokenr  r   _functional_sym_constrain_ranger  r  (_functional_sym_constrain_range_for_sizer
  _functional_assert_asyncr  r   r  r   r'  r+  r1  r9  _linalg_eighr:  rC  _linalg_eigvalslinalg_eigvalsrF  
linalg_eigrJ  rN  rP  rV  r[  r_  re  rw  linalg_inv_exrz  linalg_ldl_factor_exrg   r  linalg_ldl_solver  	linalg_lur  linalg_lu_factor_exr  linalg_lu_solver  	lu_unpackr  r  	linalg_qrr  r  r  _linalg_svdr  r  rW  r  r  linalg_solve_triangularr  r  r  _linalg_detr  r  r"  r5  reflection_pad1dr:  replication_pad1drD  rO  reflection_pad1d_backwardrT  replication_pad1d_backwardrV  rf  reflection_pad2drh  replication_pad2drk  _weight_norm_interface_backwardrs  reflection_pad2d_backwardrP  replication_pad2d_backwardrz  r  reflection_pad3dr  replication_pad3dr  reflection_pad3d_backwardreplication_pad3d_backwardr  _pdist_forwardr  _pdist_backwardr  baddbmmr  	bernoullir  
bernoulli_r  r  r  poissonr  _fused_moving_avg_obs_fq_helperr  mmr  r  r  r  r	  miopen_batch_normr  convolutionr!  _convolutionr(  r
  _has_mkldnnr
  r)  _convolution_pointwiser0  _linear_pointwiser4  has_mklr
  r  _mkl_linearr8  r
  r9  qconv2d_pointwiseqconv_pointwiser  rF  binarybinary_tensorrQ  qlinear_pointwiserV  r[  linear_dynamic_fp16linear_relu_dynamic_fp16r]  r
  r^  
max_pool2dri  int4mm_packed_weight_cpury  r~  quantize_per_tensorr  
avg_pool2dr  r  avg_pool2d_backwardr  
avg_pool3dr  avg_pool3d_backwardr  _adaptive_avg_pool2dr  _adaptive_avg_pool3dr  _adaptive_avg_pool2d_backwardr  _adaptive_avg_pool3d_backwardr  r  adaptive_max_pool2dr  r  r  adaptive_max_pool3dr
  r  r  repeat_interleaver  rr   r  r  r  r   _unsafe_indexr<  convolution_backwardrL  addbmmrT  randint_likerW  _fused_adam__fused_adamw_rm  _fused_adamrs  _int_mmrz  _convert_weight_to_int4packr  #_convert_weight_to_int4pack_for_cpur  _weight_int4pack_mmr  _weight_int4pack_mm_for_cpur  r  r  r  _dyn_quant_pack_4bit_weightr  _dyn_quant_matmul_4bitr  _weight_int8pack_mmr  _cdist_forwardr  _cdist_backwardr  _embedding_bagr0  _embedding_bag_forward_onlyr3  r6  nansumr8  median	nanmedianr:  
dim_valuesr  r   r=  logical_not_r?  repeatrG  zero_rI  mul_Scalarlogical_and_logical_or_logical_xor_rL  div_rR  add_sub_r\  r|  subra  rounddecimalsrc  ri  
__rshift__rm  
__lshift__rq  zerors  rI  r{  fillr~  relu_r  	_add_relur  rrelu_with_noiser  rrelu_with_noise_functionalr  rrelu_with_noise_r  	index_put_unsafe_index_putr  masked_fill_r  _masked_scaler  masked_scatter_r  masked_scatterr  masked_scatter_backwardr  
index_put_r  r  bmmr  	dtype_outr  r  r  r  r  r  r  r  rc   max_pool2d_with_indices_backwardr  max_pool2d_with_indicesr  fractional_max_pool2dr  max_pool3d_with_indicesr   max_pool3d_with_indices_backwardr  r!  r#  r-  grid_sampler_2d_backwardr5  r;  r=  r@  r  onesrN  zerosrQ  select_scatterrU  slice_scatterrY  diagonal_scatterr\  rT  r   rd  rl  gatherrr  r  r  r  r  r  scatter_addr  scatter_add_r  r  r   r  r  value_reducer  scatter_r  #_scaled_dot_product_flash_attentionr  r  r  #_scaled_dot_product_cudnn_attentionr  0_scaled_dot_product_fused_attention_overrideabler  9_scaled_dot_product_fused_attention_overrideable_backwardr  ,_scaled_dot_product_flash_attention_backwardr  +_scaled_dot_product_flash_attention_for_cpur  4_scaled_dot_product_flash_attention_for_cpu_backwardr  *_scaled_dot_product_attention_math_for_mpsr  '_scaled_dot_product_efficient_attentionr  0_scaled_dot_product_efficient_attention_backwardr  ,_scaled_dot_product_cudnn_attention_backwardr  _flash_attention_forwardr  +_flash_attention_forward_no_dropout_inplacer  r  _flash_attention_backwardr  _efficient_attention_forwardr$  _efficient_attention_backwardSymIntr,  ru  
_scaled_mmrw  r  _scaled_mm_v2r  scatter_reducetwotwo_outr  scatter_reduce_r  multinomialr  r  r  r  _upsample_nearest_exact1dr  _upsample_nearest_exact2dr	  "_upsample_nearest_exact2d_backwardr
	  _upsample_nearest_exact3dr   r	  values_stabler	  r%	  _thnn_fused_lstm_cellr+	  rA	  rD	  rN	  rS	  rV	  rX	  argminrY	  r\	  topkrc	  _segment_reduce_backwardrg	  kthvaluerk	  r   r}	  rz	  r	  r	  pixel_shuffler	  r	  	bucketize
Tensor_outr	  
Scalar_outr	  histcr	  _upsample_bilinear2d_aa_upsample_bicubic2d_aa_upsample_lanczos2d_aar	   _upsample_bilinear2d_aa_backward_upsample_lanczos2d_aa_backwardr	  r	  r	  r  r	  searchsortedr	  r	  embedding_dense_backwardr	  _embedding_bag_backwardr	  _embedding_bag_dense_backwardr	  *_embedding_bag_per_sample_weights_backwardr	  isinr	  	polygammar	  _local_scalar_denser	  r	  r	  r
  rU
  _grouped_mmrW
  _scaled_grouped_mmrY
  _foreach_normr`
  _softmaxrd
  constant_pad_ndrs
  rx
  _jagged_to_padded_dense_forwardr|
  r
  _native_multi_head_attentionr
  r
  special_airy_aispecial_bessel_y0special_bessel_y1special_modified_bessel_i0special_modified_bessel_i1special_modified_bessel_k0special_modified_bessel_k1!special_scaled_modified_bessel_k0!special_scaled_modified_bessel_k1special_chebyshev_polynomial_tspecial_chebyshev_polynomial_uspecial_chebyshev_polynomial_vspecial_chebyshev_polynomial_w&special_shifted_chebyshev_polynomial_t&special_shifted_chebyshev_polynomial_u&special_shifted_chebyshev_polynomial_v&special_shifted_chebyshev_polynomial_wspecial_hermite_polynomial_hspecial_hermite_polynomial_hespecial_laguerre_polynomial_lspecial_legendre_polynomial_pr
  r
  r
  r
  lerp_addcmul_addcdiv_torch._refs.nn.functionaltorch._refs.specialr
  r5   r;   r9   <module>r     sS    .    '  # + +  " U      < 7 8 ) T]t_yy~~*/--*?*?PV*W ' %a )X"

8BF#3"4hr2v6F"FG 
3(* t}}-.
 

==7  /7t 		!!499==12'  3' $$,,d.B.B.F.FGH"  I" ++33++// ( !% )!	!! ! 	!
 ! ! ! ! ! 
! ! ! ! ! !  !!" #!$ %!& ,,
'!( Tz)!* +! !> !!))4+<+<+@+@AB%' %  C%$ t%%&I  'I 	[[$++//4;;+>+>P Xy! " !!))4+<+<+@+@ABI  CI.bV $s) 4  %%t}}'8'89:"<  ;"<J $s)  %%t}}'8'89:@
  ;@
F t}}**+"& 3 ,3 t}}$$% **
 &
 $$dll&6&678
 **  9&   $,,"6"678 **  9& 		!!499==12%)$tPT   3 t%%--.8 /8" t''//0	! 1	!%
P t##++, -
 t$$,,- .
 %%t}}'8'89:*Lv *LDI *Lc *LC *L  ;*LZ tzz!!" #:	( t&&' ( t223
 "&$())) ) 4-	)
 4Z) {{T!) 4)X t../
 %)	
  {{T!	 0> t112 	
$())
) ) 	) {{T!) 3)X t##$ $("8<,,8<\\8< 4-8< D=	8<
 {{T!8< 8< 8< 8< 8< %8<v t  (() 	I
	I		I 	I LL		I
 	I 	I 	I *	I t!!))* 	
			 	 LL		
 	 	 	 +	 t  (()' * ' t""**+
 "!! W
 W W d]	 W
 d] W d] W  W  W  W , WF   $(("4"456  7 txx||    $(("4"456  7 txx||  tzz!!"6 #6 tzz~~( (
 t!!))* + t!!%%& ' t{{""# $ t##++, * -* t''//0, 1, t33;;< =
 t008896 :6& t<<DDE F
 t,,001 2

F 
C 
    F  #  N (,


 !%
$V S C 
 
F 
$ 
 
"  	  	C  !!))4+<+<+H+HIJ]N+ s T  , K" $$,,d.A.A.E.EFGB B6 B  HB  !]N+6  , "&Q QF Q t**+) )F )4 )F )  ,) t""#J JF J4 JF J  $J t}})6 )$ )6 )  ) t$$%)6 )$ )6 )  &) t&&../&  T  0" 	$$,,d.M.M.Q.QR .f .6 .f . .d t!!))*&   + ))1143L3L3P3PQRT8V$ 	
  	
 666!" % S& %%--t/D/D/H/HIJ ''' '
 ' '  K'T &&(:(:;<S#s/3 f  fff>T8U   =6 ((00$2J2J2N2NOPT8V$ 	     	 
 666!"  % Q F $$,,d.B.B.F.FGH 444 4
 4 4 4  I4n t~~S#s 	$$$ $ 	$
 666!"$  $P tTz!2 * &&(:(:;<S#f C ffn8M   =4 $$,,d.B.B.G.GHIV[$1'v '%(F"G ' 2 J'$ t''(  	""" " $J	" )"J.
.
. 49d3i .".
.
. *. 66>	.(f V   t$$%
   777 	7
 7 TM7 	7 TM7 4-7 6666)*7 &7t ,,44d6R6R6V6VWX  	
   
$  Y4 t$$%S#4( +(
+(+( +( 	+(
 +( 66>+( ) &+(^ t''(
 )
 tzz
 WW	W W 	W
 W W  WtB%;P t$$%=  &= t%%&>  '>(@ t--.\S  /S t../\T  0T<~ t$$%=  &= t%%&>  '> ((00 &&..&&11''//''22	 \& &:@GF t$$%=  &= t%%&>  '> &&..&&11''//''22	 \(( ((V t""#f  v   $ t##$Pv PV P Pf PQW P  %P $$dll&6&678/0 '  9': &&(:(:;<&* I  =I
 t$$% & t~~ I !I
 $$dll&6&678"  9" t33;;< * =*. tww3U[[4/ 3  3,B
* .2\,,\LL\ IO\ #Y_	\
 3i#o\ \ \ IOd*\~Q t%%--.$$,,$$LL$$ ,,
$$ ,,%	$$
 $$$ $$ !&$$ $$ /$$N t''(#,,#LL# ,,
# I	#
 #Y# 3i# # I# # )#L t  (() ,,LL ,,
 I	
 #Y 3i  I      *8 	889>9N9N&&:6 599##::BBC D, 599##55==>S ?S
 xx:?--:O:O66;
7 
uyy}}00	1	 
2	
 :?9N9N&&:6 599##55==>599##33;;<599##33::;4 < = ?4l 599##55<<=599##55CCD E >< 599##55==>599##55<<= > ?> 599##55<<=599##55CCD$ E >$L 599##77??@599##<<DDE	 F A	 =BMM<Q<QVV=9 599&&112 
 3
8 599&&??@@ A@, t''(#<<# %#36#?D{{#
\\# )# t&&' M (Mb(<X t''//0E 1EP t UJ   UJp t''(\K(  )K(\ t((001 2" t((001@ 2@ t1199:F ;F, t112\P  3P
	
6 	
S 	
 t''(UI+  )+\ t001\H  2H$ t''(UI'  )'T t001\(  2(
 t%%,,-* .* $$dll&6&678
  9
 ##++T-@-@-D-DEF46 

c 

  G

 		&&..		0F0F0J0JKL  M" 

!!4#5#5#<#<=>A ?AH ))1123.H 4.Hb ##T[[__56./q '  7'0 !!(()* + !!))4+=+=+E+EFG  !
 H
2   (()*  !! +!H ~B  B* 0012 3& 889: ; (()*8 +8  0012< 3< >>?@< A<"3 "3 "3 "@F 0012D!'$D 3D4 ++,-; .;( (()*< +<, t""**+ & , &F t##$G  %G* t""**+
 	
`5 ,`5F t//7785 95
 ##T[[__56=$ =  7= ##T^^%;%;<=) >) !!					 Xy! "	 t  (() * t{{""#' $'& tzz!!" # 				!!  !! 		  $))"2"234 5& 									**Z  

""DJJ$7$789 :
" &&(>(>?@ A &&(>(>?@ A tyy  !" "" 

!!4::#4#456 7 		  $))"2"234" 5" tzz!!" # t~~$$%F   & %%&'RV"  (" 0012RV; 3; &&'(KO ) &&(>(>(F(FGH" I" t  ''( )
 t!!))* + t##$	 %	 t""#6  $6 t++,! -! t&&' (,^   $((,,/05  15  2 234J  5J6?p #-[[[ 	[ 		[
 	[ 	[ 	[ 	[ 	[ 	[ 	[ [ [ [ [  ![" #[$ %[& '[( )[* +[, -[|;4|383838 38 		38
 	38 	38 	38 	38 	38 	38 	38 	38 38 38 38  !38" #38$ %38& '38lI2X t445\(  6(V t++334 # 5#L t))112Q 3Qh t++,UI _  -_D t445\\  6\~%
V %
6 %
Pt  v 3 $ t,,445# 6#$ t##$8  %8" t,,-\;'! ( .!, 		!!"#. $. t&&' ) ()X 		!!499==12   3( 

""DJJNN34   5( t""**+& ,& t!!))*& +& t$$,,-& .&. C d $/
  t{{""#' $'6
 
%R4 t''(& )&
 t  ! "
 !!	 & & ""	 88@@AB
 #<<	< < 	<
 < < 4<< C<~ 88BBCD #	  }	
 } }    4< E4S#X, 889: #((	( ( }	(
 ( ( ( ( 4<( ;(V EEFG
  $#00	0 0 }	0
 0 0 0 4<0 H0f tMMN& %222 
2 	2
 2 $Z2 
2 2 2 2 2 2 2 2 2  !2$ 4<%2 O26 99( """ 
" 	"
 
" " " " " " " " " " 4<"
". 88 #	  	
  } 4<
: AA  $!"!"!" 
!" 	!"
 
!" !" !" !" }!" 4<!"
!"H ??@A
  $"&--	- - }	-
 - - 4-- 4<- - 66>- B-` <<=> ))))	)) )) }	))
 )) )) 4<)) ?))X ==" 4-4-4- 
4- 	4-
 }4- 
4- 4- 4- 4- 4- $Z4- 4- 4<4-
4-n 99* !""" 
" 	"
 
" " " " " " " " " " "  4<!"
"0 %%--  #'$(#"&!%!#JJ	J J }	J
 }J J J J J J 4<J DjJ TzJ }J 4-J  $!J" d
#J
JZ @@HHIJ #'$(#"&!%!%&	&& 
& 	&
 }& }& & & & & & 4<& Dj& Tz& }&  4-!&" $#&$ d
%& K&R --7789 #'$(#"&%''	' ' }	'
 }' ' ' ' ' ' }' }' }' 4<' Dj'  Tz!'" }#'$ 4-%' :'T &&( #'$(#,,, 
, 	,
 
, , , , , , , , , , 4<,  Dj!," Tz#,
,4 ))   %%)""/S/S	/S /S 4-	/S
 4-/S 4-/S */S */S /S /S /S 4</S d]/S tm/S t/S
/Sd *** !%"'%474747 
47 	47
 4-47 4-47 4-47 ,,47 ,,47 47 47 47 47 47 47  4<!47" $J#47$  %47
47x !%(,$( AY
,,AY
,,AY \\AY \\	AY
 ,,
AY ,,%AY {{T!AY AYH ''() !%(,$( 
,,
,, \\ \\	
 ,,
 ,,% {{T!  *, !%$(*.*. {C
,,{C
,,{C %,,{C %	{C
 %,,{C %{C ,,
{C {{T!{C K 4'{C K 4'{C {C| ""**+, !%'+)- 
,,
,, %,, %	
 K  %,, % K  ,,
 ++$ 3i$&  -: ##'')<)<)D)DEF&  G&
 t##''( )
   (($*:*:*>*>?@	 	  A	,* 	$$d&D&D&L&LM

 	$$d&D&D&L&LM. ((00//77 "!#,- u||+, dl	
 dl: 	$$d&D&D&L&LM

 									*$N t))112
  3 t&&'4/ (4/n t&&') ()X t$$,,-$% .$%N


 ##T[[%8%89:4 ;4 t!!))* + tyy  !
Q "
Q t,,-LP  ." %%t}}';';<=Xy!K " >K  #("9"9 Q t77??@	* A	* t##++,	0 -	0 t!!))*+ ++\ t--556)F 7)FX %%t~~'@'@AB27u   C %%t~~'@'@AB
 



 	

 
 C
 

|E  E4 $$,,##++##++ & --55,,44 8 t>>FFG H$ '')<)<=>"  ?" uyy~~(() *: uyy~~  !6 "6( t  !
 
	-K  "-K` t,,- . t++, '
 -'
T t112  3, t>>?  @. tyy8=e 8  8  t~~6c 6 6F 6  6 t''(I& I )I tyy"v "& "  " t||6& 6V 6  6'^ (,$( VKVKVK \\D VK \\D 	VK
 4-VK 4-VK ,,%VK {{T!VK VKr t  $( 4- 4-	
 {{T!   !& ''() !% $(,$( <<<< \\ \\	
 ,,
 ,,
 ,,% {{T!  *6 t!!(() *  t}}v C       t##$3  %3l t~~ $888 8 	8
 8 8  8< t33;;<
 	**&\* c* 	* =*, t00889 
-" :-"` d22 3 d44 5 d44 5 d== > d== > d== > d== > dDD E dDD E tBB C tBB C tBB C tBB C tJJ K tJJ K tJJ K tJJ K t@@ A tAA B tAA B tAA B tyy  $ t||./    t||./   , 	tyy)!$,,/!$,,/
    ENP r;   