
    ^j                     X3   U d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	m
Z
 d dlmZ d dlZd dlmZ d dlmZ d dlZd dlmZmZmZ d dlmZmZmZmZmZ d dl m!Z! d dl"m#Z# d d	l$m%Z% d d
l&m'Z' d dl(m)Z) d dlm*Z*m+Z+m,Z,m-Z- d dl.m/Z0 d dl1m2Z2m3Z3 ddl4m5Z5m6Z6 ddlm7Z7m8Z8m9Z9  e
d      Z: ed      Z;ejx                  jz                  ejx                  j|                  z  Z?e	e@d<    ej                  eB      ZCej                  j                  ZEej                  j                  ZFej                  j                  ZGej                  j                  ZHej                  j                  ZI eg eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj^                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  e'eEj                  eEj                  eEj                  eEj                  j                  eEj                  j                  eEj                  eGj                  eHj                        Zwi  e       ewZxeEj                  eEj                  eEj                  eEj                  j                  eEj                  eEj                  eEj                   eEj8                  eEj^                  eEj                  eEj                  eEj                  eEj                  eEj
                  eEj                  j                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  j                  eEj                   eEj"                  eEj$                  eEj&                  j                  eEj(                  eEj*                  eEj,                  gZeejx                  j2                  ejx                  j|                  z     e@d<    eexe       de?ee?   z  deee;e:f   gee;e:f   f   fdZ eeEj*                  j                  g      dej                  dej                  dedej                  fd       Z eeEj*                  j                  g      dej                  dej                  dej                  dej                  fd       Z eeEj8                  g      dej                  dej                  d eqd!eqd"edej                  fd#       Z eeEj^                        d$ej                  d%ej@                  dej                  dz  d&ej                  dz  d'edeej                  ej                  ej                  f   fd(       Z eeEjF                  j                  g      ddd)d*ejH                  d+ejJ                  jL                  dz  d,ejJ                  jL                  dz  ddfd-       Z eeEjN                  g      e	 	 dd.ej                  d+ejJ                  jL                  dz  d,ejJ                  jL                  dz  dej                  fd/              Z eeEj
                  g      ed.ej                  dej                  fd0              Z eeEjP                  g      d1eeqejH                  z     d2ejJ                  jL                  d3edej                  fd4       Z eeEj                  g      dd5d.ej                  d6eqd7ej                  d8ej                  d9ejJ                  jL                  dej                  fd:       Z eeEjR                  j                  g      d1eeqejH                  z     d;eeq   d3edej                  fd<       Z eeEjT                  g      dej                  d$ej                  dej                  d=eeq   d>eqeeq   z  d?eqeeq   z  d@eqeeq   z  dAedBeeq   dCeqdDee   deej                  ej                  ej                  f   fdE       Z eeEjV                  jX                  g      dd.ej                  dFeqdej                  fdG       Z eeEj\                  g      e	 ddHej                  dIej                  dJej^                  dz  dej                  fdK              Z eeEj`                  g      e	 	 	 ddHej                  dLej                  dMej                  dJej^                  dz  dNejJ                  jL                  d9ejJ                  jL                  dej                  fdO              Z eeEjb                  g      e	 ddHej                  dPej                  dJej^                  dz  dej                  fdQ              Z eeEjd                  j                  g      	 ddReej                     d6eqdej                  fdS       Z eeEjf                  g      d.ej                  dej                  fdT       Z eeEjh                  g      dd5d.ej                  dUej                  d9ejJ                  jL                  dz  dej                  fdV       Z eeEjj                  g      dHej                  dej                  fdW       Z eeEjl                  eEjn                  g      dHej                  dej                  fdX       Z eeEjp                  eFjp                  g      dHej                  dYej                  dej                  fdZ       Z eeEjr                  eFjr                  g      dHej                  dYej                  dej                  fd[       Z eeEjt                        	 	 ddHej                  d6eqdz  d]edej                  fd^       Z eeEjv                        	 	 ddHej                  d6eqdz  d]edej                  fd_       Z eeEjx                  g      dHej                  d6eqdeqd`eqdej                  f
da       Z eeEjz                  j                  g      dHej                  d1eeqejH                  z     dej                  fdb       Z eeEjz                  j^                  g      dHej                  dcej^                  dej                  fdd       ZdHej                  deej@                  ej                  f   fdeZ eeEj                        dddd\d\ej                  dfdHej                  d2eqez  dcej^                  dz  dgej                  dz  dhej                  dz  diedjedkej                  dej                  fdl       Zddej                  dmdnedoej                  f   dHej                  dcej^                  dz  dhej                  dz  dkej                  d3edej                  fdpZ ej4                  eEj                  ge!      dHej                  d3edej                  fdq       Z ej4                  eEj                  ge!      dHej                  d3edej                  fdr       Z ej4                  eEj                  j                  ge!      dHej                  dseqd3edej                  fdt       Z ej4                  eEj                  j                  ge!      dHej                  dueqdseqd3edej                  f
dv       Z eeEj                  j                        dseqd1eeqejH                  z     d3edej                  fdw       Z eeGj                  j                        	 dd$ej                  dej                  d&ej                  dz  dej                  fdx       Zu eeHj                  j                        d$ej                  dyej                  dzej                  dej                  d{ej                  d|ej                  d&ej                  d}ej                  d~ej                  deqdej                  fd       Zv eej                  j                  j                        dej                  dej                  fd       Z eeEj                  g      e	 	 	 ddej                  dej                  deqdeqdedej                  fd              Z eeEj                  j                        deej                     deej                     dejJ                  jL                  deej                     fd       Z eeEj                  j                        deej                     deej                     deejJ                  jL                     deej                     fd       ZeEj                  j                  j                  ej                  j                  j                         eeEj                        d$ej                  dej                  d&ej                  dz  dej                  dz  dej                  dz  dedededeej                  ej                  ej                  f   fd              Ze j                  deeedoef   f   fd       Zdeeedoef   f   fdZ eeEj                        dHej                  dej                  dej                  dej                  fd       Z eeIj                  j                        d$ej                  deqdeqd'edcej^                  deej                  ej                  f   fd       Z eeEj                        	 ddHej                  d7ej                  dej                  dedej                  f
d       Z eeEj                        	 ddHej                  d7ej                  dej                  dedej                  f
d       Z eeEj                  j                        edej                  dej                  d6eqdej^                  dej                  f
d              Z~ eeEj                        dddHej                  d6eqd7ej                  dej                  dededej                  fd       Zd.ej                  deeq   d>eqeeq   z  dz  d?eqeeq   z  d@eqeeq   z  ded6eqdeej                  ej                  f   fdZ eeEj                        	 	 	 	 dd.ej                  deeq   d>eqeeq   z  dz  d?eqeeq   z  d@eqeeq   z  dedeej                  ej                  f   fd       Z eeEj                        	 	 	 	 dd.ej                  deeq   d>eqeeq   z  dz  d?eqeeq   z  d@eqeeq   z  dedeej                  ej                  f   fd       Z eeEj                  j                        d\d\ddddej                  dHejJ                  jL                  dedededz  dej                  dz  dej                  fd       Z eeEj                  j                        d\d\ddHejJ                  jL                  dej                  dededej                  f
d       Z ej4                  eEj                  e!      	 	 	 	 ddHej                  dej                  dedededej                  dz  deej                  ej                  f   fd       Z eeEj                  j                        	 ddej                  deqdz  dej                  fd       Z	 	 	 	 	 dd$ej                  dej                  d&ej                  dz  d>eeq   d?eeq   d@eeq   dCeqdej                  fdZy)    N)Callable)Any	TypeAliasTypeVar)	ParamSpec)core_aten_decompositionsget_decompositionsremove_decompositions)_grid_sampler_2d
_index_addembedding_dense_backwardpw_cast_for_opmath"pw_cast_for_opmath_non_tensor_args)extra_random_decomps)counters)	is_fbcode)	out_dtype)pad_listlike)elementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDsuggest_memory_formattype_to_dtype)native_layer_norm)guard_or_falsestatically_known_true   )configinductor_prims)is_gpu,needs_fallback_due_to_atomic_add_limitationsuse_scatter_fallback_T_P_GenericOperatordecomps_to_excludeopsreturnc                     t        | t              r| n| gD ]!  }|t        v st        j	                  d|        # t        j                  | t              S )Nzduplicate decomp: %s)
isinstancelistdecompositionslogwarningdecompregister_decomposition)r&   ops     h/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/decomposition.pyr/   r/      sL      T*c 5KK.45 ((n==    startendweightc                     t        |       }|t        j                  k7  r|j                  |      }|| z
  }|dk\  s|dk  rt        j                  ||d|z
         S t        j                  | ||      S )Nmemory_format      ?g            ?alpha)r   torchcontiguous_format
contiguousadd)r3   r4   r5   fmtdiffs        r1   _lerp_scalarrC      sn      
&C
e%%%nn3n/;D}$yydC&L/::99UD//r2   c                 ,   t        |       }|t        j                  k7  r|j                  |      }|| z
  }|j	                         dk\  }d|z
   }t        j
                  |||      }t        j
                  |||       }t        j                  |||d      S )Nr7   r9   r:   r   )value)r   r=   r>   r?   abswhereaddcmul)	r3   r4   r5   rA   rB   maskneg_omwwbases	            r1   _lerp_tensorrM      s      
&C
e%%%nn3n/;D::<3DfoGD'6*A;;tS%(D==q$a00r2   grad_outputindicesnum_weightspadding_idxscale_grad_by_freqc                 D    | j                   rt        S t        | ||||      S N)is_xpuNotImplementeddecomp_embedding_dense_backward)rN   rO   rP   rQ   rR   s        r1   _embedding_dense_backwardrX      s,     *Wk;8J r2   inputnormalized_shapebiasepsc                 D    | j                   rt        S t        | ||||      S rT   )is_mtiarV   decomp_native_layer_norm)rY   rZ   r5   r[   r\   s        r1   _native_layer_normr`      s%     }}#E+;VT3OOr2   )minmaxsymbolra   rb   c                     y rT    )rc   ra   rb   s      r1   sym_constrain_range_for_sizerf      s     r2   xc                 R    || j                  |      } || j                  |      } | S rT   )	clamp_min	clamp_max)rg   ra   rb   s      r1   clamprk      s/     KK
KKHr2   c                 J    | d| j                         j                         z   z  S )Nr   )negexp)rg   s    r1   siluro      s      AEEGKKM!""r2   size
fill_valuekwargsc                     |j                  d      }|.t        t        |            |d<   t        j                  | |fi |S t
        S )Ndtype)getr   typer=   fullrV   )rp   rq   rr   rt   s       r1   rw   rw      sF     JJwE}'Z(89wzz$
5f55r2   r;   dimindextensorr<   c                ~    t               s#| j                  t        j                  k(  rt        S t        | |||d|      S )NF)inplacer<   )r   rt   r=   bfloat16rV   r   )rg   rx   ry   rz   r<   s        r1   	index_addr~     s3     ;177enn4!S%eLLr2   physical_layoutc                 H   t        |      t        t        t        |                  k(  }|rt        j                  | fi |S dgt        |       z  }t        |      D ]
  \  }}|||<    t        j                  |D cg c]  }| |   	 c}fi |j                  |      S c c}w Nr   )r*   rangelenr=   empty	enumeratepermute)rp   r   rr   is_identitypermpls          r1   empty_permutedr   #  s     '4c/6J0K+LLK{{4*6**sSYo. 	DAqDG	{{_=DG=HHPPQUVV=s   <B
bias_sizesstridepaddingdilation
transposedoutput_paddinggroupsoutput_maskc                    |
d   rt        | j                  j                        st        S t	        |j                  d      dk(        r#t	        |d   dk7        r| j                  |      }n;t        j                  | dgt        t        d| j                                     z         }t        j                  | |||||||||	|
d   |
d   dg      \  }}}|||fS )N   r   r   F)r   devicerv   rV   r   rp   	new_zerosatensumr*   r   rx   convolution_backward)rN   rY   r5   r   r   r   r   r   r   r   r   	grad_biasgrad_inpgrad_weight_s                  r1   r   r   4  s     q>(:(:(?(?!@UZZ]a/05J16  ))*5	HH[1#U1koo>O5P0Q*QR	#88	QQ/ Hk1 k9--r2   decimalsc                 H    d|z  }t         j                  | |z        d|z  z  S )Ng      $@r:   )r   round)rg   r   ten_pow_decimalss      r1   	round_decr   [  s,    X~::a**+s5E/EFFr2   selfbatch2r   c                 $   t        | j                  d   dk(        r.t        |j                  d   dk(        r| |z  j                         S t        j                  r| j
                  j                  dvrkt        | j                  d   dk(        st        |j                  d   dk(        r5| j                  d      |j                  d      z  j                  d      }|S | j
                  j                  dk(  rt        | j                  d      dk(        ryt        |j                  d      dk(        r\t        d   dxx   dz  cc<   t        j                  | j                  d      |j                  d      z  dd	
      j                  d      S t        S )Nr   r   cpumpsrx   r   inductordecompose_bmmTrx   keepdim)r   shaper?   r   coordinate_descent_tuningr   rv   	unsqueezer   rp   r   r=   squeezerV   )r   r   r   outs       r1   bmmr   a  sW    TZZ]a/05JQ16 v))++ ''DKK,<,<N,R A!!348MLLOq 9
 >>"%(8(8(;;@@Q@GCJ{{5  1!238MKKOq 9
 Z 1Q6199Q&.."44!Til r2   mat1mat2betac                    |j                   j                  dvr{t        |j                  d      dk(        r^t        |j                  d      dk7        rAt        |j                  d      dk7        r$t        d   dxx   dz  cc<   ||z  }||z  || z  z   S | j                   j                  dk(  r7t        |j                  d      dk(        rt        |j                  d      dk(        rgt        d   dxx   dz  cc<   t        j                  |j                  d      |j                  d      z  dd	      j                  d      }||z  || z  z   S t        |j                  d      dk(        ryt        |j                  d      d
k        r\t        |j                  d      d
k        r?t        d   dxx   dz  cc<   |j                  |z  j                  dd	      }||z  || z  z   S t        S )Nr   r   r   r   r   decompose_addmmr   Tr      )r   rv   r   rp   r   r=   r   r   r   r   TrV   )r   r   r   r   r   r<   r   s          r1   addmmr     s    {{~-!$))B-1"45%diila&78%diila&78Z !23q83+C3;,,{{5  1!238MIIbMQ9
 Z !23q83))Q$,,r"224il  3;,,!$))A,!"34tyy|r12tyy|r12Z !23q8366D=%%!T%:C3;,,r2   input2c                    t         j                  r| j                  j                  dvrit	        | j
                  d   dk(        st	        |j
                  d   dk(        r3| j                  d      |j                  d      z  j                  d      S t	        | j                  d      dk(        rt	        | j                  d      dk7        rt	        |j                  d      dk7        r| j                  j                  dvs| j                  j                  dk(  rt	        | j                  d      dkD        rt	        |j                  d      dk(        ri| j                  |j                  k(  rPt        t        j                  |       t        j                  |      z   dk        rt        d	   d
xx   dz  cc<   | |z  S | j                  j                  dk(  rt	        | j                  d      dk(        ryt	        |j                  d      dk(        r\t        d	   d
xx   dz  cc<   t        j                  | j                  d      |j                  d      z  dd      j                  d      S t        S )Nr   r   r   r   r   r   r       r   decompose_mmTr   )r   r   r   rv   r   r   r   r   rp   rt   r   r=   numelr   r   rV   )r   r   r   s      r1   mmr     s    ''DKK,<,<N,R A!!348MLLOq 9
 NN1%(8(8(;;@@Q@GG 	diimq01!$))A,!"34!&++a.A"56;;>1KK%%diilQ&67%fkk!n&9:v||+D 1EKK4G GBNOZ 0A50&= {{5  1!238MKKOq 9
 Z 0A5099Q&.."44!Til r2   tensorsc                    dt         j                  dt        ffd}t        t	        ||             t              dk(  rRt        | dt        j                  id   }d   }||j                  k(  r|j                         S |j                  |      S dt              cxk  rt        |       k  r#n n t        j                  j                        S t              dkD  rt        fdD              rd   }t        |j                         }dk  rt        |j                         z   n|j#                  t                      |j%                        j&                  | j)                  dz         j                         S t*        S )	Nrg   r'   c                     t        | j                        dk(  rt        | j                  d   dk(        ryt        | j                        k  rt        | j                     dk(        ryy)Nr   r   FT)r   r   r   )rg   rx   s    r1   non_empty_tensorzcat.<locals>.non_empty_tensor  sU    " qww<1
a!@QWW.1B"Cr2   r   type_promotion_kindr   )rt   c              3   ,   K   | ]  }|d    u   yw)r   Nre   ).0tfiltered_tensorss     r1   	<genexpr>zcat.<locals>.<genexpr>
  s!      )%&a  )s   )r=   Tensorboolr*   filterr   r   r   DEFAULTrt   clonetor   catdefaultallr   insertr   expandflattenrV   )r   rx   r   promoted_dtype
filtered_tinpr   r   s    `     @r1   r   r     s   
ELL T 2 F#3W=>
!+
 ? G G
  &a(
 !1!11 	
 ^4	

 
S!"	1S\	1xx 0#66 q S )*:) & q!SYY&)AgcC		N"3S#./0(s}}S!((%088cAgFLLNN r2   c                 "   | j                         ret        j                  t        j                  | j                        t        d      t        j                  | j                  | j                              S t        | t        j                        \  }}t        j                  t        j                  || j                        }t        j                  | dk  |d      }t        j                  t        j                  |       t        d      |      S )Nnan)r   rt   r   r   g        )
is_complexr=   rG   isnanrealfloatatan2imagr   r   INT_TO_FLOATscalar_tensormathpir   )rg   r   rt   r   rets        r1   angler     s    ||~{{KKuu{{166166/J
 	
 "	;HHHAu 
		TWWE!((	CB
++a!eR
%C;;u{{1~uU|S99r2   yc                   t        j                  |       xr | j                         }t        j                  |      xr |j                         }|r|st        S dt         j                  dt
        fd}d}| j                  dk(  r|j                  dk(  rd}| j                  dk(  r| j                  d      } |j                  dk(  r|j                  d      }|}|||z  }t        j                  | j                  |j                        } ||       s ||      rt        S dt         j                  dt         j                  fd}	| dz   } |dz   } |	| j                  | j                  j                              }
 |	|j                  |j                  j                              }t        j                  |
|z   d	
      j                  |      }|r|d   S |S )Nrz   r'   c                 N    | j                   dk(  ry| j                         d   dk7  S )Nr   Fr   r   )ndimr   )rz   s    r1   _requires_fallbackzadd.<locals>._requires_fallback8  s'    ;;!}}r"a''r2   Fr   Tr   c                     | j                   ^ }}|dz  dk7  rt        d      g ||dz  d}| j                  |      }|S )zNReshape tensor from [*initial_dims, last_dim] to *initial_dims, last_dim/2, 2]r   r   zQThe size of the last dimension must be even to reshape it to [..., last_dim/2, 2])r   AssertionErrorview)rz   initial_dimslast_dim	new_shapereshaped_tensors        r1   reshape_tensor_complexz#add.<locals>.reshape_tensor_complexQ  s^     #),,x a<1 c 
 6l5HM515	 ++i0r2   )	start_dim)r=   	is_tensorr   rV   r   r   r   reshapepromote_typesrt   r   r   r   )rg   r   r<   x_is_complex_tensory_is_complex_tensorr   output_size_zerozcomplex_typer   
x_reshaped
z_reshapedresults                r1   r@   r@   +  s     //!,?//!,?&9(5<< (D ( vv{qvv{vv{IIaLvv{IIaL	AAI&&qww8L! 21 5u||  $ 	
AA	AA'qvv||(<=J'qvv||(<=J]]:
2bAFF|TFayMr2   c                 2    | j                         rt        S | S rT   )r   rV   r   s    r1   conj_physicalr  o  s    Kr2   c                     | S rT   re   r  s    r1   liftr
  v  s    Kr2   otherc                 b    t        j                  t        j                  |      || kD  z  | |      S rT   r=   rG   r   r   r  s     r1   fminr  {  '    ;;u{{5)UT\:D%HHr2   c                 b    t        j                  t        j                  |      || k  z  | |      S rT   r  r  s     r1   fmaxr    r  r2   Fr   c                 x    | j                   t        j                  k(  rt        j                  | ||      S t        S Nr   )rt   r=   r   anyrV   r   rx   r   s      r1   amaxr    -     zzUZZyy388r2   c                 x    | j                   t        j                  k(  rt        j                  | ||      S t        S r  )rt   r=   r   r   rV   r  s      r1   aminr    r  r2   lengthc                 n    t        j                  | |||      j                  t         j                        S )Nr7   )r=   narrowr   r>   )r   rx   r3   r  s       r1   narrow_copyr    s4     <<c5&177-- 8  r2   c                 J    t         j                  | |      j                         S rT   )r   r   r   )r   rp   s     r1   view_copy_defaultr     s    
 99T4 &&((r2   rt   c                 @    | j                         j                  |      S rT   )r   r   )r   rt   s     r1   view_copy_dtyper"    s    
 ::<U##r2   c                     t        j                  |       \  }}|D cg c]  }| j                  |    }}dgt        |      z  }t	        |      D ]
  \  }}|||<    ||fS c c}w r   )utils3compute_elementwise_output_logical_to_physical_permr   r   r   )r   r   r   r   r   permutationr   s          r1   _get_shape_permutation_liker'    sx     RRSWXOQ$34qTZZ]4E4#E
"K/* 1A ; 5s   A )rt   layoutr   
pin_memoryrequires_gradr8   r(  r   r)  r*  r8   c          	         || j                   n|}|| j                  n|}|| j                  n|}|t        j                  k7  r8t        j
                  | j                  ||||||      }|j                  |      S |t        j                  k(  sJ t        |       \  }	}
t        j
                  |	||||||      }|
t        t        t        |
                  k(  r|S |j                  |
      j                         S )N)rt   r(  r   r)  r*  r7   )rt   r(  r   r=   preserve_formatrw   r   r   stridedr'  r*   r   r   r   r   )r   rq   rt   r(  r   r)  r*  r8   r  r   r&  s              r1   	full_liker.    s      -DJJUE"NT[[F"NT[[F---JJ!'
 yy}y55 &&&8>{!'
 $uS%5677M~~k*0022r2   )rt   r   r8   rand_fn.c                j   ||j                   n|}||j                  n|}|t        j                  k7  r& | |j                  f||d|j                  |      S t        |      \  }} | |f||d|}|t        t        t        |                  k(  r|S |j                  |      j                         S )Nr   r7   )rt   r   r=   r,  r   r   r'  r*   r   r   r   r   )	r/  r   rt   r   r8   rr   r   r&  r  s	            r1   
_rand_liker1    s      -DJJUE"NT[[F---JJ

 	

 "="
)	* 5T:E; 	F d5[!1233>>+&,,..r2   c                 8    t        t        j                  | fi |S rT   )r1  r=   randr   rr   s     r1   	rand_liker5    s    ejj$1&11r2   c                 8    t        t        j                  | fi |S rT   )r1  r=   randnr4  s     r1   
randn_liker8    s    ekk42622r2   highc                 v    t        t        j                  t        j                  j
                  d|      | fi |S r   r1  	functoolspartialr   randintlow)r   r9  rr   s      r1   randint_liker@    s-    i''(8(8!TBDSFSSr2   r?  c                 v    t        t        j                  t        j                  j
                  ||      | fi |S rT   r;  )r   r?  r9  rr   s       r1   randint_like_lowrB  "  s/     i''(8(8#tDdUfUUr2   c                 F    t        j                  j                  d| |fi |S r   )r   r>  r?  )r9  rp   rr   s      r1   r>  r>  )  s"     <<AtT4V44r2   c                     t         j                  j                  j                  |      }t         j                  j                  j	                  | |||j                         d         S r   )r=   r&   
_quantized$wrapped_fbgemm_pack_gemm_matrix_fp16!wrapped_fbgemm_linear_fp16_weightrp   )rY   r5   r[   packed_weights       r1   #linear_dynamic_fp16_unpacked_weightrI  2  sO     II((MMfUM99AA}dFKKM!$4 r2   input_scaleinput_zero_pointweight_scaleweight_zero_point	out_scaleout_zero_pointout_channelc
           	          t         j                  j                  j                  ||||      }
t         j                  j                  j	                  | |||
|||	      S rT   )r=   r&   rE  _wrapped_linear_prepack#_wrapped_quantized_linear_prepacked)rY   rJ  rK  r5   rL  rM  r[   rN  rO  rP  rH  s              r1   wrapped_quantized_linearrT  >  s\     II((@@/M 99CC r2   packedc                     dt         j                  dt         j                  fd} || dddf         } || ddd f         }| dd df   j                  t         j                        |z  |z   S )Nu8r'   c                      fddD        \  }}}}t         j                  dk(  r4||dz  z   |dz  z   |dz  z   j                  t        j                        d   S |dz  |dz  z   |dz  z   |z   j                  t        j                        d   S )Nc              3   f   K   | ](  }d |f   j                  t        j                         * yw.N)r   r=   int32)r   nrW  s     r1   r   zPq_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32.<locals>.<genexpr>\  s&     GQbajmmEKK0Gs   .1)r   r   r      little   r      rZ  )sys	byteorderr   r=   float32)rW  rg   r   r  rK   s   `    r1   bitcast_u8_to_f32z=q_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32[  s    G,G
1a==H$aLAG,R8>>u}}MiXX"Wb)Q!V4q8>>u}}MiXXr2   .i)r=   r   r   rc  )rU  rd  scalesoffsetss       r1   "q_embedding_bag_byte_unpack_decomprh  Y  sz    Yell Yu|| Y vc2b5j12FsBCx 01G#ss(u}}-6@@r2   agridinterpolation_modepadding_modealign_cornersc                     | j                   t        j                   d      k(  xr' |dk(  xr  | j                  t        j                         }t	        | |||||      }|S )Nr   r   r7   )rj  rk  rl  rm  _expand_grid)r   r=   is_contiguousr>   decomp_grid_sampler_2d)ri  rj  rk  rl  rm  ro  outputs          r1   grid_sampler_2drs  g  sm     	
ELL'' 	C!#	COO%*A*AOBL $	-!#!F Mr2   start_tensorsend_tensorsc           	          t         j                  j                  | t         j                  j	                  t         j
                  j                  ||       |            S rT   )r   _foreach_addList_foreach_mulScalar_foreach_sub)rt  ru  r5   s      r1   _foreach_lerp_scalarr|    sL     !!  "";>	
 r2   scalarsc           	          t         j                  j                  | t         j                  j	                  t         j
                  j                  ||       |            S rT   )r   rw  rx  ry  
ScalarListr{  )rt  ru  r}  s      r1   _foreach_lerp_scalarlistr    sL     !!$$"";>	
 r2   running_meanrunning_vartrainingexponential_average_factorepsilonc           
          t         j                  | |||||||      \  }}	}
|r||	|
fS ||j                  d      |j                  d      fS )Nr   )r   native_batch_normr   )rY   r5   r[   r  r  r  r  r  ri  bcs              r1   miopen_batch_normr    sl     $$"	GAq! 1ay	 r2   c                      i t         t        S rT   )r+   r   re   r2   r1   fast_random_decompsr    s    5n5 455r2   c                      t         j                  rt        S t         j                  rHt        j	                  t
        j                  j                  j                  j                  d       t        S t               } | S )z"decomps can change based on configN)r   fallback_randomr+   "fallback_embedding_bag_byte_unpackpopr=   r&   	quantizedembedding_bag_byte_unpackr   r  )r  s    r1   select_decomp_tabler    sT    00599..HHPPRVW "FMr2   rI   sourcec                    ddl m}m}  || j                  |j                        rt
        j                  | |g      \  } }|j                  d      j                  d      dz
  }d | ||fD        \  }}}t
        j                  |||gd      }	t        j                  ||	|      j                  | j                        S t        S )Nr   )BackendFeaturehas_backend_featurer   r   c              3   <   K   | ]  }|j                           y wrT   )r   )r   rg   s     r1   r   z!masked_scatter.<locals>.<genexpr>  s     ,WQQYY[,Ws   )codegen.commonr  r  r   MASKED_SCATTER_WITH_INDEXr   broadcast_tensorsr   cumsum_unsafe_masked_indexr=   rG   r   r   rV   )
r   rI   r  r  r  
source_idx	self_flat	mask_flatsource_flatr  s
             r1   masked_scatterr    s     D4;;(P(PQ ++T4L9
d\\"%,,Q/!3
,W4vBV,W)	9k**;	J<QRS{{9fi8==djjIIr2   	quant_min	quant_maxc                    t        j                  |       \  }}||z
  t        ||z
        z  }t        j                  |t        j                  |g            }|t        j
                  ||z        j                  t         j                        z
  }t        j                  |||      }|j                  t         j                        |j                  t         j                        fS rT   )r=   aminmaxr   rb   r   r   r   intrk   float64int64)	rY   r  r  r\   rt   min_valmax_valscale
zero_points	            r1   choose_qparams_tensorr    s     }}U+GWw%	I(=">>EIIeU\\3%01EU[[59<<UYYGGJZI>J88EMM"JMM%++$>>>r2   
accumulatec                     | j                         }t        j                  ||g|j                  |j                        |      }|j                  | j                        S rT   )r   r=   	index_putr   r   )r   ry   r  r  	flatteneds        r1   putr    sL     IE7FNN5;;7I TZZ((r2   c                 V    t         j                  | |||      }| j                  |      S )N)r  )r   r  copy_)r   ry   r  r  r   s        r1   put_r  	  s(     ((4:(
>C::c?r2   rr  input_dtypec                     | |z  }t        j                  ||d      }t        j                  | ||      }| j                  |k7  r|j                  |      }|j                         S )NTr   )r=   r   r   fmart   r   r?   )rN   rr  rx   r  new_grad_outputsum_new_grad
grad_inputs          r1   _softmax_backward_datar    sc     "F*O99_#tDL##VG\?KJ K']];/
  ""r2   Tinclude_selfsrcreduction_typer  c                `    t        j                   j                  j                  k(  xr  j                  j                  k(   fd       |dk(  rt         j                        sۉ j                  j
                  xs  j                  j                  }t        j                        }|r) }t        j                         j                  ||      }	nN j                  |d      }t        j                         j                  ||      }	|	j                  |	dk  d      }	|j                  |      }|r||	z  S ||	z  S t        t        j                  j                  | j                  j                  j                  j                   d      rt"        S  j$                  |dz   d  j'                          j$                  d | j'                         z  }
j'                         g j$                  |dz   d   j$                  d | }g t)         j*                  |z
   j*                        dt)        d j*                  |z
        }j-                  t         j.                        j1                  |
      j3                  |      j5                  |      } j7                  ||||      S )Nc                  V    dj                    d j                    dj                    S )NzYindex_reduce(): self, index and source expected to be in the same device, but got (self) z
, (index) z, and (source) r   )ry   r   r  s   r1   <lambda>zindex_reduce.<locals>.<lambda>6  s4    "kk]*U\\N CJJ<) r2   meanr   r   Tr  )r=   _checkr   r    rt   is_floating_pointr   	ones_liker~   
index_fill
zeros_likemasked_fillr!   r   scatter_reduce_tworv   rV   r   r   r   r   r   r  repeat_interleaver   r   scatter_reduce)r   rx   ry   r  r  r  true_divisiononesr   countsrepeatsindex_shaper   scatter_indexs   ` ``          r1   index_reducer  *  sL    
LLu||#Aszz(A	
 (T

) 

44M

8M8Ms#C__T*44S%FF//#ua0C%%d+55c5$GF''
A6FmmC,,sV|?#-?  

		

  jjq#))+djj#.>.D.D.FFG;;=L4::cAgi#8L4::ds;KLKNU499s?DII.NNE!TYY_4MND		7	#				  !   r2   kernel_size	ceil_modec                 2   |dk(  rdg|z  }|dk(  rdg|z  }|s|}t        ||      }t        ||      }t        ||      }t        ||      }t        j                  t        j                  |      }t
        j                  j                  j                  ||      s0|t        j                  t
        j                        j                  kD  rt        S t        j                  | |||||      \  }}	t        j                  |	|| j                   | d  |||      }
||
fS )Nr   r   )n_dim)r   r<  reduceoperatormulr=   	_inductorlowering%should_fallback_max_pool_with_indicesiinfoint8rb   rV   prims!_low_memory_max_pool_with_offsets'_low_memory_max_pool_offsets_to_indicesr   )rg   r  r   r   r   r  rx   window_sizevalsrg  rO   s              r1   _max_pool_with_indicesr  h  s"    1}39!|#) {C0KHc*H7C(G&#&F""8<<=K 	  FFs 	G 	
 UZZ0444;;	MD' ;;	G =r2   c           	      &    t        | |||||d      S )Nr   r   r  rg   r  r   r   r   r  s         r1   max_pool2d_with_indicesr          "	;9! r2   c           	      &    t        | |||||d      S )Nr]  r   r  r  s         r1   max_pool3d_with_indicesr    r  r2   	out_int32rightsidesortersorted_sequencer  r  r  r  c                ~    t         j                  | t        j                  |g| j                        ||||      d   S )Nr  r  r   )r   searchsortedr=   rz   r   )r  r   r  r  r  r  s         r1   searchsorted_scalarr    sK     dVO$:$:;   	 	r2   r  r  
boundariesc                    t         j                  t        j                  | g|j                        |||      j                  d      S )Nr  r  r   )r   	bucketizer=   rz   r   r   )r   r  r  r  s       r1   bucketize_scalarr    sD     >>dVJ$5$56	  
 gajr2   noiselowerupper	generatorc                    |rS| dk  }t         j                  | |||      }t        j                  || |z  |       }t        j                  ||d      }	||	fS ||z   dz  }
t         j	                  | |
      t        j
                         fS )Nr   )r  r   r   )r   uniformr=   rG   
leaky_relur   )r   r   r  r  r  r  not_positiverrr  	noise_outnegative_slopes              r1   rrelu_with_noise_functionalr    s     qyLLueyLA\4!8T:KKa3	y  %-1,t^4ellnDDr2   repeatoutput_sizec                 N   t         j                  j                  rt        S |t	        |      t
        urt        S | j                  j                  dk(  rt        S | j                  t        j                  t        j                  fv sJ | j                  dk(  sJ | j                  d      }t        j                  || j                        }t        j                  ||| j                  t        j                  k(  d      }t        j                  || j!                  d      dz
        S )Nr   r   r   r  Tr  )rb   )r   tritonautotune_at_compile_timerV   rv   r  r   rt   r=   r[  r  r   r  aranger  rk   rp   )r  r  r  posrO   s        r1   repeat_interleave_Tensorr    s    
 }}-- d;/s:}}U"<<EKK5555;;!]]1F
,,{6==
9C   ;DG ;;wFKKNQ$677r2   r  c           	      8   | j                         dk(  r|j                         dk(  sJ d       |d   }|d   }|d   }| j                  d      }|j                  d      }t        j                  j	                  ||||df|df|df|      }	|	j                  d      S )Nr]  z,Expect (N,C_in,L) and (C_out,C_in//groups,K)r   r   r   )r   r   r   r   )rx   r   r   conv2dr   r   )
rY   r5   r[   r   r   r   r   input_2d	weight_2dout_2ds
             r1   conv1d_to_conv2dr    s     99;!

 1 61
 AYFajG{H r"H  $I [[  {!A ! F >>"r2   )NNrT   )Nr   r   )NF)r   r   F)F)Nr   r   F)g      ?gUUUUUU?FN)Nr   r  r  r   )r<  loggingr   r  ra  collections.abcr   typingr   r   r   typing_extensionsr   r=   torch._decomp_decompr.   torch._prims_common_prims_commonr$  $torch.ao.quantization.fx._decomposedr   r	   r
   torch._decomp.decompositionsr   rq  r   r   rW   r   r   $torch._decomp.decompositions_for_rngr   torch._dynamo.utilsr   torch._environmentr   !torch._higher_order_ops.out_dtyper   torch._inductor.utilsr   r   r   r   r   torch._refsr   r_   %torch.fx.experimental.symbolic_shapesr   r    r   r   r   r    r!   r"   r#   _opsOperatorBaseOpOverloadPacketr$   __annotations__	getLogger__name__r,   r&   r   r  r  rE  quantized_decomposed_adaptive_avg_pool2d_backwardadaptive_max_pool2dadaptive_max_pool3dindex_selectaddmvr  bitwise_and_bitwise_or_
clamp_min_distelu
empty_likeflipgeluhardtanhlcmr  linalg_vector_norm_log_softmax max_pool2d_with_indices_backward_native_batch_norm_legit#_native_batch_norm_legit_functional$_native_batch_norm_legit_no_training_batch_norm_with_update"_batch_norm_with_update_functional_batch_norm_no_updatebatch_norm_backwardr  native_group_normnll_loss2d_backwardpermute_copyrrelu_with_noise_backward_softmaxsin_sqrt__to_copytril_indicestriu_indicesunbind_copyr  upsample_bilinear2dvechann_windowrI  rT  inductor_decompositionsr+   _unsafe_indexr  #_unsafe_masked_index_put_accumulate+_scaled_dot_product_flash_attention_for_cpur   r  rj   ri   r~   gluselect_scatterslice_scatterro   splitr   r   r   unbindbaddbmmrH   addcmul__foreach_addcmulrz  _foreach_addcmul_addcdivaddcdiv__foreach_addcdiv_foreach_addcdiv_lerplerp_r%   r*   
OpOverloadr/   r   rC   rM   r   rX   	ShapeTypetupler`   rf   SymInttypesNumberrk   rw   r   r   r   r   r   r   rt   r   r   r   r   r@   r  r
  detach_r  r  r  r  r  	view_copyr   r"  
StrideTyper'  r.  r,  r(  r   r8   r1  r5  r8  r@  	low_dtyperB  r>  r  rh  rs  _foreach_lerpr|  r  r  r  py_impl_CDispatchKeyAutogradcachedictr  r  r  choose_qparamsrz   r  r  r  r  strr  r  r  r  r  r  r  r  	Generatorr  r  r  re   r2   r1   <module>r     s       
 $ * * '   # + 
  F ( ( 7 .  F W $  T]t_#jj55

8S8SS ) Sg!yy~~		II	YY!!
yy55 ,.**.  . 	  . 		.
 	

. 	. 	. 	. 	. 			. 	. 	. 			. 			. 	.  	!." 	#.$ 	%.& 	'.( 	--).* 	%%+., 	00-.. 	11/.0 	$$1.2 	//3.4 	""5.6 	  7.8 	9.: 	;.< 	=.> 	  ?.@ 	A.B 	&&C.D 	E.F 			G.H 	

I.J 	K.L 	M.N 	O.P 	Q.R 	S.T 	  $$U.V 	W.X 	55Y.Z 	++[.0 b K,.J2IJ
 	,,44<<NNNN!!NNHHIIJJLLHHKKLLLLMM  LLMM  IIJJ=Q D..1L1LLM B n&8 9>	D!12	2>xB (2r6"223> ))*+0 05<< 0 05<< 0 ,0 ))*+1<<1#ll149LL1
\\1 ,1  6678\\  	
  \\ 9" ../
P<<
Poo
P LL4
P ,,
	
P
 

P 5<<u||34
P 0
P ::BBCD &*%)	LL 
		d	" 
		d	"	
 
 E %# &*%)	||				d	"	 
		d	"	 \\		 $ &	 $#ELL #U\\ #  %# $	
sU\\!
"	""	 	 \\		 %	 () !"M||M	M <<M LL	M ;;M \\M *M* ,,4456W
sU\\!
"W#YW W \\	W 7W  2234#.#.<<#. LL#. S		#.
 $s)O#. 49_#. DIo#. #. I#. #. d#. 5<<u||34#. 5#.L ,,-.G G GU\\ G /G
 
# %)
,,LL {{T! \\	  $> %
 %)  !#
,,#
,,# ,,# {{T!	#
 ++

# ;;# \\#  &#L 	" %)&
,,&LL& {{T!& \\	&  #&Z ))*+ :%,,:	: \\: ,:z %:U\\ :ell : &:& 
#
 (,	@||@||@ ;;$	@
 \\@ $@F ++,-   . DLL12u||   3 EJJ/0Iu|| IELL IU\\ I 1I EJJ/0Iu|| IELL IU\\ I 1I 		" 
,,	t  \\	 # 		" 
,,	t  \\	 # ))*+

,,
	
 
 	

 \\
 ,
 //01)
,,)
sU\\!
") \\) 2) --./$
,,$;;$ \\$ 0$
 
,,
 
5??E,,,-
  '
 !%"&"&).)>)>)3
,,)3e)3 ;;	)3
 LL4)3 LL4)3 )3 )3 &&)3 \\)3 ()3` !%"&).)>)>/c5<<'(/
,,/ ;;	/
 LL4/ &&/ / \\/@ /1EF2ELL 2C 2ELL 2 G2 02FG3U\\ 3S 3U\\ 3 H3  1 1 9 9:<PQTu|| T3 T# T%,, T RT  1 1 ; ;<>RSV
,,V V(+V7:V
\\V TV ,,-5
5
sU\\!
"5 5 \\	5 .5 	EEMMN !%<<LL ,,
 \\	 O 
;;CCD<< ll LL	
 ,, || ,, || LL  \\ E4 		++EEF
Au|| 
A 
A G
A --./  ||
,,  	
  \\  0@ **112
%
ell#
 KK
 
%,,	
 3
 **556
%
ell#
 %++$$%
 
%,,	
 7
 ''(<(<(E(EF../<<LL ,,
 ,,%	
 $  !&  5<<u||34 0 G< 6T#xS'9"9: 6 6	T#xS'9"9: 	 ++,
,,
,, LL \\	 -$ ,;;BBC?<<?? ? 
	?
 ;;? 5<<%&? D? !
 	
)
,,
)<<
) LL
) 	
)
 \\
) "
) 		"
 	
,,<< LL 	
 \\ # 33;;<##LL# 
# 	#
 \\#  =#( ))* :
,,:	: <<: 
	:
 : : \\: +:z5||5c5 $s)Od"5 49_	5
 DIo5 5 
5 5<<%&5p 445 &*  !
||
c
 $s)Od"
 49_	

 DIo
 
 5<<%&
 6
 445 &*  !
||
c
 $s)Od"
 49_	

 DIo
 
 5<<%&
 6
 ))001
 "&	\\	
++

	 		
 	 *	 LL4	 \\	 2	& --.
 
++

 	
  \\ / t??AUV %(,E
,,E<<E E 	E
 E %E 5<<%&E WE& ..556 #8LL8t8 \\8 784 !%)<<)LL) ,,
) #J	)
 3Z) Cj) ) \\)r2   