
    ^jA                       d Z ddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
mZmZ ddlmZmZ ddlZddlmc mZ ddlmZ ddlmZmZmZmZmZ ddlmZmZ dd	l m!Z!m"Z"m#Z#m$Z$ dd
l%m&Z&m'Z' ddl(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7 ddl8m9Z9 ddl:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@ ddlAmBZBmCZC ddlDmEZEmFZF ddlGmHZHmIZI ddlJmKZK ddlLmMZM erddlNmOZO dZP e.eQd      ZR e.eQd      ZSdeTfdZUdeTfdZVdej                  dz  dej                  dz  fd ZXd!ede
d"ef   fd#ZY G d$ d%      ZZ G d& d'      Z[ G d( d)      Z\ e[       Z]g d*Z^ eKeeej                  eg      Z` ej                         abd+ecdefd,Zd G d- d.      Zed/afd/agd/ahd/aidajej                  	 	 dDd0e
d"ef   d1eTd2eTded3   fd4       Zlej                  ded3   fd5       ZmdEd6Znd7eej                     d8eeI   d9eeI   d:eId;eeI   d<eeI   d=eIdeoej                     fd>Zpd?eeT   d@ej                  dAeej                  dz     dBej                  deoej                  dz     f
dCZqy)Fa  
Provides functionality for compiling PyTorch's autograd (automatic differentiation) system.

This module implements compiled autograd, which traces and optimizes backward pass
computations at runtime. The key components are:

- AutogradCompilerInstance: Traces and compiles autograd graphs using FX
- Context managers (_enable/_disable): Control when compiled autograd is active
- Utility functions: Support graph manipulation, tensor operations, and hooks

Compiled autograd can significantly improve backward pass performance by removing
Python overhead and enabling additional optimizations. It works by capturing
backward computations into an FX graph that can be compiled and optimized,
while maintaining the same semantics as eager mode autograd.
    N)Counterdefaultdict)Callable	GeneratorSequence)AnyTYPE_CHECKING)enable_python_dispatcher)call_accumulate_gradcall_backward	call_hookFakeCompiledAutogradEngineunwrap_maybe_dynamic_intGetItemSourceLocalSource)countersget_chromium_event_loggerlazy_format_graph_codeset_locals_to_steal)AutogradLazyBackwardCompileInfo%CachedAutogradLazyBackwardCompileInfo)compile_contextCompileContext	CompileIdSource)getArtifactLoggertrace_structuredclone_preserve_strides)FakeTensorMode)
FakeTensor)GraphModule)BackwardState)disable_autocast_cachedisable_proxy_modes_tracingfetch_object_proxyProxyTorchDispatchModePythonKeyTracertrack_tensor_tree)
DimDynamicShapeEnv)preserve_node_metaset_stack_trace)FloatLikeTypeIntLikeType)
OrderedSet)CapturedTraceback)Proxya  You can turn off compiled autograd by either:
1. Moving the unsupported autograd call outside of the torch.compile'd region.
2. Wrapping the unsupported autograd call in the torch._dynamo.compiled_autograd._disable() context manager.
3. Setting torch._dynamo.config.compiled_autograd=False for the torch.compile call containing the unsupported autograd call.
4. Setting torch._dynamo.config.compiled_autograd=False at the start of the program.compiled_autogradcompiled_autograd_verbosereturnc                  h    t         j                  j                  j                  j	                  d      S )Nr5   )torch_logging	_internal	log_stateis_artifact_enabled     j/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_dynamo/compiled_autograd.py snapshot_verbose_logging_enabledr@   S   s(    >>##--AA# r>   c                  ^    t         j                  j                  j                  j                  S N)r8   	_inductorconfigtriton
cudagraphsr=   r>   r?   snapshot_cudagraph_enabledrG   Y   s    ??!!((333r>   xc                      | t        |       S | S rB   r   )rH   s    r?   maybe_clonerJ   ]   s    }%a((Hr>   CompiledFunction.c                 \   t        | j                  t              r| j                  j                  S t        | j                  t              rLt
        j                  j                  j                         5  | j                  j                         cd d d        S t        d      # 1 sw Y   y xY w)NzEUnexpected Lazy Backward Compilation Info Type. Please file an issue.)
isinstance_lazy_backward_infor   	bw_moduler   r8   _subclassesfake_tensorunset_fake_temporarilybw_module_fnAssertionError)rK   s    r?   extract_bw_modulerU   c   s    ,,.M  33===	,,.S
 **AAC 	G#77DDF	G 	G S
 	
	G 	Gs   3B""B+c                       e Zd ZdeddfdZdej                  j                  ddfdZde	ej                  df   ddfd	Zd
e	ej                  df   ddfdZy)
NaNCheckeraccumulate_gradr6   Nc                 <    || _         g | _        i | _        g | _        y rB   )rX   params_indicesparams_to_checkoutput_names)selfrX   s     r?   __init__zNaNChecker.__init__   s"    .)+8:')r>   graphc                 j   t        t        |j                              }|j                  dt              }|j                  d      d   j
                  d   }| j                  t        |      k7  r%t        d| j                   dt        |       d      | j                  | k7  rt        d	| j                   d|        |D ]  }|j
                  d   }|j                  t        j                  urt        d
|j                         |j
                  d   |urt        d      t        |j
                  d   t              s$t        dt        |j
                  d                | j                  j!                  |j
                  d           |D cg c]  }|j"                   c}| _        y c c}w )Ncall_functionoptargetoutputrc   r   z#accumulate_grad mismatch: expected z, got z acc_grad_nodesz5accumulate_grad mismatch with output_nodes: expected z7Expected param_node target to be operator.getitem, got z-Expected param_node.args[0] to be inputs_node   z+Expected param_node.args[1] to be int, got )nextiternodes
find_nodesr   argsrX   boolrT   rd   operatorgetitemrM   inttyperZ   appendnamer\   )r]   r_   inputs_nodeacc_grad_nodesoutput_nodesnode
param_nodes          r?   prep_with_graphzNaNChecker.prep_with_graph   s   4,-))'; * 
 ''8'4Q7<<Q?4#77 5d6J6J5K LN+,O=  $45 GH\H\G] ^''(* 
 # 	;D1J   (8(88$MjN_N_M`a  q!4$%TUUjooa0#6$A$zWXGYBZA[\  &&zq'9:	;" 4@@4TYY@@s   F0inputs.c                     | j                   sy | j                  D ]a  }||   j                  }|8t        j                  |      j                         rt        d| dt               ||   | j                  d| d<   c y )Nz9Compiled autograd running under anomaly mode with inputs[z6] already having NaN gradient. This is not supported. zinputs[])	rX   rZ   gradr8   isnananyrT   TURN_OFF_MSGr[   )r]   rz   idxr}   s       r?   prep_with_inputszNaNChecker.prep_with_inputs   s    ## && 		AC#;##D;;t$((*(STWSX YGGSnV 
 6<C[D  73%q!12		Ar>   outc                 Z   | j                   r|rt        d      g }| j                  j                         D ][  \  }}|j                  t        d      t        j                  |j                        j                         sK|j                  |       ] |rt        ddj                  |       d      y g }t        |      D ]G  \  }}t        j                  |      j                         s*|j                  | j                  |          I |rt        ddj                  |       d      y )Nz2Expected empty output for accumulate_grad backwardz4Expected param.grad to be non-None for backward modez9Compiled Autograd returned NaN gradients for parameters: ,.z;Compiled Autograd returned NaN gradients for output nodes: )rX   rT   r[   itemsr}   r8   r~   r   rr   RuntimeErrorjoin	enumerater\   )r]   r   
nan_params
inputs_strparam	nan_gradsir}   s           r?   checkzNaNChecker.check   s;   $H  %'J%)%9%9%?%?%A 2!
E::%(N  ;;uzz*..0%%j12 "OPSPXPXYcPdOeefg   $&I$S> ;4;;t$((*$$T%6%6q%9:; "QRURZRZ[dReQffgh  r>   )__name__
__module____qualname__rm   r^   r8   fxGraphry   tupleTensorr   r   r=   r>   r?   rW   rW   ~   sy    * * *"AUXX^^ "A "AHAuU\\3->'? AD A"u||S01 d r>   rW   c            
       J    e Zd ZddZdededef   dededef
d	Zdedefd
Z	y)OpNamespacer6   Nc                 "    t               | _        y rB   )r   custom_function_name_counterr]   s    r?   r^   zOpNamespace.__init__   s    :A))r>   rs   fn.is_custom_functionis_traceablec                    |r1d|z   }| j                   |   }| j                   |xx   dz  cc<   | | }t        | |      rt        d| d      t        |||      |r,t	        | |t
        j                  j                               |S t
        j                  j                  dt        dt        dt        ffd       }t	        | ||       |S )	NCppNoderg   zOp with name 'z' already existsrl   kwargsr6   c                       | i |S rB   r=   )rl   r   results     r?   run_non_traceable_cpp_in_eagerz7OpNamespace.add.<locals>.run_non_traceable_cpp_in_eager   s    t.v..r>   )
r   hasattrrT   Opsetattrr8   _dynamoallow_in_graphdisabler   )r]   rs   r   r   r   countr   r   s          @r?   addzOpNamespace.add   s     t#D55d;E--d3q83VE7#D4 >$7G!HIID"01D$ < <V DE  ]]""/c /S /S / #/ D$ >?r>   c                     t        | |      S rB   )getattr)r]   rs   s     r?   getzOpNamespace.get  s    tT""r>   r6   N)
r   r   r   r^   strr   r   rm   r   r   r=   r>   r?   r   r      sY    D S#X !	
  
:# # #r>   r   c                   N    e Zd Zdededef   deddfdZded	edefd
ZdefdZ	y)r   rs   r   .r   r6   Nc                 <    || _         || _        || _        d| _        y )Nz#torch._dynamo.compiled_autograd.ops)r   r   r   r   )r]   rs   r   r   s       r?   r^   zOp.__init__  s!     "4?r>   rl   r   c                 &     | j                   |i |S rB   )r   )r]   rl   r   s      r?   __call__zOp.__call__  s    tww'''r>   c                 :    | j                   dz   | j                  z   S )Nr   )r   r   r   s    r?   __repr__zOp.__repr__  s    $t}}44r>   )
r   r   r   r   r   r   rm   r^   r   r   r=   r>   r?   r   r     s[    @@%c3h/@EI@	@(c (S (S (5# 5r>   r   )rz   sizesscalarshookspacked_datacompiled_autograd_idc           	      B    t        t        t        | d d                   S )N)r   frame_idframe_compile_id)r   r   r   )r   s    r?   make_compile_contextr   %  s(    %9!%	
 r>   c                      e Zd Zdedef   ddfdZdej                  dedz  de	fdZ
ed	ed
edefd       Zdeej                     dee   deeez     deeeeef         dededeeeej                     ee   ee   f   fdZdee   ddfdZdee   deej                     deej                     dedededz  dee   dee   fdZdee   dee   deej                     dedej2                  j4                  j6                  dedz  dee   deej                  dz  df   fdZdee   d ee   d!ee   d"ed#ee   d$ee   d%edeej                     fd&Zd'ee   d(ej                  d)ee   d*ej                  deej                     f
d+Zdej                  fd,Zdej                  dej                  fd-Z d.ed/edef   d0ed1edef
d2Z!d.ed3ee   d4ed5ee   deej                     f
d6Z"d/edef   d4ed5ee   deej                     fd7Z#d8ed9ee   d4ed5ee   deej                     f
d:Z$d;ed<edej                  fd=Z%	 dmd>ej                  d?ej                  dz  d@ej                  ez  dAedz  dej                  dz  f
dBZ&dCedef   d4edDedejN                  jP                  fdEZ)dFedGedej                  fdHZ*deej                     dFedIedeej                     fdJZ+deej                     dFedIedeej                     fdKZ,dee   dFedeej                     fdLZ-d9eej                     deej                     dFedeej                     fdMZ.dNej                  dFedeej                     fdOZ/dPejN                  j`                  dee   fdQZ1dRedefdSZ2dndTZ3de4e   fdUZ5dVede6fdWZ7d9edeedef   ef   fdXZ8ed4ee   deejN                  jr                     fdY       Z:edRejN                  jr                  defdZ       Z;dnd[Z<dnd\Z=dnd]Z>dnd^Z?dnd_Z@dnd`ZAdndaZBdbedefdcZC	 dmddee   deedeeeef      dz  dee   fdfZDdgedeEfdhZFdiedjedkej2                  j                  dz  ddfdlZHy)oAutogradCompilerInstancecompiler_fn.r6   Nc                 4   || _         t        j                         | _        | j                  j                  | _        t               | _        t        dd| j                        | _        t               | _
        t        | j                  d      | _        d | _        y )NT)allow_fallback_kernelsallow_non_fake_inputs	shape_envsymbolic)r   
contextlib	ExitStackstackcloser,   r   r!   fake_tensor_moder)   	fx_tracerr(   
proxy_modehooks_proxy)r]   r   s     r?   r^   z!AutogradCompilerInstance.__init__2  st    &))+
ZZ%%
! .#'"&nn!

 )*0L)-r>   rH   sourcec                     t        |t        j                        st        dt	        |             | j
                  j                  ||      S )NExpected torch.Tensor, got )r   )rM   r8   r   rT   rq   r   from_tensor)r]   rH   r   s      r?   	wrap_fakez"AutogradCompilerInstance.wrap_fake@  sC    !U\\* #>tAwi!HII$$0060BBr>   rs   r   c                 ,    t        t        |       |      S rB   r   )rs   r   s     r?   r   zAutogradCompilerInstance.sourceE  s    [.44r>   rz   r   r   originsrX   
check_nansc                     t         d   dxx   dz  cc<   t        t               _        t	        t
               _        t         j                         _         j                  j                          |rt        |      nd  _        t        j                          _        t               j!                  d j                  d j                  id       t"        j$                  j'                          j(                  _        t"        j,                  j/                  t0               j(                  _        i  j(                  _        i  _         fdt8        D        \  } _         _         _         _          jB                  jE                  tG                      |\  }}	}
 jB                  jE                  tI                      |d	   }	 tK        |      D ]*  \  }} jM                  | jO                  d
|            ||<   , 	  jY                  |||       tK        |      D cg c]@  \  }} jZ                  j]                  | jO                  d|      t^        j`                        B }}}tc        te        |            D cg c]  } j:                  |    }}tK        |      D ][  \  }} j(                  jg                  dth        ||   fi       ||<   tk        |t
              r@||    j6                  |jl                  <   ]  jY                  |||	      }tK        |      D ]  \  }} jO                  d|      }tk        |t
              r/ jZ                  j]                  ||t^        j`                        ||<   Wtk        |tn              rL jZ                  jq                   jZ                  js                  ||t^        j`                        ||      ||<   tu        dtU        |              jY                  | j<                  |
       tK        |      D ]+  \  }} j<                  |    j6                  |jl                  <   -  jB                  jE                   jv                          jB                  jE                   jx                          jB                  jE                  t{                       jv                  jZ                  tu        d       jv                  jZ                  } jB                  jE                  t"        j,                  j|                  j~                  j                  |             t        t        j                               |||fS # tP        $ r$}tS        dtU        |       dtV               |d }~ww xY wc c}}w c c}w )Nr4   capturesrg   graph_idTlog_pt2_compile_event)
tracer_clsc              3   Z   K   | ]"  }j                   j                  d |di        $ yw)placeholderr=   N)r   create_proxy).0rs   r]   s     r?   	<genexpr>z9AutogradCompilerInstance.begin_capture.<locals>.<genexpr>i  s.      
 NN''tRD
s   (+r   rz   zFound tensor of type z,, which is not supported by FakeTensorMode. r   ra   r   )r   dynamic_dim)hintr   zUnexpected scalar type: z+fake_tensor_mode.shape_env must not be None)Dr   rh   COMPILE_COUNTERidr   rp   aot_id_counterr   r   	__enter__rW   nan_checkertimetime_nsstart_time_nsr   log_event_startr8   nnModuler   rootr   r   r)   r_   tensor_attrssymnode_proxy_lookup_graph_placeholderssizes_proxyscalars_proxyr   packed_data_proxyr   enter_contextr-   r
   r   r   r   	ExceptionNotImplementedErrorrq   r   bind_objects_to_proxiesr   $create_unspecified_symint_and_symbolr+   DYNAMICrangelenr   r   rM   rw   floatcreate_symfloatnodecreate_unspecified_symbolrT   r   r   r%   experimentalsymbolic_shapes_suppress_guardsr   r   current_compile_id)r]   rz   r   r   r   rX   r   
args_proxyinputs_originssizes_originsscalars_originsrH   r   eval	sym_sizesr   proxiessymintr   symvalenvs   `                     r?   begin_capturez&AutogradCompilerInstance.begin_captureI  s    	$%j1Q61'.9#.>3DGG<&&(:D:o6$!\\^!#33!"&	 	4 	
 $hhoo/$xx~~~I&(#$&!
+
	
" 	

  !3!569@6 	

  !9!;< 1I	#F+ LQ"nnQHc0JKsL 	$$VZH &e,
 S NN??GS)""
	 
 16c)n0EF14##A&FF"9- 	DIAv44(	GAJ fc*9@))&++6	D ..y'=Q!'* 	LHC[[C0F#s##~~RR&& 
 C'#~~AANN<<%$.$6$6 = 
 !  B   %%?cKK'	L( 	$$Wd.@.@/R"7+ 	KIAv595G5G5JD%%fkk2	K 	

  !6!67

  1

  !7!9:  **2 !NOO##--

  HH!!11BB3G	

 1134	
 	
G  	%'Qy0\]i\jk	
 Gs%   8U AU<=V	U9U44U9compile_reasonsc                 D    st        d      t        dd fd       y )Nz!compile_reasons must not be emptyartifactc                      dddS )N!compiled_autograd_compile_reasonsjsonrs   encodingr=   r=   r>   r?   <lambda>z>AutogradCompilerInstance.log_compile_reasons.<locals>.<lambda>  s    ;"! r>   c                       S rB   r=   )r  s   r?   r  z>AutogradCompilerInstance.log_compile_reasons.<locals>.<lambda>  s     r>   metadata_fn
payload_fn)rT   r   )r]   r  s    `r?   log_compile_reasonsz,AutogradCompilerInstance.log_compile_reasons  s)      !DEE /	
r>   pinputspsaved_tensorssaved_tensorspctxctxmaybe_backward_state_idxopaque_object_indicesc                 ,    j                         D cg c]  } j                  |       }	}|D 
cg c]  }
 j                  |
    }}
j                  }t	        |      |j
                  }|j                  }|j                  |j                  |j                  }~t        j                         r(|j                  D ]  }|j                  st        d       t        j                  j                   dt"        t        j$                     dt"        t&           dt"        t(           dt"        t(           dt(        f
fd       } j*                  j-                  d|||	||fi 	      d | j                  |   dt.        t        j$                     f fd
} |       }|:dt(        dt0        dt"        t(           dt        j$                  f fd} |||      }n ||      }t3        j4                   j                  |      }|S c c}w c c}
w )Nz@torch.compile does not currently support higher order gradients.ctx_saved_tensorsctx_symintsctx_opaque_objs	flat_argsr6   c                      | |||      S rB   r=   )r*  r+  r,  r-  bw_prologue_fns       r?   call_aot_bwd_prologuezOAutogradCompilerInstance.proxy_call_aot_backward.<locals>.call_aot_bwd_prologue   s     "!	 r>   ra   kindrd   rl   r   c            	      	   dt         j                  j                  dt        fd}  | j                        }t        |t        d u      z
        D cg c]  }|   	 }}j                         }t        |      t        j                        k7  r-t        dt        |       dt        j                               |D cg c]  }j                  |       }}||d t        |       |j                         d}i d }t              j                     rdj                      z  j                  xx   dz  cc<   d	t        dt        ffd
}	j                  j                  D ]  }
|
j                  dk(  r1||   j                   } |	|
j"                        |_        ||
<   |dz  }D|
j                  dk(  rt        |
j$                        dk7  r!t        dt        |
j$                               |
j$                  d   D cg c]U  }t'        |t         j                  j(                        r-t         j                  j+                  |   j,                        n|W }}|
j                  dk(  r|
j.                  }j,                  j1                  |      }t3        j,                  j4                  |t7        |             j,                  j9                  d|di       } |	|
j"                        |_        ||
<   |
j                  dk(  r|
j.                  t         j:                  j<                  j>                  j@                  u r3t         j:                  j<                  jB                  j@                  |
_        j,                  j                  jE                  |
fd      } |	|
j"                        |_        ||
<   Z|
j                  dk(  r|
j.                  }j,                  j1                  |      }t3        j,                  j4                  |t7        |             j,                  j                  jE                  |
fd      }||_        ||
<   t        d       |t        d      dt         jF                  fd}|D cg c]/  }t'        |t         j                  j*                        r |       n|1 }}jI                  ||       |S c c}w c c}w c c}w c c}w )Nr_   r6   c                 Z    d}| j                   D ]  }|j                  dk(  r|dz  } |S  |S )Nr   r   rg   )rj   rc   )r_   num_argsrw   s      r?   
num_inputszkAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.num_inputs!  sA    !KK Dww-/ A   r>   zsymints length mismatch: z vs r   _rg   	node_namec                     d d|  S )Naotr7  r=   )r8  deduped_aot_ids    r?   make_uniquezlAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.make_uniqueK  s    ^,Ai[99r>   r   re   z(Expected output node to have 1 arg, got get_attrr=   ra   c                     |    S rB   r=   nvalue_remaps    r?   r  ziAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.<lambda>n      A r>   call_modulec                     |    S rB   r=   r?  s    r?   r  ziAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.<lambda>w  rB  r>   zshouldn't get herez&No output node found in backward graphc                  t    t               5  t        j                  ddddd      cd d d        S # 1 sw Y   y xY w)Nr   {   r&   r8   zerosr=   r>   r?   dummyzfAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph.<locals>.dummy  s1    02 8 ;;q!Q378 8 8s   .7)%r8   r   r   rp   r_   r   _get_compiled_autograd_symintsr   symintsrT   to_proxyrr   r   r   rj   rc   rw   rs   rl   rM   Noder3   r   rd   get_fresh_qualnamer   r   r   create_nodeopsatenviewdefaultreshape	node_copyr   r   )r6  r5  r   	pall_argsrK  r  psymintsargs_idxpoutputsr<  rw   phr@  rs   qualnamer   rI  ooutputsr;  rA  aot_idrO   r&  pbackward_statepgradsr]   s                      @@r?   copy_paste_aot_backward_graphzWAutogradCompilerInstance.proxy_call_aot_backward.<locals>.copy_paste_aot_backward_graph   se    %((..  S   ")//2H#(Ct8S4T)T#Uq	I  88:G7|s3;;//$/G~T#ckkBRAST  3::Qa(:H:(0InG%*  1 HK48H ![N""6*Ad&9&9&&A%B"CC'1,':s :s : "-- -?77m+"8,11B)$))4BG(*K%MHWW(499~*,Fs499~FVW  "&1	   &a7 {1~t~~F H   WW
*;;D#~~@@FHDNN//79d;ST!^^77
HbRTUF"-dii"8FK(.K%WW/{{eiinn&9&9&A&AA ',iinn&<&<&D&D!^^11;;6F #.dii"8FK(.K%WW-;;D#~~@@FHDNN//79d;ST!^^11;;6F %-FM(.K%()=>>[-?^ $%MNN85<< 8
 JRDE:a8a?G  (((;N{ ;F Zs   S!S&AS+4S0subclass_meta
is_runtimeunwrapped_argsc                 D    t         j                  j                  dt        dt        f fd       }t	        j
                  j                  |      }j                  j                  d|t        |      i       }j                         }j                  |g|g       |S )Nrd  r6   c                  *    j                  |       S )N)rc  )creation_fn)rd  rc  rb  s    r?   make_subclasszkAutogradCompilerInstance.proxy_call_aot_backward.<locals>.proxy_subclass_constructor.<locals>.make_subclass  s     (44&: 5  r>   ra   r1  )r8   r   r   r   pytreetree_maprL  r   r   r   allocate_dummyr   )rb  rc  rd  rh  punwrapped_argspoutputre   r]   s   ``     r?   proxy_subclass_constructorzTAutogradCompilerInstance.proxy_call_aot_backward.<locals>.proxy_subclass_constructor  s    
 --3 3  .
 #)//$--"P..55((/ 6  ,,.,,fXyAr>   )make_subclass_override)rJ  rL  r   _forward_clsrU   metadatamaybe_subclass_metadata_aot_id_bw_prologue_fn_bw_epilogue_fnr8   is_grad_enabledoutput_inforequires_gradr   r   r   r   r   r0   r   r   r   listrm   ri  rj  )r]   r"  r#  r$  r%  r&  r'  r(  r  rW  r   popaque_objectsrK   rq  rr  bw_epilogue_fnoutput_alias_infor0  ra  r]  rn  resultspresultsr^  rO   r/  r_  r`  s   `    `                 @@@@@r?   proxy_call_aot_backwardz0AutogradCompilerInstance.proxy_call_aot_backward  s<   * /2.P.P.RSDMM!$SS -
 S!
 
 ++%&67	#,,"2"J"J!)))99)99  "%-%9%9 !$22&Z  
	%	%	'5	!+.	 &c]	  }		
 	 
&	 ,, (	  - 
 #/"../GHOk	tELL/A k	 k	Z 01"."  !) 	2 %0JG %W-G??4==':S T
s
   HHoutput_metadatasbackward_idxc           
         | j                   t        d      | j                   |   }| j                  |      }	| j                  |      }
t        |j                  d      r| j                  |	|
|||||      }nbt        |j                  dd      r#t        d|j                  j                   d      | j                  j                  dt        ||
g|	i       }|t        d	      t               5  g }t        |      D ]M  \  }}|||   |j                  d        |\  }}}}|j                  t        j                   ||||
             O | j#                  ||       d d d        t%        |      S # 1 sw Y   t%              S xY w)Nz2hooks_proxy must be set before proxy_call_backwardrs  boxed_grads_callFzboxed_grads_call=True on z* is not supported with compiled autograd. ra   r1  z,proxies must not be None after backward call)sizedtypelayoutdevice)r   rT   rL  r   rp  r  r   r   r   r   r   r   r&   r   rr   r8   emptyr   r   )r]   rz   r  r$  r  r&  r'  r(  r%  r"  r#  r  grad_insr   output_metadatar  r  r  r  s                      r?   proxy_call_backwardz,AutogradCompilerInstance.proxy_call_backward  s    # !UVV---'}53##Y/22(%G s'');UC"/0@0@0I0I/J K@ @  nn11$$" 
  2 	G ? !OPP(* 	<24H(12B(C $_"*gcl.BOOD).=+tKKTvfU ((7;	< X	< Xs   0A0E33F
base_sizesbase_stridesbase_storage_offset
view_sizesview_stridesview_storage_offsetc           	          || j                  |      | j                  |      | j                  |      | j                  |      | j                  |      | j                  |      f}| j                  t        |d gdz        S )N   )rL  
proxy_callcopy_slices_prologue)	r]   rz   r  r  r  r  r  r  rl   s	            r?   call_copy_slices_prologuez2AutogradCompilerInstance.call_copy_slices_prologue  st     MM*%MM,'MM-.MM*%MM,'MM-.
 3TD6A:FFr>   needs_input_gradr   res
grad_slicec                 R    | j                  t        ||||fd gt        |      z        S rB   )r  copy_slices_epiloguer   )r]   r  r   r  r  s        r?   call_copy_slices_epiloguez2AutogradCompilerInstance.call_copy_slices_epilogue  s4      vsJ7FS)**
 	
r>   c                 p    t               5  t        j                  ddg      cd d d        S # 1 sw Y   y xY w)Nr   i[rG  r   s    r?   rk  z'AutogradCompilerInstance.allocate_dummy  s-    (* 	/;;9~.	/ 	/ 	/s   ,5c                 l    t               5  t        j                  |      cd d d        S # 1 sw Y   y xY wrB   )r&   r8   
empty_like)r]   rH   s     r?   allocate_dummy_likez,AutogradCompilerInstance.allocate_dummy_like  s+    (* 	'##A&	' 	' 	's   *3fn_namer   r   r   c                 2    t         j                  ||||      S )zBinds ops.fn_name = fn)rP  r   )r]   r  r   r   r   s        r?   bind_functionz&AutogradCompilerInstance.bind_function  s     www$6EEr>   gradsrl   r  c                 Z    t         j                  |      }| j                  ||g||      S )z:Proxies a call to ops.fn_name(grads, *args) into the graph)rP  r   r  )r]   r  r  rl   r  rc   s         r?   apply_functionalz)AutogradCompilerInstance.apply_functional$  s+     WWWrE>D>?CCr>   c                 0    t        j                   fd|      } j                  j                  d||i       }|D cg c]  } j	                          }} j                  |t        t        |            D cg c]  }||   	 c}       |S c c}w c c}w )z*Proxies a call to fn(*args) into the graphc                 &    j                  |       S rB   rL  )r  r]   s    r?   r  z5AutogradCompilerInstance.proxy_call.<locals>.<lambda>3  s    t}}Q/? r>   ra   rl   r   )ri  rj  r   r   rk  r   r   r   )	r]   r   rl   r  
proxy_args	proxy_outr7  r   r   s	   `        r?   r  z#AutogradCompilerInstance.proxy_call/  s     __%?F
NN//Rj 0 
	 2AAA$%%'AA$$VE#f+DV-Wqil-WX B-Ws   B9B
r7  r]  c                 P   t         j                  d      }t        j                  | j                  |g|      }| j
                  j                  d||i       }t        |      t        |      k7  r$t        dt        |       dt        |       d      | j                  ||       |S )zEProxies a call to ops.validate_outputs(outputs, *args) into the graphvalidate_outputsra   r  zoutput_metadata length (z) != outputs length ())
rP  r   ri  rj  rL  r   r   r   rT   r   )r]   r7  r]  rl   r  rc   r  new_proxy_outputss           r?   r  z)AutogradCompilerInstance.validate_outputs;  s     WW'(__T]]W4Dt4DE
 NN77Rj 8 
 3w</ *3+?*@@UVYZaVbUccde  	$$W.?@r>   old_varnew_varc                     | j                  |      }| j                  |      }| j                  j                  dt        j                  ||fi       }| j                         }| j                  |g|g       |S Nra   r  )rL  r   r   r8   r   rk  r   )r]   r  r  old_var_proxynew_var_proxyr  r   s          r?   
accumulatez#AutogradCompilerInstance.accumulateK  ss    g.g.NN//UYYm]-KTV 0 
	 $$&$$fX	{;r>   variablevariable_gradr}   has_post_hooksc           	      J   |d u }|t        |      }|}|j                  }| j                  j                  dt        | j                  |      | j                  |      | j                  |      |fi       }| j                  |      }| j                  |g|g       ||_        |ry |S r  )rm   r}   r   r   r   rL  r  r   )r]   r  r  r}   r  old_callr  r   s           r?   rX   z(AutogradCompilerInstance.accumulate_gradU  s     "T)! "$ZN D$MMMNN// h'm,d#	  0 

	 ))(3$$fX	{;r>   hookr   c                     | j                   j                  dt        |g|D cg c]  }| j                  |       c}|      S c c}w Nra   )r   r   r   rL  )r]   r  rl   r   rH   s        r?   proxy_call_hookz(AutogradCompilerInstance.proxy_call_hookv  sO     ~~**,01q$--"1 
 	

 2s   Ahook_iddata_idc                     | j                   t        d      | j                   |   }| j                  |   }| j                  ||d      }| j	                         }| j                  |g|g       |S )Nz*hooks_proxy must be set before unpack_hookunpack_hook	hook_type)r   rT   r   r  rk  r   )r]   r  r  r  dataproxyr   s          r?   r  z$AutogradCompilerInstance.unpack_hook  s    # !MNN(%%g.$$# % 

 !!#$$cUUG4
r>   r   c                    | j                   t        d      | j                   |   }| j                  |||   d      }t               5  t	        ||         ||<   | j                  ||   g|g       d d d        |S # 1 sw Y   |S xY w)Nz.hooks_proxy must be set before tensor_pre_hooktensor_pre_hookr  r   rT   r  r&   rJ   r   )r]   rz   r  r   r  r  s         r?   r  z(AutogradCompilerInstance.tensor_pre_hook  s     # !QRR($$1I' % 

 )* 	?#F1I.F1I((&)ug>	? 	? s   )A;;Bc           	      R   | j                   j                  dt        j                  j                  j
                  j                  || j                  ||         fi       }t               5  t        ||         ||<   | j                  ||   g|g       d d d        |S # 1 sw Y   |S xY wr  )r   r   r8   _Cr   r4   call_cpp_tensor_pre_hooksrL  r&   rJ   r   )r]   rz   r  r   r  s        r?   cpp_tensor_pre_hookz,AutogradCompilerInstance.cpp_tensor_pre_hook  s     ++HH..HHdmmF1I./	
 )* 	?#F1I.F1I((&)ug>	? 	? s   ))BB&c                    | j                   t        d      | j                   |   }| j                  ||d      }t               5  |D cg c]  }t	        |       }}| j                  ||       d d d        |S c c}w # 1 sw Y   |S xY w)Nz'hooks_proxy must be set before pre_hookpre_hookr  r  )r]   rz   r  r  r  rH   s         r?   r  z!AutogradCompilerInstance.pre_hook  s    # !JKK(&&  ' 

 )* 	:.45k!n5F5((9	:  6	: s   A?
A:A?:A??B	c                    | j                   t        d      | j                   |   }| j                  |||d      }t               5  |D cg c]  }t	        |       }}| j                  ||       d d d        |S c c}w # 1 sw Y   |S xY w)Nz(hooks_proxy must be set before post_hook	post_hookr  r  )r]   r]  rz   r  r  r  rH   s          r?   r  z"AutogradCompilerInstance.post_hook  s     # !KLL(&&!	 ' 
 )* 	;/67!{1~7G7((':	;  8	; s   B A;B ;B  B
inputc                 Z   t        |t        j                        st        dt	        |             | j
                  t        d      | j
                  |   }| j                  ||d      }t               5  t        |      g}| j                  ||g       d d d        |S # 1 sw Y   S xY w)Nr   z1hooks_proxy must be set before post_acc_grad_hookpost_acc_grad_hookr  )
rM   r8   r   rT   rq   r   r  r&   rJ   r   )r]   r  r  r  r  r  s         r?   r  z+AutogradCompilerInstance.post_acc_grad_hook  s     %. #>tE{m!LMM# !TUU($$* % 

 )* 	7u%&C((ug6	7 
	7 
s   6 B  B*r_   c                    i }d}t        |j                        }|d   j                  dk7  rt        d|d   j                   d      |d   }t        |j                  j                               }t        t              }||   |d   k7  rt        d      |t        |      z   dz
  }||   |d   k7  rt        d	      t        |      D ]  \  }	}
|s)|
j                  d
   j                  j                  dk(  rd}1|
j                  d
   j                  j                  dk(  }t        |
j                  d
   j                               dk(  }|s|st        |
j                  j                               }t        d |D              s|
||	<    |rn|j                         D ]B  }
t        j!                  d|
       |
j                  d
   j#                         |
j                  d
<   D t        |j                               S g S )NFr   rz   z0Expected first node target to be 'inputs', got ''z3First getitem node does not match first inputs userrg   z1Last getitem node does not match last inputs userr  cudaTcpuc              3     K   | ]  }t        |j                  t        j                  j                        xr |j                  j
                  d v xs3 t        |j                  t              xr |j                  j                     yw))primsrQ  N)rM   rd   r8   _ops
OpOverload	namespacer   r   r   users     r?   r   zDAutogradCompilerInstance.move_graph_nodes_to_cuda.<locals>.<genexpr>  sw      
  #4;;

0E0EF G KK115FF
 #4;;3 ? $ > >>
s   BB	zMoving node %s from cpu to cuda)ry  rj   rd   rT   userskeysr   r   r   metar  rq   r  allvaluesverbose_logdebugr  )r]   r_   to_movehas_cuda_inputsrj   rz   inputs_usersfirst_getitem_idxlast_getitem_idxr   rw   is_cpu	is_scalar
node_userss                 r?   move_graph_nodes_to_cudaz1AutogradCompilerInstance.move_graph_nodes_to_cuda  s   ,.U[[!8??h& B58??BSSTU  qFLL--/0 34"#|A6 !VWW,s</@@1D!"l2&66 !TUU . 	&GAt"tyy'7'>'>'C'Cv'M"&YYu%,,11U:FDIIe,11349I)!$**//"34
 
 !+
 
 "&GAJ-	&4 ( ;!!"CTJ#'99U#3#8#8#:		% ;
 ''	r>   rw   c                 @   t        |t        j                  j                        xry |j                  dk(  xrh |j
                  t        j                  j                  j                  j                  t        j                  j                  j                  j                  fv S r  )rM   r8   r   rM  rc   rd   rP  rQ  sym_sizerp   	sym_numelrS  )r]   rw   s     r?   is_sym_nodez$AutogradCompilerInstance.is_sym_node  sn    tUXX]]+ O?*O		''++UYY^^-E-E-M-MNO	
r>   c                    t               d }t        | j                  j                  j	                  d            D ].  \  }}j                  |j                  j                                0 |t        t              dz
  k7  r!t        dt        t              dz
   d|       dt        j                  j                  dt        ffd}t        | j                  j                  j                        }| j                  j                  j!                  |       t        | j                  j                  j                        }t"        j%                  d	||z
         y )
Nr   rf   rg   z	Expected z placeholders, got rw   r6   c                 p    | v s!| j                   dk(  r| j                  t        v ry| j                         S )Nra   T)rc   rd   _impure_targets	is_impure)rw   unpack_nodess    r?   r  z/AutogradCompilerInstance.dce.<locals>.is_impure1  s2    |#?*t{{o/M>>##r>   zDCE removed %d nodes)r1   r   r   r_   rk   updater  r  r   r   rT   r8   r   rM  rm   rj   eliminate_dead_coder  r  )r]   r   rw   r  beforeafterr  s         @r?   dcezAutogradCompilerInstance.dce#  s   
 3=, !5!5!@!@M!@!RS 	3GAt

 12	3'(1,, C 34q899LQCP 	$EHHMM 	$d 	$ T^^))//000;DNN((../0&5.Ar>   c                    g }g }t        | j                  j                  j                        }t	        |       t	        |      }|j
                  dk7  rt        d|j
                   d      |j                  D ]e  }|j                  t        j                  urt        d|j                         |j                  r|j                  |       U|j                  |       g t               }|D ]  }t        |j                  t              s!t        dt!        |j                               |j                  d   |k7  rt        d      t        |j                  d   t"              s$t        d	t!        |j                  d                t%        |      }|j'                  |j                  d          |j                  d   |f|_         |D ]'  }	| j                  j                  j)                  |	       ) |S )
Nr   z#Expected placeholder 'sizes', got 'r  z&Expected operator.getitem target, got z$Expected used.args to be tuple, got r   z&Expected used.args[0] to be sizes_noderg   z%Expected used.args[1] to be int, got )ri   r   r_   rj   rh   rs   rT   r  rd   rn   ro   rr   setrM   rl   r   rq   rp   r   r   
erase_node)
r]   
used_sizesunused_sizesit
sizes_nodegetitem_nodeused_sizes_idxusednext_size_idxunuseds
             r?   remove_unused_sizesz,AutogradCompilerInstance.remove_unused_sizes=  s   
 $..&&,,-R"X
??g% 5joo5FaH  ',, 		2L""(*:*::$<\=P=P<QR  !!!!,/ ##L1		2 $'5 	6Ddii/$:4		?:KL  yy|z)$%MNNdiilC0$;D1<N;OP   /Mtyy|,1}5DI	6" # 	4FNN  ++F3	4 r>   r   c                 l    t        | j                  j                  | j                  j                  |      S rB   )r#   r   r   r_   )r]   r   s     r?   create_graph_modulez,AutogradCompilerInstance.create_graph_modulel  s%    4>>..0D0DbIIr>   c                 :     j                   j                  dt        j                  di         j                  j                           j                   j                  dd j                   j                   j                  |            fi        g t               r% j                   j                   j                         j                   j                  j                  D ]%  }dD ]  }||j                  v s|j                  |=   ' t        dd  fd        j                           j!                           j#                           j%                           j'                           j)                           j+                           j-                           j.                  r/ j.                  j1                   j                   j                          j3                          j5                  d	 j6                         t9        d
g       t;        dddd      }t<        j?                  d|       t@        jC                  d|       t        dfd       dtD        dtF        f   d
tF        dtF        dtF        dtF        dtF        dtH        tF        tF        f   f fd}tK               jM                  dtO        jP                         d j6                  i jR                  d        jT                  jW                  d d d        | jY                        fS )Nra   r=   re   )tensor_metaexample_valuer  r  c                      dddS )N&compiled_autograd_graph_pre_reorderingstringr  r=   r=   r>   r?   r  z6AutogradCompilerInstance.end_capture.<locals>.<lambda>  s    @$! r>   c                      t         j                  j                   j                  j                  d j                   d      j                  d      S )NCompiledAutogradPreReorderingFprint_output)r#   r   r   r_   r   print_readabler   s   r?   r  z6AutogradCompilerInstance.end_capture.<locals>.<lambda>  sG    {##$$"477)=9  n%n0	 r>   r  r  rz   zCompiled autograd graphT)include_deviceinclude_stridecoloredz%scompiled_autograd_graphc                  (     j                  d      S )NFr  )r  )r_   s   r?   r  z6AutogradCompilerInstance.end_capture.<locals>.<lambda>  s    u333G r>   )r   compiled_fn.r   r   r   packed_inputsr6   c           	         	 da j                  rj                  j                  |       g }t        |      D ]i  \  }}|v s|dkD  rI|j	                  t        j                  d|             t
        j                  j                  |d   d       Y|j	                  |       k D ](  }	||	   j                         j                  d      ||	<   * t               5  t        j                        5   | |||||      }
j                  rj                  j                  |
       |
cd d d        cd d d        da S # 1 sw Y   nxY wd d d        da y # 1 sw Y   da y xY w# da w xY w)NTr   r  rg   )non_blockingF)in_compiled_autograd_regionr   r   r   rr   r8   r  r   maybe_mark_dynamic
pin_memoryr  _disabler   r   r   )r!  rz   r   r   r   r"  filtered_sizesr   integerr   r   runtime_inputs_to_mover]   r	  s              r?   runtime_wrapperz=AutogradCompilerInstance.end_capture.<locals>.runtime_wrapper  sf   4.2+##$$55f=!#$-e$4 ;LCn,"Q;*11%++a2IJ!MM<<^B=OQRS*11':; 0 OA &q	 4 4 6 ; ; ; NF1IO Z !5dgg!> %C ''((..s3   /4+   /4+ /4+e+sO   A E! BE! E25D<'	E0	E! <E	EE! EE! E! !E%r4   r   r   )-r   r   r   _exec_final_callbacks_stubr   r   rO  
create_argrL  rG   r  r_   rj   r  r   delay_unpack_hook_nodesreorder_tensor_pre_hook_nodes'reorder_pre_hook_nodes_to_schedule_asapreorder_accumulate_grad_nodes%reorder_pre_hook_nodes_to_mimic_eager reorder_post_acc_grad_hook_nodesreorder_post_hook_nodesr   r   ry   r  r  r   r   r   compiled_autograd_loginfor  r  r   r   r   r   log_event_endr   r   r   r   __exit__r   )	r]   r]  rw   fieldlazy_graph_coder,  r_   r+  r	  s	   `     @@@r?   end_capturez$AutogradCompilerInstance.end_captureo  s   ##&AA		
 	

""^^&&t}}W'=>@		
 -/%'%)%B%B4>>CWCW%X"
 NN((.. 	)D@ )DII%		%()	)
 	1	
 	$$&**,446**,224--/$$& 	
,,T^^-A-AB 113((+;DGG9)EFEH:.0%
 	""49$0%G	

$	4!#s(+$	4$	4 $	4 	$	4
 $	4 $	4 38_$	4L 	"#11LLN!"& 	2 	
 	%%dD$7 0 0 777r>   c                 z    | D cg c]*  }t        |      t        j                  j                  u s)|, }}|S c c}w rB   )rq   r8   r   rM  )rl   r@  rj   s      r?   get_all_nodesz&AutogradCompilerInstance.get_all_nodes  s4     !=qDGuxx}}$<== >s   *88c                     | j                   dk(  sG| j                   dk(  r9| j                  t        j                  u r| j                  d   j                   dk(  ryy)Nr   ra   r   TF)rc   rd   rn   ro   rl   )rw   s    r?   is_placeholderz'AutogradCompilerInstance.is_placeholder  sF    77m#GG&x///		!=0r>   c                 *   | j                   j                  j                  dt              D ]  }|j                  d   |j                  d   |j                  d   }}}d}t        |t        j                  j                        r-|j                  t        j                  u r|}|j                  d   }t        | j                  |||g            }||j                  us| j                  |      r|j!                  |       ||j!                  |        y)a  
        Usage of AOTAutograd causes all the accumulate_grad_ nodes to get pushed to the end of
        the graph.  This differs from eager mode, which schedules them as soon as possible. This
        pass attempts to reorder the graph to mimic eager behavior.
        ra   rb   r   rg      N)r   r_   rk   r   rl   rM   r8   r   rM  rd   rn   ro   maxr>  prevr@  rr   )r]   rw   rx   variable_grad_node	grad_noder  args          r?   r2  z6AutogradCompilerInstance.reorder_accumulate_grad_nodes  s     NN((33'; 4 
 	-D 		!		!		! -6*J
  L9ehhmm4$$(8(88((--a0	""J0BI#NOC $))#D,?,?,D

4 +JJ|,-	-r>   c                     | j                   j                  j                  dt              D ]H  }|j                  j                  dd      dk7  r#t        |j                        }|j                  |       J y)zp
        We can delay unpack hooks until they are needed, even later than in the eager autograd engine.
        ra   rb   r  Nr  )	r   r_   rk   r   r   r   minr  prepend)r]   rw   
first_users      r?   r/  z0AutogradCompilerInstance.delay_unpack_hook_nodes  si     NN((33y 4 
 	%D {{{D1]BTZZJt$	%r>   c                 f   | j                   j                  j                  dt              D ]  }|j                  j                  dd      dk7  r#|j                  d   }|j                  d   }||j                  usP| j                  |      rb|j                  |       |j                  |        y)a  
        Usage of AOTAutograd causes all the tensor_pre_hook nodes to get pushed
        to the end of the graph. This differs from eager mode, which schedules
        them as soon as possible. This pass attempts to reorder the graph to
        mimic eager behavior.
        ra   rb   r  Nr  r   rg   )
r   r_   rk   r   r   r   rl   rD  r@  rr   )r]   rw   r  
input_nodes       r?   r0  z6AutogradCompilerInstance.reorder_tensor_pre_hook_nodes(  s     NN((33y 4 
 	*D {{{D15FF99Q<L1J*43F3Fz3R!!,/##D)	*r>   c                    | j                   j                  j                  dt              D ]X  }|j                  j                  dd      dk7  r$|j                  d   }| j                  |j                  d         }g }g }|g}|D ]o  }|j                  dk(  s|j                  t        j                  u s0|j                  |j                  d          |j                  |       |j                  |       q t        ||      D ]'  \  }}	|j                  |       |j                  |	       ) t        |      }
|
|j                   us| j#                  |
      r0|
j                  |       |D ]  }|j                  |        [ y)a  
        In this function, we schedule the pre hooks as soon as possible. This
        does not match eager behavior (schedule pre hook right before its
        registered node), but it can make acc grad be scheduled properly when
        the pre hooks are registered to them. After reordering acc grad node, we
        will reorder the pre hooks again to mimic eager behavior.
        ra   rb   r  Nr  r   rg   )r   r_   rk   r   r   r   rl   r>  rc   rd   rn   ro   rr   zipremoverC  rD  r@  )r]   rw   r  input_nodes	to_remove	to_append
hook_blockr@  abrG  s              r?   r1  z@AutogradCompilerInstance.reorder_pre_hook_nodes_to_schedule_asap<  sh    NN((33y 4 
 	+D {{{D1Z?99Q<L,,TYYq\:KIIJ  )44?*qxx8;K;K/K$$QVVAY/$$Q'%%a(	)
 Iy1 &1""1%""1%& k"C$))#D,?,?,D

<(# +A ''*+3	+r>   c                    g }| j                   j                  j                  dt              D ]3  }|j                  j                  dd      dk7  r#|j                  |       5 t        |      D ]  }|j                  d   }t        |j                  j                               }t        |      dk(  rDt        d |D              st        d      t        t!        |d   j                  j                                     }||j                  us|j#                  |       |j#                  |       |D ]  }|j#                  |         y)	a%  
        Usage of AOTAutograd causes all the pre_hook nodes to get pushed to the
        end of the graph. This differs from eager mode, which schedules them
        right before their registered node execution. This pass attempts to
        reorder the graph to mimic eager behavior.
        ra   rb   r  Nr  r   c              3   v   K   | ]1  }|j                   d k(  xr |j                  t        j                  u  3 yw)ra   N)rc   rd   rn   ro   r  s     r?   r   zQAutogradCompilerInstance.reorder_pre_hook_nodes_to_mimic_eager.<locals>.<genexpr>w  s8       ?*Nt{{h>N>N/NNs   79zIExpected all users of pre_hook to be operator.getitem call_function nodes)r   r_   rk   r   r   r   rr   reversedrl   ry  r  r  r   r  rT   rh   ri   rJ  )r]   	pre_hooksrw   hook_getitem_noder  registered_nodero   s          r?   r3  z>AutogradCompilerInstance.reorder_pre_hook_nodes_to_mimic_eager`  s9    	NN((33y 4 
 	#D {{{D1Z?T"	# Y' 	5D $		!*+E5zQ  !  %_  #4a(;(;(=#>?Odii/''(9:''-$ 5G#++G45)	5r>   c                 $   g }| j                   j                  j                  dt              D ]3  }|j                  j                  dd      dk7  r#|j                  |       5 t        |      D ]  }|j                  d   }|j                  d   }d}t        |j                  j                               D ](  }|j                  dk(  s|j                  t        u s&|} n |t        d      |j                  |       |j                  |        y)	a  
        Usage of AOTAutograd causes all the post_acc_grad_hook nodes to get
        pushed to the end of the graph. This differs from eager mode, which
        schedules them as soon as possible. This pass attempts to reorder the
        graph to mimic eager behavior.
        ra   rb   r  Nr  r   rg   z8post_acc_grad_hook must have corresponding acc grad node)r   r_   rk   r   r   r   rr   rY  rl   ry  r  r  rc   rd   r   rT   )r]   post_acc_grad_hooksrw   r  rx   acc_grad_noder@  s          r?   r4  z9AutogradCompilerInstance.reorder_post_acc_grad_hook_nodes  s    !NN((33y 4 
 	-D {{{D15II&&t,	- 01 	&D99Q<L1J !M***//12 44?*qxx;O/O$%M
 $$N 
   .%%	&r>   c           	      N   g }| j                   j                  j                  dt              D ]3  j                  j                  dd      dk7  r#|j                         5 t        |      D ]  j                  d   }j                  d   }j                  d   }t        |      dkD  r@g }|j                  t        |             |D ]=  }|j                  fd	t        |j                  j                               D               ? t        |      }|j                  dk(  r|j                   t"        u r|j                  d   }d}	t        |j                  j                               D ]G  }
|
j                  dk(  s|
j                   t        u s&|
j                  j                  dd      d
k(  sF|
}	I |	$|	j                  |       |j                         n|j$                  us~| j'                  |      r|j                  |       |j                          y)a  
        Usage of AOTAutograd causes all the post_hook nodes to get pushed to the
        end of the graph. This differs from eager mode, which schedules them as
        soon as possible. This pass attempts to reorder the graph to mimic eager
        behavior.
        ra   rb   r  Nr  r   rg   rB  c              3      K   | ]F  }|j                   d k(  r1|j                  t        u rj                  j	                  dd      dk(  s| H yw)ra   r  Nr  )rc   rd   r   r   r   )r   r  rw   s     r?   r   zCAutogradCompilerInstance.reorder_post_hook_nodes.<locals>.<genexpr>  sH      -?2 KK94 KKOOK>+M -s   AAr  )r   r_   rk   r   r   r   rr   rY  rl   r   extendry  r  r  rC  rc   rd   r   rD  r@  )r]   
post_hooksr  rv   rQ  input_nodes_and_usersrM  rG  rx   post_acc_grad_hook_noder@  rw   s              @r?   r5  z0AutogradCompilerInstance.reorder_post_hook_nodes  s    
NN((33y 4 
 	$D {{{D1[@d#	$ Z( )	*D99Q<L99Q<L))A,K< 1$ %'!!((k):;) 	
%,, - $Z%5%5%:%:%< =- 	 +,Cvv(SZZ;O-O XXa[
*.'j..3356 4A/HH	1HHLLd;?SS23/4 +6+22<@ ''-$))#D,?,?,D

<(##D)S)	*r>   tc                 Z    |y t        |t              r|D cg c]  } j                  |       c}S t        |t              rt         fd|D              S t        |t        j
                  t        j                  f      r j                  |j                     S t        |t        j                        s|S t         j                  |      }t        |t        j                  j                  j                  j                        st!        dt#        |             |j$                  S c c}w )Nc              3   @   K   | ]  }j                  |        y wrB   r  )r   rH   r]   s     r?   r   z4AutogradCompilerInstance.to_proxy.<locals>.<genexpr>  s     5aq)5s   zExpected _ProxyTensor, got )rM   ry  rL  r   r8   SymIntSymFloatr   rw   r   r'   r   r   r  proxy_tensor_ProxyTensorrT   rq   r  )r]   rf  rH   rk  s   `   r?   rL  z!AutogradCompilerInstance.to_proxy  s    9a./0DMM!$00a51555a%,,78,,QVV44!U\\*H)$..!<%((//<<II
 !#>tL?Q>R!STT!!! 1s   D(objectsr  c                 \   t        |t        j                  j                        r|rt	        |      t	        |      k7  r$t        dt	        |       dt	        |       d      g }t        t	        |            D ]1  }||   \  }}| j                  ||d        |j                  ||          3 |}n$t        t	        |            D cg c]  }||   	 }}t	        |      t	        |      k7  r$t        dt	        |       dt	        |       d      t        ||d | j                         |S c c}w )Nzorigins length (z) != objects length (r  zobjects length (z) != proxies length (constanttracer)rM   r8   r   r3   r   rT   r   set_node_originrr   r*   r   )r]   rm  r  r   bound_proxiesr   nodecall_indexr8  s           r?   r   z0AutogradCompilerInstance.bind_objects_to_proxies  s#    guxx~~.w<3w</(*3w<.8McRYl^[\]  !#s7|, 5A07
-NI((NDI!((45 (/4S\/BC!71:CCw<3w<' "3w<.0Ec'l^STU  	'7T$..Q Ds   D)indexc                     | j                   t        d      | j                   |   }t               }t        ||d | j                         |S )Nz2hooks_proxy must be set before bind_backward_statero  )r   rT   r$   r*   r   )r]   ru  r  bw_states       r?   bind_backward_statez,AutogradCompilerInstance.bind_backward_state  sH    # !UVV  ' ?(EDPr>   r8  rt  pyobjc                    d}|;|j                   }t        |d      r#|j                  t        d      |j                  }| | d| d}t        j                         j                         d   }|j                  d|      }t        |       y )N rs  zThis compiled backward function was saved by AOTAutogradCache, which does not support
                    compiled autograd. Please turn off AOTAutogradCache using `TORCHINDUCTOR_AUTOGRAD_CACHE=0`.z (NodeCall r  r  z:raw_stack_trace = CapturedTraceback.extract().format()[-1])
rp  r   rN   r   rs  r2   extractformatreplacer.   )	r]   r8  rt  ry  maybe_aot_idforward_clsnew_coderaw_stack_tracenew_stack_traces	            r?   rr  z(AutogradCompilerInstance.set_node_origin  s     ,,K{I.22:&s   +22[k.9IK+335<<>rB)11H(
 	(r>   rB   r   )Ir   r   r   r   r   r^   r8   r   r   r"   r   staticmethodr   r   r   ry  rp   r0   r/   r   rm   r  r!  r   r  autogradfunctionBackwardCFunctionr  r  r  rk  r  r  r  r  r  r  rX   r   r3   r  r  r  r  r  r  r  r   r  r  r   r  r  r#   r  r<  rM  r>  r@  r2  r/  r0  r1  r3  r4  r5  rL  r   r$   rx  Functionrr  r=   r>   r?   r   r   1  s   .HS#X$6 .4 .C5<< C$ C: C
 5S 5s 5} 5 5y
U\\"y
 Cyy
 kM12	y

 d5c?+,y
 y
 y
 
sD&[(94;NN	Oy
v
c
 

^#^ !.^  -	^
 ^ ^ #&*^  $Cy^ 
#^@:: #3-:  -	:
 : ^^$$66: #&*:  $Cy: 
u||d"C'	(:xGG SMG sm	G
 !G SMG smG !G 
%,,	G*
"4.
 
 c]	

 LL
 
%,,	
/ /
'U\\ 'ell 'FF S#XF !	F
 F 
F	D	D }	D 		D
 "#	D 
%,,		D
38$
,/
BJ3-
	%,,	
'}47JRSV-	%,,	 #    '+,, ||d* llT!	
 t 
	B
S#X&
/2
>A
	
3   5<<(36;>	ell	 5<<(36;>	ell	x} s tELL?Q ELL)3;ELL3ITW	ell	"\\,/	ell	.4ehhnn 4c 4l
 
 
B4-SX -^Jc Jk J|83 |85#s(1CS1H+I |8| HSM d588==.A  
 UXX]] t  -<%*("+H$5L#&J8*t"# "# ". 15	#  eCHo&-	
 
#8  )) ) ~~&&-	)
 
)r>   r   Fr   dynamicignore_active_disable_ctx)NNNc              #   4  K   |st         rd  y |r(t        |      t        urt        dt        |             ddlm} |j                  j                  dk(  r
da	 d  day t        j                  j                         rddlm} t        j                  j                  j                   j#                  t%        j&                  t(        |       |      \  }}t+               r7t        j                  j                  j                   j-                  t.               dat2        }t2        dz  a	 t        j4                  j7                  d      5  d  d d d        |sdat        j                  j                  j                   j#                  ||       t2        dz  at2        |k7  rt        d	      y # daw xY w# 1 sw Y   fxY w# |sdat        j                  j                  j                   j#                  ||       t2        dz  at2        |k7  rt        d	      w xY ww)
Nz!Expected dynamic to be bool, got r   )
eval_frameforce_eagerTF)cudagraph_treesrg   zINested Compiled Autograd Contexts must return before their parent context)active_disable_ctxrq   rm   rT   torch._dynamor  _stancestance%compiled_autograd_enabled_force_eagerr8   r  is_availabletorch._inductorr  r  r   r4   set_autograd_compiler	functoolspartialr   r@   set_verbose_loggerr  compiled_autograd_enableddepthr  set_multithreading_enabled)r   r  r  r  r  prior_compilerprior_dynamicprior_depths           r?   _enabler  C  s    4 %);G}D($7WG  	-$$5 591>8=5 zz&&(;
   22HH!!":KH' 01  22EEkR(,%KQJE^^>>uE  &05-  22HH"M 
K'(c  (= 9>5*  &05-  22HH"M 
K'(c  (sQ   AHF+  CH)F> F2F> AH+F//H2F;7F> >AHHc               #     K   t         j                  j                  j                  j	                  d d      \  } }dat        sda	 d  | rdadat         j                  j                  j                  j	                  | |       y # | rdadat         j                  j                  j                  j	                  | |       w xY ww)NFT)r8   r  r   r4   r  r  r  )r  r  s     r?   r(  r(    s     
 	**@@uM !&!
(,%"**@@M	
 (,%"**@@M	
s   ACB 	;C<C  Cc                     da t        rt        d      t        j                  j
                  j                  j                  d d       t        j                  j
                  j                  j                  d        t        j                  j
                  j                  j                          t        j                         ay )NFz6Cannot reset while in_compiled_autograd_region is True)r  r%  rT   r8   r  r   r4   r  r  clear_cache	itertoolsr   r   r=   r>   r?   resetr    s}     %"UVV	HH&&<<T5I	HH&&99$?	HH&&224oo'Or>   rz   r  r  r  r  r  r  c                     | d   }|j                  ||      }|t        d      |j                  |       ||z
  }	|j                  |||	      }
||
|
j	                  t
        j                        gS )Nr   z-grad must not be None in copy_slices_prologue)memory_format)new_empty_stridedrT   copy_
as_stridedcloner8   contiguous_format)rz   r  r  r  r  r  r  r}   r   offsetr  s              r?   r  r    s|     !9D##J=F|LMM
LL #66F"":|VDJJ
 0 0u?V?V 0 WXXr>   r  r   r  r  c                     d gt        |       z  }t        t        |             D ]D  }| |   s	||   |dk(  r)||   }|t        d      |j                  |       |||<   =||   ||<   F |S )Nr   z/res[0] must not be None for needs_input_grad[0])r   r   rT   r  )r  r   r  r  grad_inputsr   to_copys          r?   r  r    s     /3Vc:J6K-KK3'() (A1v~Ava&?(I    )!'A!$QA( r>   )TTr   )r__doc__r   r  r  rn   r   collectionsr   r   collections.abcr   r   r   typingr   r	   r8   torch.utils._pytreeutils_pytreeri  torch._dispatch.pythonr
   torch._dynamo.external_utilsr   r   r   r   r   torch._dynamo.sourcer   r   torch._dynamo.utilsr   r   r   r   /torch._functorch._aot_autograd.runtime_wrappersr   r   torch._guardsr   r   r   r   torch._loggingr   r   torch._prims_commonr    torch._subclassesr!   torch._subclasses.fake_tensorr"   torch.fxr#   %torch.fx.experimental._backward_stater$   "torch.fx.experimental.proxy_tensorr%   r&   r'   r(   r)   r*   %torch.fx.experimental.symbolic_shapesr+   r,   torch.fx.tracebackr-   r.   torch.typesr/   r0   torch.utils._ordered_setr1   torch.utils._tracebackr2   torch.fx.proxyr3   r   r   r6  r  rm   r@   rG   r   rJ   rU   rW   r   r   rP  r   r-  r  r   r   rp   r   r   r  r  r%  r  r  contextmanagerr  r(  r  ry  r  r  r=   r>   r?   <module>r     s         , 9 9 %  $ $ ;  <  M L > 6 , 4   ?  G B 2 / 4 $X *(4GH *EF$ 4D 45<<$& 5<<$+> 
 
c0B 
6Z ZD"# "#J5 5  m M "==	 ")//#	s 	s 	A) A)J( "  ). % $  	  &*M#s(#MM  $M  	M M` 
),- 
 
,	(YU\\"Y%Y ;'Y %	Y
 %Y ;'Y %Y 
%,,Y*tnLL 
%,,%	& 	
 
%,,
r>   