
    ^jG                        d Z ddlZddlZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlmZ ddlmZ ddlmZ d	d
lmZmZ d	dlmZmZmZmZmZmZ d	dlmZmZ ddlmZ ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+ erddl,m-Z-  G d de.      Z/ G d de.      Z0 ee1d      Z2dejf                  de4fdZ5 G d de+      Z6y) a
  
This module implements variable tracking for PyTorch optimizers during Dynamo tracing.

The OptimizerVariable class provides specialized handling for optimizer instances by:
- Optimizing the tracing of expensive optimizer initialization
- Managing optimizer state and parameter group tracking
- Handling tensor sources and guards for optimizer state tensors
- Supporting CUDA graph execution through static tensor address management
- Providing special handling for parameter gradients and optimizer state tensors

Key features include:
- Efficient initialization tracing via _init_group optimization
- Automatic marking of optimizer state tensors as static for CUDA graphs
- Proper source tracking for parameter groups, gradients, and state tensors
- Guard installation for optimizer state structure
- Support for both CPU and GPU tensor handling
- Cleanup of static tensor references via finalizers

The module integrates with Dynamo's broader tracing system while providing
optimizer-specific optimizations and safety guarantees.
    N)Iterable)AnyTYPE_CHECKING)TensorVariable)Source)getArtifactLogger)tree_map_only   )GuardBuilderinstall_guard)
AttrSourceConstDictKeySourceDictGetItemSourceGetItemSourceGlobalWeakRefSource
GradSource)GLOBAL_KEY_PREFIXunpack_iterable   )VariableTracker)ConstantVariable)ConstDictVariable)HashableTracker)ListVariable)GetAttrVariable)UserDefinedObjectVariable)InstructionTranslatorBasec                       e Zd Zy)ArgMappingExceptionN__name__
__module____qualname__     l/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_dynamo/variables/optimizer.pyr   r   9       r%   r   c                       e Zd Zy)GuardInstallExceptionNr    r$   r%   r&   r)   r)   =   r'   r%   r)   
perf_hintsxreturnc                 ,   ddl m} | j                  r || j                  j                  d      }t
        j                  j                  j                  |       d u}|r6|j                  t        d      |xs |j                  j                  |       S |S y)Nr   )get_managerFz&CUDA graph manager has no current nodeT)torch._inductor.cudagraph_treesr.   is_cudadeviceindextorch_dynamoutilsget_static_address_typecurrent_nodeAssertionError_is_cuda_graph_recorded_tensor)r+   r.   manageris_static_addresss       r&   _is_static_for_cudagraphsr<   D   s    ;yyahhnne4!MM//GGJRVV##+$%MNN! J''FFqI
 %$ r%   c                       e Zd Zdddhej                  Z	 	 	 ddej                  j                  dee	e
f   dz  dee   dz  deej                  ef   dz  de	ddf fdZd	d
dedee   deeef   ddf
 fdZd	d
dedef fdZddZddZde	de	deee	   eee	f   f   fdZddZddZd	d
dej                  defdZd	d
dee   de	dee	   de	ddfdZddZ xZS )OptimizerVariablegrad_to_sourcetensor_to_sourcestatic_tensor_namesNvaluekwargsr,   c                     t        |   |fi | || _        |xs i | _        |xs i | _        |xs
 t               | _        y N)super__init__rB   r?   r@   setrA   )selfrB   r?   rA   r@   rC   	__class__s         r&   rG   zOptimizerVariable.__init__`   sG     	)&),1
,2 0 6B#6#?#% r%   txr   nameargsr   c                 `   |dk(  rt        | j                  d      st        
|   ||||      S 	 | j	                  |       | j                           | j                  |i |\  }} | j                  j                  |i |}| j                  |       | j                  |||||       dt        | j                         }|j                  || j                         | j                  |       t        j                  |      S t        
|   ||||      S # t        t         f$ r
}	Y d}	~	&d}	~	ww xY w)zVThis is an optimization to avoid tracing the very slow initialization of the optimizer_init_group__optimizer_N)hasattrrB   rF   call_methodgraph_break_if_pending_mutationmove_step_if_cpuget_python_argsrO   map_sources_and_install_guardsupdate_list_argsidstore_global_weakref_by_idcreate_finalizerr   creater   r)   )rI   rK   rL   rM   rC   py_args	py_kwargsret_valmangled_name_rJ   s             r&   rR   zOptimizerVariable.call_methodo   s*    = 4::}5w*2tT6BB44R8%%'%9T%9%94%J6%J"0$**00'GYG33B7%%b$K ".bn-=>--lDJJG%%b) (..w77
 w"2tT6::	 ()>? s   CD D-(D-c           
      p   |dv rW| j                   st        d      t        | |t        t	        | j
                  |            t        | j                   |            S |dk(  rFddlm} | j
                  j                  D ]  }|d   D ]  } ||d	         | j                  |       t        | 1  ||      S )
NrO   z3OptimizerVariable requires a source for var_getattr)py_typesourceparam_groupsr
   mark_static_addressparamsTguard)rc   r8   r   typegetattrrB   r   
decoratorsrf   rd   _set_capturablerF   var_getattr)rI   rK   rL   rf   groupprJ   s         r&   rn   zOptimizerVariable.var_getattr   s     M";;$I  #WTZZ67!$++t4	  >!800 7x 7A'677   $w"2t,,r%   c           	         | j                   j                  D ]t  }|d   D ]j  }|j                  j                  }|j                  j                  t        |      d       }|sA|j                  |      sSddlm	}  |dd| d| dg        l v y )	Nrg   r
   )unimplementedz(optimizer: pending mutation on parameterz
variable: z, parameter: zSPending mutations on a parameter (e.g. due to using closure) require a graph break.)gb_typecontextexplanationhints)
rB   rd   outputside_effectsid_to_variablegetrX   has_pending_mutationexcrr   )rI   rK   grp   rx   variablerr   s          r&   rS   z1OptimizerVariable.graph_break_if_pending_mutation   s    
 (( 	Ax[ !yy55'66::2a5$G A A( K3! J",XJmA3 G$y 		r%   c                      ddl m} dt        t        t        f   dt
        f fd} j                  j                  D ]  } ||      sd|d<     j                  xr t         j                  d      }|j                  t        j                  | j                  j                  |            }|j                  D ]B  }t        t        j                   d            }t        j                   d      |j                  |<   D y )	Nr   LazyVariableTrackerro   r,   c                     d}d}| j                  dg       D ]:  }||j                  xs |j                  z  }||j                  j                  vz  }< d| v xr |xr |S )NTrg   
capturable)rz   r0   is_xpurB   state)ro   all_uninitializedall_gpurp   rI   s       r&   safe_to_set_capturablezAOptimizerVariable._set_capturable.<locals>.safe_to_set_capturable   sq     $GYYx, ?19900!Qdjj.>.>%>>!?  5(J->J7Jr%   Tr   rd   ) r   dictstrr   boolrB   rd   rc   r   realize_allr   builditemsr   r   r[   )	rI   rK   r   r   ro   rc   param_groups_vtparam_group_vtkeys	   `        r&   rm   z!OptimizerVariable._set_capturable   s    )	K$sCx. 	KT 	K ZZ,, 	+E%e,&*l#	+ HDKK!H-99!!"djj&=&=vF
 .33 	FN!"2"9"9,"GHC(8(?(?(EN  %	Fr%   c                      dt         dt         f fd}|D cg c]
  } ||       }}|j                         D ci c]  \  }}| ||       }}}||fS c c}w c c}}w )z9Get python values equivalent to the variable tracker argsargr,   c                    t        | t              r | j                         r| j                         S t        | t              r| j
                  sg S t        | t              rt        | j                  t              rtt        | j                  j                  t              rP| j                  j                  j                  dk(  r-j                  j                  | j                  j                     S t        )Nrd   )
isinstancer   is_python_constantas_python_constantr   r   r   rc   r   baser   memberrB   rd   r2   r   )r   rI   s    r&   map_argz2OptimizerVariable.get_python_args.<locals>.map_arg   s    #/C4J4J4L--//C.syy	3 12szz=9szz
;JJOO**n<zz..szz/?/?@@%%r%   )r   r   )	rI   rM   rC   r   r   new_argskv
new_kwargss	   `        r&   rU   z!OptimizerVariable.get_python_args   sg    
	& 	& 	&  -11SGCL1106?1am?
?## 2?s
   AAc                     | j                   j                  j                         D ];  \  }}d|v s|d   j                  s|d   j	                  |j
                        |d<   = y )Nstep)rB   r   r   is_cputor1   )rI   rp   r   s      r&   rT   z"OptimizerVariable.move_step_if_cpu   sW    

((..0 	;HAu5=#7#7 %f 0 0 :f	;r%   c                    ddl m ddlm} i | _        i | _        dt        dd ffd}t        t        j                  || j                  j                         | j                  xr t        | j                  d      }|j                  t        j                   || j                  j"                  |            }| j                  xr t        | j                  d	      }t        j                   || j                  j                  |      }|j%                          |t'        d
      |j(                  j*                  j-                  |       t/        | j                  j"                  |j0                        D ]  \  }}	t3        |d         dkD  r|d   D ]  }
|
j4                  d }t7        | j                  j                  j9                               D ]  \  }}||
u s|} n |sR|j                  t        j                   || j                  j                  |
   t;        |t=        ||                          n |	j?                  |tA        jB                  d            }d}g }t/        |d   tE        ||            D ]  \  }}|j                  }|t'        d| d      || j
                  |<   tG        |d      }|j4                  C|| j                  |j4                  <   tI        |j4                        ryd}|jK                  |       tM        |jO                  tP        jR                                |rtT        jW                  tX        jZ                        s|D cg c]  }|j\                   }}tT        j_                  d|        t7        | j                  j                  ja                               D ]  \  }}t;        |t=        ||            }|j(                  j*                  j-                  |       t7        |ja                               D ]a  \  }}tc        |t        j                        s!|| j                  vs0|| j
                  vs?t;        |t=        ||            | j
                  |<   c  y c c}w )Nr
   re   r   r   r+   r,   c                      | d       y )NTrh   r$   )r+   rf   s    r&   mark_staticzEOptimizerVariable.map_sources_and_install_guards.<locals>.mark_static  s    .r%   rd   r   zstate_source must not be Nonerg   r   Tzparam z does not have a sourcegradF)zGrad tensors %s will be copied during cudagraphs execution.If using cudagraphs and the grad tensor addresses will be the same across runs, use torch._dynamo.decorators.mark_static_address to elide this copy.)2rl   rf   lazyr   r?   r@   r   r	   r3   TensorrB   r   rc   r   r   r   r   rd   realizer8   rw   guard_on_key_orderaddzipr   lenr   	enumeratekeysr   r   getitem_constr   r[   r   r   r<   appendr   
make_guardr   CONSTANT_MATCHperf_hint_logisEnabledForloggingDEBUGrL   warningvaluesr   )rI   rK   r   r   params_groups_sourcer   state_sourcestate_vtro   group_vtparam	key_indexir   	params_vt
all_staticnon_static_gradsrp   p_vtparam_sourcegrad_sourcesrcnon_static_grad_namesidxrB   p_state_source	inner_idxr   rf   s                               @r&   rV   z0OptimizerVariable.map_sources_and_install_guards  s   4-  "	/3 	/4 	/ 	ellK1A1AB  ${{Vz$++~/V-99!!"djj&=&=?ST
 {{Gz$++w'G"((TZZ-=-=|L 	 !@AA
		$$((6  #4::#:#:O<Q<QR 8	OE8 5?#a'"8_ "Ezz-$(	$-djj.>.>.C.C.E$F &DAq Ez,-	 %& %/;; / 5 5$&$(JJ$4$4U$;$5(4(:<(S%&!"	 "%"( !..r3C3J3J83TUIJ!uXI0NO W4#{{'(6!4K)LMM+7%%a(( 
 66%2=D''/4QVV<%*
(//<!+"8"89T9T"UV!W( -"<"<W]]"K=M(Nc(N%(N%%
 *c8	x $DJJ$4$4$;$;$=> 	JC.0sCN II((,,^< )%,,. 9 	1q%,,/!4!44!6!66/@&(:>9(U0D))!,	 )Os   Q7tensor_valuec                    ddl m} || j                  v rX ||d       | j                  |   }| j                  j	                  |j
                  j                  |j                               n|| j                  v r| j                  |   }ni ||d       |j                  t        |      }t        |      }| j                  j	                  |j
                  j                  |j                               t        j                  |||      S )z%Wrap state tensor in a TensorVariabler
   re   Trh   )rl   rf   r@   rA   r   rw   module_key_namerL   r?   rY   r   r   r   r   )rI   rK   r   rf   rc   global_names         r&   wrap_tensorzOptimizerVariable.wrap_tensoro  s     	5 4000D9**<8F$$(()B)B6;;)OPT000((6F  D9778I<XK(5F$$(()B)B6;;)OP$$Rv>>r%   r\   r]   c           	      ,   t        ||      D ]  \  }}t        |t              st        |t              st	        d      t        |      D ]  \  }}	|j                  j                  j                  |       t        |	t        j                        r,|j                  j                  | j                  ||	             q|j                  xr t        |j                  |      }
|j                  j                  t!        j"                  ||	|
               y)z7Update the args and kwargs to the traced optimizer callz-py_arg should be a list in optimizer variableN)r   r   r   listr8   r   rw   rx   mutationr3   r   r   r   r   rc   r   r   r   )rI   rK   rM   rC   r\   r]   r   py_argr   valrc   s              r&   rW   z"OptimizerVariable.update_list_args  s     tW- 	QKC#|,!&$/(G  (/ QFAsII**33C8!#u||4		(()9)9"c)BC!$!Lcjj!0L		(()>)>r3)OPQ	Qr%   c                     | j                   | j                  |j                  j                  dt        j
                  j                  dd ffd}|j                  j                  |       y )Ngmr,   c                 D     d fd}t        j                  |       y )Nc                     D ]  } j                   j                  | d        j                  j                  | d        j                  rj                  j	                          j
                  snj
                  j	                           y rE   )_bufferspop_parametersparams_flatclearparams_flat_unwrap_subclasses)rL   r   names_to_deletetcs    r&   clear_static_tensor_refsz\OptimizerVariable.create_finalizer.<locals>.init_finalizer.<locals>.clear_static_tensor_refs  sl    + ADKKOOD$/NN&&tT2~~,,.7788>>@Ar%   r,   N)weakreffinalize)r   r   r   r   rB   s   ` r&   init_finalizerz:OptimizerVariable.create_finalizer.<locals>.init_finalizer  s    A U$<=r%   )rA   rB   rw   tracing_contextr3   fxGraphModuleadd_graph_finalizer)rI   rK   r   r   r   rB   s      @@@r&   rZ   z"OptimizerVariable.create_finalizer  sW    22

YY&&
	>uxx33 
	> 
	> 			%%n5r%   )NNN)rK   r   r,   Nr   ) r!   r"   r#   r   _nonvar_fieldsr3   optim	Optimizerr   r   r   rH   r   r   r   rG   r   r   rR   rn   rS   rm   tuplerU   rT   rV   r   r   r   rW   rZ   __classcell__)rJ   s   @r&   r>   r>   X   s    
#	1	1	N 8</3>B@{{$$@ S*_-4@ !X_	@
 u||V34t;@ @ 
@";'"; "; ?#	";
 S/)*"; 
";H---58-	-:&F<$$$'$	tCy$sCx.(	)$>;
hT?-?=B\\?	?8Q'Q 'Q 	Q
 #Q Q 
Q.6r%   r>   )7__doc__r   r   collections.abcr   typingr   r   r3   torch._dynamo.variables.tensorr   torch._guardsr   torch._loggingr   torch.utils._pytreer	   guardsr   r   rc   r   r   r   r   r   r   r5   r   r   r   r   constantr   dictsr   hashabler   listsr   miscr   user_definedr   torch._dynamo.symbolic_convertr   	Exceptionr   r)   r!   r   r   r   r<   r>   r$   r%   r&   <module>r     s   ,   $ %  9   , - 0  7 ! & $ %  ! 3 H	) 		I 	 "(L9 $ ([61 [6r%   