
    ^j&A                     n   U d Z ddlZddlmZ ddlmZ ddlmZ ddl	m
Z
mZ ddlZddlmZmZ 	 ddlmZ  ee      Zee
ee
   f   Zeed	<   eeee   f   Zeed
<   daedz  ed<   da eed<   d1dZ!d1dZ"defdZ#defdZ$de
dedz  fdZ%de
dee
   fdZ&de
dee
   fdZ'ddddee
   dedz  dedeee
f   fdZ( ee      Z)eeee
   f   ed<   da*ee
   dz  ed<   dee
   fd Z+g Z,eee
ee   f      ed!<   da-edz  ed"<   ed#e.ee.e
f   z  fd$       Z/d1d%Z0d&ejb                  jd                  ddfd'Z3deeee
   f   fd(Z4d1d)Z5d*a6eed+<   i Z7eeef   ed,<   dejb                  jp                  defd-Z9d.e.defd/Z:edejb                  jp                  d#e.ee.e
f   z  fd0       Z;y# e$ r dZY ~w xY w)2a  Annotate CUDA graph kernel nodes during capture.

During CUDA graph capture, ``mark_kernels`` records the current capture
frontier and the direct dependents already attached to that frontier.
On scope exit it walks only the newly added dependent edges to find the
nodes created within the scope. Each kernel or memcpy node found is
annotated by its ``toolsId`` so it can later be matched to profiler
trace events.

``mark_kernels`` now snapshots capture state from whatever stream is
current on scope entry, so that stream must already be participating in
the capture. ``mark_stream`` handles this by starting ``mark_kernels``
before switching to the target stream.

The annotations can be pickled and later merged into a Chrome profiler
trace using ``torch.cuda._annotate_cuda_graph_trace``.

Requires ``cuda.bindings`` package and a CUDA driver that supports
``cudaGraphNodeGetToolsId`` (CUDA >= 13.1 or appropriate cuda-compat).
When unavailable, ``mark_kernels`` silently becomes a no-op.

Usage during capture::

    from torch.cuda._graph_annotations import (
        enable_annotations,
        mark_kernels,
        resolve_pending_annotations,
        remap_to_exec_graph,
    )

    enable_annotations()

    with torch.cuda.graph(graph):
        with mark_kernels("phase_A"):
            y = workload_a(x)
        with mark_kernels("phase_B"):
            z = workload_b(y)
        resolve_pending_annotations()

    remap_to_exec_graph(graph)
    N)defaultdict)contextmanager)	getLogger)Any	TypeAlias)_check_cuda_bindings_HAS_CUDA_BINDINGS)runtime_CaptureState_ExistingDirectDependents_tools_id_availableF_annotations_enabledreturnc                      da y)z#Enable kernel annotation recording.TNr        h/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/cuda/_graph_annotations.pyenable_annotationsr   L   s
      r   c                      da y)z$Disable kernel annotation recording.FNr   r   r   r   disable_annotationsr   R   s
     !r   c                  $   t        t        d      s6t        j                  j	                  d      } t
        j                  d|        yt        j                  d      ^}}|t        j                  j                  k(  rt
        j                  d       yy)zProbe whether cudaGraphNodeGetToolsId is supported by the driver.

    Calls with a null node: cudaErrorInvalidValue means the API exists
    in the driver (good), cudaErrorCallRequiresNewerDriver means it
    does not (bad).
    cudaGraphNodeGetToolsIdzcuda-bindingszcudaGraphNodeGetToolsId API not found in cuda-bindings. Current version: %s, required: >= 13.1.0. CUDA graph kernel annotations will be disabled. To enable annotations, upgrade cuda-bindings: pip install --upgrade cuda-bindingsFr   zvcudaGraphNodeGetToolsId requires a newer driver (missing cuda-compat?); CUDA graph kernel annotations will be disabledT)hasattr_cuda_runtime	importlibmetadataversionloggerwarningr   cudaError_t cudaErrorCallRequiresNewerDriverinfo)cuda_bindings_versionerr_s      r   _probe_tools_idr'   X   s     =";< ) 2 2 : :? K2
 "	
 33	GC! 	$$EE	F 	=	

 r   c                  L    t         syt        t         S t               at         S )z5Return True if cudaGraphNodeGetToolsId is not usable.T)r	   r   r'   r   r   r   _is_tools_id_unavailabler)   |   s+     &&&&)+"""r   streamc                     t        t        j                  |             \  }}}}}}|t        j                  j                  k7  ry|t        |d|       fS )zBReturn ``(graph, frontier)`` for an active capture, else ``None``.N)r   r   cudaStreamGetCaptureInfocudaStreamCaptureStatuscudaStreamCaptureStatusActivelist)r*   status_idgraph_deps
_edge_data	_num_depss          r   _get_capture_stater6      s`    7K..	
84FCz9 	00NN	O $uZi()))r   r2   c                     t        t        j                  |             \  }}|dk(  rg S t        t        j                  | |            \  }}t        |d|       S )z,Return the current root nodes for the graph.r   )pNumRootNodesN)r   r   cudaGraphGetRootNodesr/   )r2   r&   	num_rootsrootss       r   _get_root_nodesr<      se    '++	
LAy
 A~	+++	
E9
 jy!""r   nodec                     t        t        j                  |             \  }}}|dk(  rg S t        t        j                  | |            \  }}}t        |d|       S )z-Return the direct dependents of a graph node.r   )pNumDependentNodesN)r   r   cudaGraphNodeGetDependentNodesr/   )r=   r&   num_dependents
dependentsr4   s        r   _get_dependent_nodesrC      sk    /44	
Aq.
 	-A44^	
.*J
N
 
?N+,,r   )existing_direct_dependentsinclude_start_nodesstart_nodesrD   rE   c                   |xs i }| D ch c]  }t        |       }}i }t        |       }|r| D ]  }||t        |      <    |r|j                         }|j                  t        |      t	                     }t        |      D ]=  }t        |      }	|	|v s|	|v r|j                  |	       |||	<   |j                  |       ? |r|S c c}w )a)  Walk dependent edges starting at ``start_nodes``.

    ``existing_direct_dependents`` maps each node in ``start_nodes`` to
    the direct dependent node keys that were already present at scope
    entry. Those edges are skipped so the traversal only follows nodes
    added after scope entry.
    )intr/   popgetsetrC   addappend)
rF   rD   rE   r=   seendescendantsstackold_dependents	dependentdependent_keys
             r   _collect_descendantsrT      s     "<!Ar"-.$CI.D."$KE 	*D%)KD	"	* yy{377D	35I-d3 	$I	NM.-42GHH]#)2K&LL#	$  ' /s   C_kernel_annotations_ANNOTATABLE_TYPESc                      t         4t        j                  j                  t        j                  j                  ha t         S )N)rV   r   cudaGraphNodeTypecudaGraphNodeTypeKernelcudaGraphNodeTypeMemcpyr   r   r   _get_annotatable_typesr[      s6    !++CC++CC
 r   _pending_scopes_last_capture_graph_id
annotationc              #     K   t         r
t               rd yt        | t              rd| i} t	        j
                  t        j                  j                         j                        }t        |      }|d y|\  }}d}|D ci c].  }t        |      t        |      D ch c]  }t        |       c}0 }}}|s!t        |      D ch c]  }t        |       }}d |rt        ||      }	n>t        |      D cg c]  }t        |      |xs
 t               vr| }
}t        |
d      }	|	syt!               }g }|	j#                         D ]R  }t%        t	        j&                  |            }||vr&|j)                  t%        t	        j*                  |                   T |rt,        j)                  | |f       yyc c}w c c}}w c c}w c c}w w)aw  Context manager that records new scope nodes for later annotation.

    During capture, records the current stream frontier and its existing
    direct dependents on entry. On scope exit, traces only the dependent
    nodes added since entry. After capture, ``resolve_pending_annotations``
    merges overlapping scopes and stores the final toolsId annotations.
    If the scope is the first captured work, the entry frontier is empty,
    so ``mark_kernels`` falls back to the newly created graph roots.

    Must be called inside an active ``torch.cuda.graph()`` capture. The
    nodes you expect to annotate must be reachable from the stream frontier
    that is current on entry. If work runs on a different already-capturing
    branch, it must first be synchronized with the current stream so that
    branch becomes reachable from the entry frontier. If the current stream
    is not capturing, or if ``cudaGraphNodeGetToolsId`` is not available,
    the context manager is a no-op.

    Args:
        annotation: Arbitrary object appended to the annotation list for
            every kernel/memcpy node captured within this scope.
    Nstr
init_value)rD   T)rE   )r   r)   
isinstancer`   r   cudaStream_ttorchcudacurrent_streamcuda_streamr6   rH   rC   r<   rT   rK   r[   valuesr   cudaGraphNodeGetTyperM   r   r\   )r^   r*   capture_stater2   frontierentry_root_keysr=   depentry_direct_dependentsscope_nodes	new_rootsannotatable	tools_ids	node_types                 r   mark_kernelsru      s    .  #;#=*c"Z(
''::,,.::F 'v.M#OE8'+OPXHLD	(<T(BCCHCC  1@1GH3t9HH	*'>
 (.
4y!9CE: 
	 

 +9$O(*KI""$ 
(..
	
 K' 55	

  
I67 S D I
sC   BGG"G 4G9GG#G "G"BG GGc                     t         sy	 t        t              } t         D ]   \  }}|D ]  }| |   j                  |        " | rt	        t        |             dz	  a| j                         D ]  \  }}t        |      dk(  rt        |   j                  |d          0i }|D ]O  }t        |t              r+|j                         D ]  \  }}|j                  ||        >|j                  d|       Q t        |   j                  |        	 t         j!                          y# t        $ r t        j                  d       Y 5w xY w# t         j!                          w xY w)z7Resolve pending scope toolsIds into kernel annotations.N       r   namez"resolve_pending_annotations failed)r\   r   r/   rM   nextiterr]   itemslenrU   rc   dict
setdefault	Exceptionr   	exceptionclear)per_tools_idr^   rs   tools_idannotationsmergedkeyvalues           r   resolve_pending_annotationsr   K  sY    4?4E%4 	:!J	% :X&--j9:	:
 %)$|*<%=%C"%1%7%7%9 	9!Hk;1$#H-44[^D%'F) :
j$/&0&6&6&8 6
U))#u56 %%fj9:  )008	9  	  ?=>? 	s*   DD  
E  E>E  EE Etorch_cuda_graphc                    t         syt        j                  | j                               }t	        t        j
                  |            }t        }i }t         j                         D ]K  \  }}|dz	  }|||k7  r|||<   |dz  }|dz  |z  }	|	|v r||	   j                  |       >t        |      ||	<   M t         j                          t         j                  |       y)a\  Remap annotation keys from capture graph ID to exec graph ID.

    During capture, toolsId encodes the capture graph's ID in the upper
    32 bits. After instantiation, the profiler uses the exec graph's ID.
    This function rewrites the keys so annotations match the trace.

    Must be called after the ``torch.cuda.graph()`` context exits.
    Nra   rw   l    )rU   r   cudaGraphExec_traw_cuda_graph_execr   cudaGraphExecGetIdr]   r|   extendr/   r   update)
r   exec_handleexec_graph_idcapture_graph_idremappedr   ann_listgraph_idnode_idnew_tools_ids
             r   remap_to_exec_graphr   m  s     //#779K )((	
M .%'H1779 4(r>'H8H,H!)HXZ'%+w68#\"))(3%)(^H\"4 x(r   c                      t         S )zBReturn the current kernel annotation map (toolsId -> annotations).)rU   r   r   r   get_kernel_annotationsr     s    r   c                  T    t         j                          t        j                          y)z9Clear all recorded kernel annotations and pending scopes.N)rU   r   r\   r   r   r   clear_kernel_annotationsr     s    r   <   _stream_id_counter_stream_id_mapc                 h    | j                   }|t        vrt        t        |<   t        dz  at        |   S )z;Return a small, stable stream ID for the given CUDA stream.rx   )rh   r   r   )r*   r   s     r   _get_stream_idr     s7     

C
. 0sa#r   pg_keyc                 P    | t         vrt        t         | <   t        dz  at         |    S )z:Return a unique stream ID for the given process group key.rx   )r   r   )r   s    r   get_stream_for_pgr     s,     ^#!3va&!!r   c              #   R  K   t         s-t        j                  j                  |       5  d ddd       y| j                  t        j                  j                         j                  k(  rt        |      5  d ddd       yt        |t              rd|i}t        |t              rt        |       |d<   t        |      5  t        j                  j                  |       5  d ddd       ddd       y# 1 sw Y   yxY w# 1 sw Y   yxY w# 1 sw Y   *xY w# 1 sw Y   yxY ww)u~  Switch to stream, inject its ID into annotation, and mark kernels.

    If *stream* is already the current stream, no stream switch or stream ID
    injection happens — the kernels stay on whatever stream is active (which
    keeps the trace faithful when e.g. FSDP uses the current stream for
    copy-in instead of a separate one). When switching to a different stream,
    this snapshots the current capturing branch before the target stream
    runs marked work. If the target stream is already capturing, the marked
    work must still be synchronized with the current stream so it is
    reachable from that snapped frontier.
    Nr`   r*   )r   re   rf   r*   rh   rg   ru   rc   r`   r~   r   )r*   r^   s     r   mark_streamr     s       ZZv& 		UZZ668DDD*% 		 	 j#&,Jj$'#1&#9Jx *% 	""6* 	 			 	 	 	se   &D'C7A	D'6D;AD' D!D&D.	D'7D <D'DD'D	DD$ D')r   N)<__doc__importlib.metadatar   collectionsr   
contextlibr   loggingr   typingr   r   re   torch.cuda._utilsr   r	   cuda.bindingsr
   r   ImportError__name__r   tupler/   r   __annotations__r~   rH   rK   r   r   boolr   r   r   r'   r)   r6   r<   rC   rT   rU   rV   r[   r\   r]   r`   ru   r   rf   	CUDAGraphr   r   r   r   r   Streamr   r   r   r   r   r   <module>r      s  (T  # %  !  F 
8	 !d3i0y 0'+CSM': 9 :
 $( TD[ ' # d " !! !H#$ #*s *}t'; *#3 #49 #"-s -tCy -( DH %	!c! !:D @! 	!
 
#s(^!J 4?t3D [d3i0 D '+ CHtO *C  02eCcN+, 1 &* d
 ) R8S4S>1 R8 R8j D()%***>*> ()4 ()VS$s)^ 4 
  C !#S#X #5::,,  "c "c " 

)) sT#s(^7K  K  Ms   F) )F43F4