
    ^j                       U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZmZ d dlmZ d dlmZmZ d dlZej&                  j(                  Z ej,                  e      ZdZdZd ZdZd	Zd
ZdZdZ dZ!dZ"dZ# ejH                  d      Z%dZ&dZ' ejH                  d      Z(d	Z)dZ*dZ+ddddddddddd
Z,i Z-de.d <   da/da0d!e.d"<   da1d!e.d#<   da2d!e.d$<   da3d!e.d%<   da4d!e.d&<   da5d!e.d'<   da6d!e.d(<   da7d!e.d)<   da8d!e.d*<   da9d!e.d+<   d,a:d-e.d.<   d,a;d-e.d/<   da<da=da>da?da@daAdaBdaCdQd0ZDd1 ZEdRd2ZFdRd3ZGdSd4ZHdTd5ZIdQd6ZJdUd7ZKdVd8ZL G d9 d:eM      ZN	 	 	 	 	 	 	 	 dWd;ZOdXd<ZPdYd=ZQ G d> d?      ZRdaSd@e.dA<   dBaTdBaUdZdCZVdSdDZWd[dEZXdeeddF	 	 	 	 	 	 	 	 	 	 	 d\dGZYd]dHZZd]dIZ[	 d^	 	 	 d_dJZ\	 d^	 	 	 d_dKZ]dZdLZ^d`dMZ_dadNZ`dbdOZadZdPZby)c    )annotationsN)CallableIterable)Path)Anycast   !TORCH_CUPTI_MONITOR_LIBCUPTI_PATH      i  @ g      ?zcupti_monitor_meta.binzcupti_monitor_raw_buffers.bins   CUPMRBUF   z
<8sIIIIQQQs   CUPMMETAz<8sII   UnknownzDriver CompilerzBuffer FlushInstrumentationResourcez Runtime Triggered Module LoadingzLazy Function LoadingzCommand Buffer FullzActivity Buffer RequestzUVM Activity Init)
r   r   i   i   i   i   i   i   i   i   zdict[str, str]_DEMANGLE_CACHE
int | None_USER_EXTERNAL_CORRELATION_KIND_CUPTI_ACTIVITY_KIND_MEMCPY_CUPTI_ACTIVITY_KIND_MEMSET_CUPTI_ACTIVITY_KIND_DRIVER_CUPTI_ACTIVITY_KIND_RUNTIME&_CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL_CUPTI_ACTIVITY_KIND_OVERHEAD_CUPTI_ACTIVITY_KIND_CUDA_EVENT$_CUPTI_ACTIVITY_KIND_SYNCHRONIZATION)_CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION tuple[int, ...]_DISABLED_RUNTIME_CBIDS_DISABLED_DRIVER_CBIDSc                     t         j                  j                  t              } | r| S ddlm}  |d      j                  S )Nr   )load_nvidia_dynamic_libcupti)osenvironget_LIBCUPTI_PATH_ENVcuda.pathfinderr#   abs_path)overrider#   s     h/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/profiler/_cupti_monitor.py_find_cupti_libraryr-   Y   s5    zz~~01H #7+444    c            	     8   t         t         S 	 ddlm}  | a t	        t         j
                  j                        at	        t         j                  j                        a
t	        t         j                  j                        at	        t         j                  j                        at	        t         j                  j                        at	        t         j                  j"                        at	        t         j                  j&                        at	        t         j                  j*                        at	        t         j                  j.                        at	        t         j                  j2                        at7        d t         j8                  j:                  t         j8                  j<                  t         j8                  j>                  t         j8                  j@                  t         j8                  jB                  t         j8                  jD                  fD              a#t7        d t         jH                  jJ                  t         jH                  jL                  t         jH                  jN                  fD              a(t         jR                  a*t         jV                  a,t         jZ                  a.t         j^                  a0t         jb                  a2t         jf                  a4t         jj                  a6t         jn                  a8t         S # t        $ r}t        d      |d }~ww xY w)Nr   )r$   z|torch.profiler._cupti_monitor requires the cupti-python package. Install cupti-python to use the experimental CUPTI monitor.c              3  2   K   | ]  }t        |        y wNint.0cbids     r,   	<genexpr>z(_require_cupti_python.<locals>.<genexpr>   s      
$ 	D	
$   c              3  2   K   | ]  }t        |        y wr1   r2   r4   s     r,   r7   z(_require_cupti_python.<locals>.<genexpr>   s      # 	D	#r8   )9_ccr$   ModuleNotFoundErrorImportErrorr3   ExternalCorrelationKindCUSTOM1r   ActivityKindMEMCPYr   MEMSETr   DRIVERr   RUNTIMEr   CONCURRENT_KERNELr   OVERHEADr   
CUDA_EVENTr   SYNCHRONIZATIONr   EXTERNAL_CORRELATIONr   tupleRuntime_api_trace_cbidcudaGetDevice_v3020cudaSetDevice_v3020cudaGetLastError_v3020cudaEventCreate_v3020cudaEventCreateWithFlags_v3020cudaEventDestroy_v3020r    Driver_api_trace_cbidcuKernelGetAttributecuDevicePrimaryCtxGetStatecuCtxGetCurrentr!   activity_kernel11_dtype_KERNEL_DTYPEactivity_memcpy6_dtype_MEMCPY_DTYPEactivity_memset4_dtype_MEMSET_DTYPEactivity_api_dtype
_API_DTYPE#activity_external_correlation_dtype_EXTERNAL_CORRELATION_DTYPEactivity_overhead3_dtype_OVERHEAD_DTYPEactivity_cuda_event2_dtype_CUDA_EVENT_DTYPEactivity_synchronization2_dtype_SYNCHRONIZATION_DTYPE)imported_ccexcs     r,   _require_cupti_pythonrg   j   sV   . 
. C&)#*E*E*M*M&N#"%c&6&6&=&=">"%c&6&6&=&=">"%c&6&6&=&=">#&s'7'7'?'?#@ -01A1A1S1S-T*$'(8(8(A(A$B!&)#*:*:*E*E&F#+.s/?/?/O/O+P(03--1- $ 
$ &&::&&::&&==&&<<&&EE&&==

$ 
 # # %%::%%@@%%55
#  //M..M..M''J"%"I"I22O66 @@J]  J
 	s   K? ?	LLLc                     t                t        t        t        t                    t        t        t        t                    t        t        t        t
                    fS r1   )rg   r3   r   r   r   r   r   r.   r,   _default_always_on_activitiesri      sB    D<=>D123D123 r.   c                    t                t        t        t        t                    t        t        t        t                    t        t        t        t
                    t        t        t        t                    t        t        t        t                    t        t        t        t                    t        t        t        t                    t        t        t        t                    t        t        t        t                    f	S r1   )rg   r3   r   r   r   r   r   r   r   r   r   r   r   r.   r,    _default_trace_window_activitiesrk      s    D<=>D123D123D234D123D?@AD345D567D:;<
 
r.   c                     	 ddl m}  | j                         \  }}|| j                  j
                  k(  r|d uS || j                  j                  k(  ryt        d|       # t        $ r Y yw xY w)Nr   driverFzcuCtxGetCurrent failed with rc=)cuda.bindingsrn   r<   rT   CUresultCUDA_SUCCESSCUDA_ERROR_NOT_INITIALIZEDRuntimeError)cuda_driverrcctxs      r,   _has_active_cuda_contextrw      s|    	

 ))+GB	[!!...$	[!!<<<
8=
>>  s   A   	A,+A,c                     t        j                  t        j                         dz        j                  } t        j                         | t        j                         fS )Nl    )ctypesc_int32	threading	get_identvaluer%   getpidget_native_id)
opaque_tids    r,   _current_thread_resource_tupler      s@    	 3 3 5
 BCIIJIIKY%<%<%>??r.   c                 >    t         j                  j                  xs dS )N )torchversioncudar   r.   r,   _cuda_version_stringr      s    ==##r.   c                N    t        j                  | dd      j                         S )NT),:)	sort_keys
separators)jsondumpsencode)payloads    r,   _safe_json_dumpsr      s    ::g*ELLNNr.   c                    t        | dd      S )Nabr   )	buffering)open)paths    r,   _open_binary_appendr      s    da((r.   c                      e Zd Zy)_CuptiErrorN)__name__
__module____qualname__r   r.   r,   r   r      s    r.   r   c                p    ~~| dk(  ry 	 ddl m}  |       } |j                  |       S # t        $ r Y y w xY w)Nr   )get_kernel_annotations)torch.cuda._graph_annotationsr   	Exceptionr'   )graph_node_idrecord_kindcorrelation_idr   r   s        r,   "_default_graph_annotation_resolverr      sK     	^H,. ;??=))  s   ) 	55c                    | s|S t        j                  | t         j                        j                  }||S |j	                  d      S )Nreplace)errors)ry   r   c_char_pr}   decode)ptrdefaultr}   s      r,   _decode_c_stringr     s?    KKV__-33E}<<y<))r.   c                    t         j                  |       }|(t        j                  j	                  |       x}t         | <   |S r1   )r   r'   r   _C	_demangle)namecacheds     r,   _demangle_symbolr     s:      &F~).););D)AA&Mr.   c                     e Zd Zdeedd	 	 	 	 	 	 	 	 	 	 	 d&dZd'dZd(dZd)dZd*dZ	d'dZ
d'd	Zd'd
Zd'dZddd+dZd,dZd-dZd.dZd/dZ	 d0	 	 	 d1dZd'dZ	 d0	 	 	 d1dZd2dZd2dZd3dZd4dZd'dZd'dZd'dZd'dZd'dZd'dZd'd Z d5d!Z!	 	 	 	 	 	 	 	 	 	 d6d"Z"	 	 	 	 	 	 d7d#Z#d8d$Z$	 	 	 	 	 	 	 	 	 	 d6d%Z%y)9CuptiMonitorN
activitiesbuffer_sizeflush_period_sannotation_resolverc               v   t                t        |      | _        t        |xs
 t	                     | _        || _        || _        |xs t        | _	        t        j                  t                     | _        | j                          t        j                          | _        t        j$                  | j"                        | _        d| _        d| _        t-               | _        t        j0                         | _        t        j0                         | _        d | _        d | _        d | _        d| _        d| _        g | _         d| _!        d| _"        i | _#        d| _$        i | _%        d| _&        d | _'        d | _(        d| _)        d| _*        d| _+        d | _,        d| _-        d| _.        d| _/        d| _0        d| _1        d| _2        y )NFr   r   r   )3rg   r   
output_dirrI   ri   r   r   r   r   r   ry   CDLLr-   _lib_setup_prototypesr{   Lock_lock	Condition_processing_done_started_callbacks_registeredset_enabled_activitiesEvent_worker_stop_flush_stop_worker_thread_flush_thread_worker_error_trace_window_active_trace_window_prepared_trace_window_events_trace_window_extra_activities_trace_window_start_ns_trace_window_user_annotations_next_user_external_id_thread_resource_map_processing_inflight_time_converter_timestamp_callback_session_start_unix_ns_session_start_approx_ns!_session_start_calibrated_unix_ns_raw_buffers_fp_buffers_completed_final_allocated_buffers_valid_bytes_outstanding_warned_dropped_records_raw_chunk_count)selfr   r   r   r   r   s         r,   __init__zCuptiMonitor.__init__  s    	z*
 M.K.MN&,E#E 	  KK 3 56	 ^^%
 ) 3 3DJJ ?%*"-0U %OO-$??,7;6:37$)!&+#:<!?A+&'#>@+&'#?A!$%!##' &'#()%12.#"# )*%#(  ! !r.   c                Z   t        j                  t         j                        g| j                  j                  _        t         j                  | j                  j                  _        t         j                  t         j                  g| j                  j                  _        t         j                  | j                  j                  _        t         j                  g| j                  j                  _        t         j                  | j                  j                  _        t         j                  g| j                  j                  _        t         j                  | j                  j                  _        t        | j                  d      rbt         j                  t         j                  g| j                  j                  _        t         j                  | j                  j                  _        t        | j                  d      rbt         j                  t         j                  g| j                  j                  _        t         j                  | j                  j                  _        t         j                  g| j                  j                   _        t         j                  | j                  j                   _        t         j                  t         j"                  t        j                  t         j                        g| j                  j$                  _        t         j                  | j                  j$                  _        t         j                  t         j                  t        j                  t         j"                        g| j                  j&                  _        t         j                  | j                  j&                  _        t         j                  g| j                  j(                  _        t         j                  | j                  j(                  _        t         j                  g| j                  j*                  _        t         j                  | j                  j*                  _        t         j                  t         j,                  g| j                  j.                  _        t         j                  | j                  j.                  _        t         j                  t        j                  t         j,                        g| j                  j0                  _        t         j                  | j                  j0                  _        t         j                  t        j                  t         j2                        g| j                  j4                  _        t         j                  | j                  j4                  _        y )NcuptiActivityEnableRuntimeApicuptiActivityEnableDriverApi)ry   POINTERc_uint32r   cuptiGetVersionargtypesc_intrestypec_void_pcuptiActivityRegisterCallbackscuptiActivityEnablecuptiActivityDisablehasattrc_uint8r   r   cuptiActivityFlushAllc_size_tcuptiActivityGetNextRecord!cuptiActivityGetNumDroppedRecordscuptiActivityEnableHWTrace&cuptiActivityRegisterTimestampCallbackc_uint64&cuptiActivityPushExternalCorrelationId%cuptiActivityPopExternalCorrelationIdr   cuptiGetResultStringr   s    r,   r   zCuptiMonitor._setup_prototypesL  sO   .4nnV__.M-N		!!*,2LL		!!)OOOO=
		009 <B<<		00828,,		%%.06		%%-39<<.		&&/17		&&.499=>@DII33< ?EllDII33;499<=?DII22; >D\\DII22:4:OO3D		''028,,		''/OOOONN6??+9
		,,5
 8>||		,,4OOOONN6??+@
		33<
 ?Ell		33;9?8H		,,57=||		,,4EK__DU		88ACI<<		88@LLOOE
		88A DJ<<		88@LLNN6??+D
		77@ CI,,		77?LLNN6??+3
		&&/ 28		&&.r.   c                    t        j                         }| j                  | j                  j	                  t        j
                  |            d       t        |j                        S )Nr   )ry   r   _checkr   r   byrefr3   r}   )r   r   s     r,   _cupti_versionzCuptiMonitor._cupti_version  sI    //#II%%fll7&;<	
 7==!!r.   c                    t        j                         }| j                  j                  |t        j                  |            }|t
        k(  r&|j                  |j                  j                         S d| S )Nzrc=)ry   r   r   r   r   _CUPTI_SUCCESSr}   r   )r   ru   resultrc2s       r,   _result_stringzCuptiMonitor._result_string  s]    "ii,,Rf1EF. V\\%=<<&&((RDzr.   c                T    |t         k7  rt        | d| j                  |             y )Nz failed with )r  r   r  )r   ru   r   s      r,   r   zCuptiMonitor._check  s1    mD4G4G4K3LMNN  r.   c                    | j                   ry t        j                         }t        j                         }| j	                  | j
                  j                  t        j                  |      t        j                  |            d       d| _         y )Nr   T)	r   _PY_PROFILER._cupti_monitor_buffer_request_callback_address/_cupti_monitor_buffer_complete_callback_addressr   r   r   ry   r   )r   request_addrcomplete_addrs      r,   register_callbackszCuptiMonitor.register_callbacks  so    %%#RRT$TTVII44-. -	
 &*"r.   c                    t         j                         }| j                  | j                  j	                  t        j                  |            d       || _        y )Nr   )r	  (_cupti_approximate_time_callback_addressr   r   r   ry   r   r   )r   callback_addrs     r,   register_timestamp_callbackz(CuptiMonitor.register_timestamp_callback  sI    $MMOII<<. 5		
 $1 r.   c                6   | j                   rt        d      t        j                          t        j	                  | j
                         | j                          t        j                         | _        | j                          t        j                         | _        t        t        j                               | _        | j!                  | j                        | _        | j$                  j'                  dd       | j)                          | j+                          | j,                  j/                          | j0                  j/                          d | _        t5        j6                  | j8                  dd      | _        | j:                  j=                          | j>                  dkD  rAt5        j6                  | j@                  dd      | _!        | jB                  j=                          | jE                          | jG                  | jH                         d| _         y )Nz CUPTI monitor is already startedT)parentsexist_okztorch-cupti-monitor-worker)targetr   daemonr   ztorch-cupti-monitor-flush)%r   rs   r	  _cupti_monitor_reset_buffers _cupti_monitor_configure_buffersr   r  $_ApproximateClockToUnixTimeConverterr   r  timetime_nsr   r3   _get_approximate_timer   _convert_timer   r   mkdir_open_outputs_write_meta_filer   clearr   r   r{   Thread_worker_loopr   startr   _flush_loopr   _record_current_thread_infoenable_activitiesr   r   s    r,   r%  zCuptiMonitor.start  s   ==ABB11355d6F6FG!+PPR((*&*lln#(+L,N,N,P(Q%151C1C))2
. 	dT:! !'..$$-

 	!!#"!*!1!1''0"D
 $$&((*t/r.   c                <   | j                   sy | j                  | j                  j                  t              d       t        t        | j                              D ]-  }| j                  | j                  j                  |      d       / | j                  j                          d| _         | j                  j                          | j                  R| j                  j                  d       | j                  j                         rt        j!                  d       d | _        | j"                  j                          t$        j'                          | j(                  R| j(                  j                  d       | j(                  j                         rt        j!                  d       d | _        | j*                  t-        d      | j*                  | j/                          t$        j1                         | _        t$        j5                          d | _        d | _        y )	Nr   r   F      @timeoutz1CUPTI monitor flush thread did not stop within 5sz2CUPTI monitor worker thread did not stop within 5szCUPTI monitor worker failed)r   r   r   r   !_CUPTI_ACTIVITY_FLAG_FLUSH_FORCEDreversedrI   r   r   r"  r   r   r   joinis_aliveloggerwarningr   r	  _cupti_monitor_shutdown_buffersr   r   rs   _close_outputs _cupti_monitor_allocated_buffersr   r  r   r   )r   activitys     r,   stopzCuptiMonitor.stop  s   }}II++,MN#	
 !t'?'?!@A 	HKK		..x8&	
 	  &&()##C#0!!**,RS!%D446*$$S$1""++-ST"&D)<=4CUCUU(4(U(U(W%113##' r.   Fforcedc               n    |rt         nd}| j                  | j                  j                  |      d       y )Nr   r   )r-  r   r   r   )r   r9  flags      r,   flushzCuptiMonitor.flush  s*    4:0DII33D9;RSr.   c                    |dk(  ry| j                   |S t        | j                   j                  t        |                  S )Nr   )r   r3   
to_unix_ns)r   r}   s     r,   r  zCuptiMonitor._convert_time  s=    A:'L4''223u:>??r.   c                   g }|D ]y  }|| j                   v r| j                  | j                  j                  |      d       | j	                  |       | j                   j                  |       |j                  |       { t        |      S )Nr   )r   r   r   r   _apply_activity_filtersaddappendrI   )r   r   newly_enabledr6  s       r,   r(  zCuptiMonitor.enable_activities  s    " 	+H4333KK		55h?AVW((2$$((2  *	+ ]##r.   c                    |D ]W  }|| j                   vr| j                  | j                  j                  |      d       | j                   j	                  |       Y y )Nr   )r   r   r   r   remove)r   r   r6  s      r,   disable_activitieszCuptiMonitor.disable_activities  sY    " 	6Ht777KK		..x8& $$++H5	6r.   c                `   |t         k(  rMt        | j                  d      r7t        D ].  }| j	                  | j                  j                  |d      d       0 |t        k(  rOt        | j                  d      r8t        D ].  }| j	                  | j                  j                  |d      d       0 y y y )Nr   r   r   )	r   r   r   r    r   r   r   r!   r   )r   r6  r6   s      r,   r@  z$CuptiMonitor._apply_activity_filters  s    33II69
 0 II;;D!D3
 22wII58
 / II::4C28
2r.   c                j   | j                   st        d      | j                  rt        d      t        |xs
 t	                     }| j                          | j                  |      }| j                  5  g | _        d| _        d| _	        || _
        d| _        i | _        d d d        |S # 1 sw Y   |S xY w)Nz9CUPTI monitor must be started before prepare_trace_windowz"A trace window is already preparedTFr   )r   rs   r   rI   rk   r'  r(  r   r   r   r   r   r   r   r   rC  s      r,   prepare_trace_windowz!CuptiMonitor.prepare_trace_window+  s     }}K  &&CDD:K)I)KL
((*..z:ZZ 	5(*D%*.D'(-D%2?D/*+D'24D/	5 	5 s   3+B((B2c                   | j                   st        d      | j                  st        d      | j                  rt        d      | j	                          | j                  d       | j                  d       | j                  5  g | _        | j                  t        t        j                                     | _        d| _        d d d        y # 1 sw Y   y xY w)	Nz7CUPTI monitor must be started before start_trace_windowNo prepared trace windowz A trace window is already activeFr8  r*  	timeout_sT)r   rs   r   r   r'  r<  _wait_for_processing_idler   r   r  r3   r	  r  r   r   s    r,   start_trace_windowzCuptiMonitor.start_trace_window@  s    }}I  **9::$$ABB((*

%
 &&&5ZZ 	-(*D%*.*<*<L6689+D' )-D%	- 	- 	-s   A CCc                H    | j                  |      }| j                          |S r1   )rJ  rP  rI  s      r,   begin_trace_windowzCuptiMonitor.begin_trace_windowS  s&     11*=!r.   c           	        | j                   st        d      | j                          | j                  5  | j                  }| j
                  }t        | j                        }| j                  j                         D ci c]  \  }}|t        |       }}}d d d        | j                         | j                  d       | j                  d       | j                  5  t        | j                        }g | _        d| _         d| _        d| _        d| _        i | _        d d d        | j!                        t        |      |d	S c c}}w # 1 sw Y   xY w# 1 sw Y   <xY w)
NrL  Tr8  r*  rM  Fr   r   )eventsextra_activitiesuser_annotationsthread_resource_mapstart_ns)r   rs   r'  r   r   r   dictr   r   itemsrF  r<  rO  listr   r   _filter_trace_window_events)r   extrarX  rV  pidmappingrW  rT  s           r,   end_trace_windowzCuptiMonitor.end_trace_windowZ  sY   **9::((*ZZ 	77E22H#D$G$GH7;7P7P7V7V7X#'3sGT']"# #		 	&

$
&&&5ZZ 	5$334F(*D%*/D'(-D%24D/*+D'24D/	5 66vxH $U 0#6 
 	
#		 		5 	5s+   AE?E
E!A E
EEE%c                   | j                   5  | j                  t        | j                        | j                  | j                  rt
        j                         n| j                  t
        j                         | j                  dz  | j                  | j                  t        | j                        | j                  | j                  | j                   dcd d d        S # 1 sw Y   y xY w)Ni   )startedr   buffers_completedbuffers_allocatedbuffers_pendingvalid_total_mbdropped_recordsraw_chunks_writtenr   trace_window_preparedtrace_window_activetrace_window_start_ns)r   r   r[  r   r   r	  r5  r   _cupti_monitor_pending_buffersr   r   r   strr   r   r   r   r   s    r,   statszCuptiMonitor.statsx  s    ZZ 	=="4#;#;<%)%<%<== &2%R%R%T22#/#N#N#P"&"3"3{"C#'#8#8&*&;&;!$//2)-)D)D'+'@'@)-)D)D	 	 	s   CCC$c                   | j                          | j                  5  | j                  r| j                  rt        | j
                  vr
	 d d d        y | j                  }| xj                  dz  c_        || j                  |<   d d d        | j                  | j                  j                  t        t        j                              d       |S # 1 sw Y   NxY w)Nr   r   )r'  r   r   r   r   r   r   r   r   r   r   r   ry   r   )r   r   external_ids      r,   push_user_annotationz!CuptiMonitor.push_user_annotation  s    ((*ZZ 
	DMM22<//0 
	D 
	D 55K''1,'?CD//<
	D 	II<</1M 5		
 #
	D 
	Ds   ,C0CCc                   | j                          | j                  5  | j                  r| j                  rt        | j
                  vr
	 d d d        y 	 d d d        t        j                         }| j                  | j                  j                  t        t        j                  |            d       t        |j                        S # 1 sw Y   uxY w)Nr   )r'  r   r   r   r   r   ry   r   r   r   r   r   r   r3   r}   )r   last_ids     r,   pop_user_annotationz CuptiMonitor.pop_user_annotation  s    ((*ZZ 	MM22<//0 	 	0		 //#II;;/g1F 4		
 7==!!	 	s   ,CCc                F    t        | j                  t        z        | _        y r1   )r   r   _RAW_BUFFER_FILEr   r   s    r,   r   zCuptiMonitor._open_outputs  s    24??EU3UVr.   c                `    | j                   "| j                   j                          d | _         y y r1   )r   closer   s    r,   r4  zCuptiMonitor._close_outputs  s-    +  &&(#'D  ,r.   c                .   t         | j                         t               t        t              d| j
                  | j                  | j                  | j                  t        | j                  dz        dt        | j                        t               d}t        |      }t        | j                   t"        z  d      5 }|j%                  t&        j)                  t*        t         t-        |                   |j%                  |       d d d        y # 1 sw Y   y xY w)Napproximate_clockg    eAT)meta_versioncupti_versioncuda_versionhes_enabledtimestamp_modesession_start_unix_nssession_start_approx_ns session_start_calibrated_unix_nsr   flush_period_nsraw_buffer_dumpr   libcupti_pathwb)_META_VERSIONr  r   bool_hes_enabledr   r   r   r   r3   r   r[  r   r-   r   r   r   
_META_FILEwrite_META_HEADERpack_META_MAGIClen)r   metar   fps       r,   r!  zCuptiMonitor._write_meta_file  s    )!00202-1%)%@%@'+'D'D040V0V++"4#6#6#<=#t/02
 #4($//J.5 	HH\&&{M3w<PQHHW	 	 	s   9A	DDc                    t               \  }}}| j                  5  || j                  j                  |i       |<   d d d        y # 1 sw Y   y xY wr1   )r   r   r   
setdefault)r   r^  r   sys_tids       r,   r'  z(CuptiMonitor._record_current_thread_info  sN    #A#C ZZZ 	PHOD%%00b9*E	P 	P 	Ps    AAc                H   	 | j                   j                  | j                        sE| j                  r| j	                  d       | j                   j                  | j                        sDy y # t
        $ r+}|| _        | j                  j                          Y d }~y d }~ww xY w)NFr8  )	r   waitr   r   r<  BaseExceptionr   r   r   )r   rf   s     r,   r&  zCuptiMonitor._flush_loop  s    	$&&++D,?,?@==JJeJ, &&++D,?,?@  	$!$D!!##	$s   A(A- -	B!6!BB!c                   	 	 t         j                         }|y |\  }}}}}| j                  5  | xj                  dz  c_        | xj                  dz  c_        | xj
                  t        |      z  c_        d d d        	 | j                  t        |      t        |      t        |      t        |             t         j                  t        |             | j                  5  | xj                  dz  c_        | j                  j                          d d d        | j                          )# 1 sw Y   xY w# 1 sw Y   'xY w# t         j                  t        |             | j                  5  | xj                  dz  c_        | j                  j                          d d d        w # 1 sw Y   w xY wxY w# t        $ r+}|| _        | j                  j                          Y d }~y d }~ww xY w)Nr   )r	  _cupti_monitor_get_completedr   r   r   r   r3   _process_completed_buffer_cupti_monitor_return_bufferr   
notify_all_maybe_warn_backpressurer  r   r   r   )r   item
buffer_ptr
valid_sizerv   	stream_id_layout_epochrf   s           r,   r$  zCuptiMonitor._worker_loop  s   	$ $@@B< IME
JYZZ 9--2-++q0+%%Z8%9;22C#i.#j/3z? !==c*oN.. ;11Q61--88:; --/- 9 9; ; !==c*oN.. ;11Q61--88:; ; ;  	$!$D!!##	$s|   F7 F7 A	D,7F7  8E 8*F7 "0D8F7 ,D51F7 8E=F7 +F4/0F(	F4(F1-F44F7 7	G+ !G&&G+c                    | j                   ry t        j                         }|t        k\  rd| _         t        j                  d|       y y )NTzCUPTI monitor allocated %d activity buffers; the processing worker is not keeping up with CUPTI, so memory use will grow. Reduce traced activity or buffer size.)r   r	  r5  _OUTSTANDING_WARN_THRESHOLDr1  r2  )r   	allocateds     r,   r  z%CuptiMonitor._maybe_warn_backpressure  sH    ## AAC	33'+D$NN9 	 4r.   c                ^   t        j                          |z   }| j                  5  	 t        j                         dk(  r| j                  dk(  r
	 d d d        y |t        j                          z
  }|dk  r
	 d d d        y | j                  j                  t        d|             }# 1 sw Y   y xY w)Nr   g?r+  )r  r   r	  rl  r   r  min)r   rN  deadline	remainings       r,   rO  z&CuptiMonitor._wait_for_processing_idle  s    99;*"" 
	I ??AQF11Q6
	I 
	I %tyy{2	>
	I 
	I %%**3tY3G*H 
	I 
	Is   )B#B#<'B##B,c                   | j                   st        j                         }| j                  j	                  t        j
                  |      t        j                  |      t        j                  |            }|t        k(  r(| xj                  t        |j                        z  c_	        | j                  ||||       y t        j
                         }	 | j                  j                  t        j
                  |      t        j                  |      t        j                  |            }|t        k(  ri|j                  }|t        d      | j                  |      }	|	<| j                   5  | j"                  r| j$                  j'                  |	       d d d        |t(        k(  rnt+        d| j-                  |             t        j                         }| j                  j	                  t        j
                  |      t        j                  |      t        j                  |            }|t        k(  r)| xj                  t        |j                        z  c_	        y y # 1 sw Y   xY w)Nz+CUPTI returned null activity record pointerz'cuptiActivityGetNextRecord failed with )r   ry   r   r   r   r   r   r   r  r   r3   r}   _write_raw_bufferr   rs   _decode_recordr   r   r   rB  _CUPTI_ERROR_MAX_LIMIT_REACHEDr   r  )
r   rv   r  r  r  droppedru   
record_ptrrecord_addrtrace_events
             r,   r  z&CuptiMonitor._process_completed_buffer  s    **oo'G<<$fooi&@&,,wBWB ^#%%W]]);;%""3	:zJ__&
55
+
+Z(B
 ^#(..&&'TUU"11+>* J44 55<<[IJ 339$:M:Mb:Q9RS  //#YY88OOC &//)"<fll7>S
 !!S%77!  J Js   )(II'c                &   |dk(  r|S t               }g }g }g }|D ]  }|j                  d      }|dv rct        |j                  dd            }	|	|k\  rB|j                  |       t        |j                  dd            }
|
dk7  r|j	                  |
       {|dv r|j                  |       |dk(  r|j                  |       t        |j                  dd            }	|	dk(  s|	|k\  s|j                  |        |D ]3  }t        |j                  dd            }
|
|v s#|j                  |       5 |D ]Y  }t        |j                  dd            }
t        |j                  dd            }	|
|v s|
dk(  sC|	|k\  sI|j                  |       [ |S )Nr   kind>   rt   cuda_runtimerX  r   >   kernel
gpu_memcpy
gpu_memsetexternal_correlation)r   r'   r3   rB  rA  )r   rT  rX  retained_correlationsretained_eventspending_gpu_eventspending_external_eventseventr  event_start_nsr   s              r,   r\  z(CuptiMonitor._filter_trace_window_eventsA  s    q=M*-%02358: 	.E99V$D66!$UYYz1%=!>!X-#**51%(3CQ)G%HN%*-11.A=="))%0--'..u5 :q!9:N"n&@&&u-+	.. - 	.E +;Q!?@N!66&&u-	.
 ( 	.E +;Q!?@N :q!9:N!66!#((B&&u-	. r.   c                ~   t               }t        t        j                  j	                  |      j
                        }|t        k(  r7|j                  j                  |d      }| j                  t        |j                        t        t        |j                              }t        |j                        }| j                  t        |j                               }| j                  t        |j"                              }dt        |j$                        t        |j&                        t        |j(                        t        |j                        t        |j                        t        |j*                        ||||dS |t,        k(  r|j.                  j                  |d      }| j                  t        |j                        t0        t        |j                              }| j                  t        |j                               }| j                  t        |j"                              }i dddt        |j$                        dt        |j&                        d	t        |j(                        d
t        |j                        dt        |j2                        dt        |j                        dt        |j*                        d|d|dt        |j4                        dt        |j6                        dt        |j8                        dt        |j:                        dt        |j<                        d|ddS |t>        k(  rr|j@                  j                  |d      }| j                  t        |j                        tB        t        |j                              }| j                  t        |j                               }| j                  t        |j"                              }dt        |j$                        t        |j&                        t        |j(                        t        |j                        t        |j                        t        |j*                        ||t        |j4                        t        |j
                        t        |jD                        t        |j<                        |ddS |tF        tH        fv r|jJ                  j                  |d      }| j                  t        |j                               }| j                  t        |j"                              }|tF        k(  rdndt        |jL                        ||t        |jN                        t        |jP                        t        |j                        t        |jR                        dt        |jL                         d	S |tT        k(  r^|jV                  j                  |d      }dt        |jX                        t        |jZ                        t        |j                        dd S |t\        k(  r|j^                  j                  |d      }| j                  t        |j                               }| j                  t        |j"                              }t        |j`                        }	d!|	t        |jb                        d"||t        |j                        td        jg                  |	d#|	       d$S |th        k(  r|jj                  j                  |d      }| j                  t        |jl                              }
d%t        |j$                        t        |j&                        t        |j(                        t        |jn                        t        |j                        |
t        |jp                        d%d&	S |tr        k(  r|jt                  j                  |d      }| j                  t        |j                               }| j                  t        |j"                              }t        |jv                        }d'|||t        |j                        t        |j&                        t        |j(                        t        |jx                        t        |jp                        t        |jR                        d(| d)S y )*NT)readonlyr  )r  	device_id
context_idr  r   r   graph_idrX  end_ns
annotationr   r  r  r  r  r  r   runtime_correlation_idr   r  rX  r  bytes	copy_kindsrc_kinddst_kindflagsr  r   Memcpyr  Memset)r  r  r  r  r   r   r  rX  r  r  r}   memory_kindr  r  r   r  rt   cbid_)	r  r6   rX  r  
process_id	thread_idr   return_valuer   r  )r  external_kindrp  r   r   overheadr   	overhead_)r  overhead_kindobject_kind	object_idrX  r  r   r   
cuda_event)	r  r  r  r  event_idr   device_timestamp_nscuda_event_sync_idr   	cuda_syncsync_)r  	sync_typerX  r  r   r  r  r  r  r  r   )=rg   r3   ry   r   from_addressr}   r   ActivityKernel11from_ptrr   r   _RECORD_KIND_KERNELr   r   r   r  r%  endr  r  r  r  r   ActivityMemcpy6_RECORD_KIND_MEMCPYr  r  r  r  r  flags_r   ActivityMemset4_RECORD_KIND_MEMSETr  r   r   ActivityAPIr6   r  r  r  r   ActivityExternalCorrelationr  rp  r   ActivityOverhead3r  r  _OVERHEAD_KIND_NAMESr'   r   ActivityCudaEvent2device_timestampr  r  r   ActivitySynchronization2typecuda_event_id)r   r  ccr  recordr  kernel_namerX  r  r  event_tsr  s               r,   r  zCuptiMonitor._decode_recordr  s    #$6<<,,[9??@99((11+1MF11F(()#F))*J
 +6;;7K))#fll*;<H''FJJ8F  !1!12!&"3"34 !1!12"%f&;&;"<!$V%9%9!:0$ (#  ..''00t0LF11F(()#F))*J
 ))#fll*;<H''FJJ8FS!1!12 c&"3"34 S!1!12	
 !#f&;&;"< )#f.K.K*L  V%9%9!: C0 H & V\\* S!1!12 C0 C0 V]]+  j!" # ( ..''00t0LF11F(()#F))*J
 ))#fll*;<H''FJJ8F$ !1!12!&"3"34 !1!12"%f&;&;"<!$V%9%9!:0$ V\\*V\\*"6#5#56V]]+(  $ 02MNN^^,,[4,HF))#fll*;<H''FJJ8F 77 '"FKK($ !&"3"34 !1!12"%f&;&;"< #F$7$7 8FKK 012  <<33<<[SW<XF.!$V%9%9!:"6#5#56"%f&;&;"<.  00))22;2NF))#fll*;<H''FJJ8F 4 45M"!."6#5#56$ "%f&;&;"<,00!/  22**33K$3OF))#f.E.E*FGH$ !1!12!&"3"34 !1!120"%f&;&;"<'/&)&*C*C&D$
 
 770099+PT9UF))#fll*;<H''FJJ8FFKK(I#&$ "%f&;&;"<!&"3"34 !1!12 4 45&)&*C*C&D #F$7$7 8	{+  r.   c           
        | j                   t        d      t        t        j	                               }| j                  |      }t        j                  t        t        | j                  t        |      t        |      t        |      ||      }| j                   j                  |       | j                   j                  t        j                  ||             | xj                  dz  c_
        y )Nzraw buffer file is not openr   )r   rs   r3   r	  r  r  _RAW_CHUNK_HEADERr  _RAW_CHUNK_MAGIC_RAW_CHUNK_VERSIONr   r  ry   	string_at)r   rv   r  r  r  	approx_nsunix_nsheaders           r,   r  zCuptiMonitor._write_raw_buffer   s     '<==::<=	$$Y/"''!!	N
OH	
 	""6*""6#3#3J
#KL"r.   )r   str | os.PathLike[str]r   Iterable[int] | Noner   r3   r   floatr   ,Callable[[int, int, int], Any | None] | NonereturnNoner  r  )r  r3   )ru   r3   r  rm  )ru   r3   r   rm  r  r  )r9  r  r  r  )r}   r3   r  r3   )r   Iterable[int]r  r   )r   r  r  r  )r6  r3   r  r  r1   r   r   r  r   r  dict[str, Any]r   rm  r  r   r  r   )rN  r  r  r  )
rv   r3   r  r3   r  r3   r  r3   r  r  )rT  list[dict[str, Any]]rX  r3   r  r  )r  r3   r  dict[str, Any] | None)&r   r   r   _DEFAULT_BUFFER_SIZE_DEFAULT_FLUSH_PERIOD_Sr   r   r  r  r   r  r  r%  r7  r<  r  r(  rF  r@  rJ  rP  rR  r`  rn  rq  rt  r   r4  r!  r'  r&  r$  r  rO  r  r\  r  r  r   r.   r,   r   r     s   
 ,0/ 7LP7"*7" )	7"
 7" 7" J7" 
7"r8>t"O*1"H"(H ', T@	$6& 26.	*-( 26.	
<&*"&W(
*P
$$:I)8)8#&)847)8EH)8	)8V/*/69/	/bl\###&#47#EH#	#r.   r   CuptiMonitor | None_monitor_singletonFc                    t         ry t        j                  j                         s
t	               rt        d      ddlm}  | j                  d      d   }|| j                  j                  k7  rt        d|       t        j                  t                     }t        j                  g|j                  _        t        j"                  |j                  _        |j                  d      }|t&        k7  rt)        d|       da y )Nz>enable_hes_early() must be called before CUDA context creationr   rm   zcuInit failed with rc=r   z*cuptiActivityEnableHWTrace failed with rc=T)r  r   r   is_initializedrw   rs   ro   rn   cuInitrp   rq   ry   r   r-   r   r   r   r   r   r  r   )rt   ru   libs      r,   enable_hes_earlyr  ;  s    zz  "&>&@L
 	
 4			A	q	!B	[!!...3B4899 ++)+
,C/5~~.>C""+-3\\C""*		'	'	*B	^FrdKLLLr.   c                     t         S r1   )r  r   r.   r,   is_hes_enabledr  V  s    r.   c                     t         S r1   )r  r   r.   r,   get_monitorr  Z  s    r.   r   c                   t         t        d      t        | ||||      }|j                          |a t        st        j                  t               da|S )Nz*CUPTI monitor collection is already activer   T)r  rs   r   r%  _atexit_registeredatexitregister_stop_collection_atexit)r   r   r   r   r   monitors         r,   start_collectionr!  ^  s[     %GHH%/G MMO /0!Nr.   c                 h    t         } | y 	 | j                          | j                         d a S # d a w xY wr1   )r  r7  rn  )r   s    r,   stop_collectionr#  x  s3     G"}}!Ts   - 1c                 8    t         y t         j                         S r1   )r  rn  r   r.   r,   monitor_statsr%    s    !##%%r.   c                N    t         t        d      t         j                  |       S Nz&CUPTI monitor collection is not active)r  rs   rR  r   s    r,   rR  rR    s'     !CDD00<<r.   c                N    t         t        d      t         j                  |       S r'  )r  rs   rJ  r(  s    r,   rJ  rJ    s'     !CDD22:>>r.   c                 N    t         t        d      t         j                          y r'  )r  rs   rP  r   r.   r,   rP  rP    s     !CDD))+r.   c                 L    t         t        d      t         j                         S r'  )r  rs   r`  r   r.   r,   r`  r`    s#    !CDD..00r.   c                :    t         y t         j                  |       S r1   )r  rq  )r   s    r,   rq  rq    s    !22488r.   c                 8    t         y t         j                         S r1   )r  rt  r   r.   r,   rt  rt    s    !1133r.   c                 8    	 t                y # t        $ r Y y w xY wr1   )r#  r   r   r.   r,   r  r    s     s   
 	)r  rm  )r  r   )r  r  )r  ztuple[int, int, int])r   r	  r  r  )r   r   )r   r3   r   r3   r   r3   r  z
Any | None)r   r   r   rm  r  rm  )r   rm  r  rm  r  )r  r  )r   r  r   r   r   r3   r   r  r   r  r  r   )r  r  r1   r  r  r
  r  )c
__future__r   r  ry   r   loggingr%   structr{   r  collections.abcr   r   pathlibr   typingr   r   r   r   	_profilerr	  	getLoggerr   r1  r  r(   r  r  r-  r  r  r  rv  r  r  Structr  r  r  r  r  r  r  r  r   __annotations__r:   r   r   r   r   r   r   r   r   r   r   r    r!   rV   rX   rZ   r\   r^   r`   rb   rd   r-   rg   ri   rk   rw   r   r   r   r   rs   r   r   r   r   r   r  r  r  r  r  r  r!  r#  r%  rR  rJ  rP  r`  rq  rt  r  r   r.   r,   <module>r9     s   "     	    .    xx!!			8	$ " 8 !# $% !&  %
2   !FMM,/ v}}W%    /$"&   #% $
.2  2*. Z .*. Z .*. Z .+/ j /59 &
 9,0 z 0.2  237 $j 78< ): <+-  -*,  ,
"   5"JZ?@
$O)	, 	**%(*:=***a# a#H +/ ' . 6 (,+3HL& % 	
  F 4	"& (,=$== (,?$??,194r.   