
    ^j,                        U d Z ddlmZ ddlZddlZddlZddlmZ ddlm	Z	 e	rddl
mZ ej                   G d d             Zdd	Zdd
ZddZddZh dZddZh dZddZddZeeeeeegZded<   ddZy)a  
Validates Chrome traces emitted by ``torch.profiler`` against rules derived
from production issues.

Usage::

    from torch.profiler._trace_validator import validate_trace

    passed, violations = validate_trace("trace.pt.trace.json")
    for v in violations:
        print(v)
    )annotationsN)defaultdict)TYPE_CHECKING)Callablec                  .    e Zd ZU dZded<   ded<   ddZy)	Violationz)A single rule violation found in a trace.str	rule_namemessagec                8    | j                    d| j                   S )Nz: r
   r   )selfs    j/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/profiler/_trace_validator.py__str__zViolation.__str__#   s    ..!DLL>22    N)returnr	   )__name__
__module____qualname____doc____annotations__r    r   r   r   r      s    3NL3r   r   c                N   | j                  d      rt        j                  nt        } || dd      5 }t        j                  |      }d d d        t        t              r|n|j                  dg       }|D cg c]  }t        |t              s| c}S # 1 sw Y   NxY wc c}w )Nz.gzrtzutf-8)encodingtraceEvents)	endswithgzipopenjsonload
isinstancelistgetdict)pathopenerfhdataeventses         r   _load_eventsr,   '   s    --.TYYDF	dW	- yy}d+T-1LF5!At!4A55  6s   B9B"B"Bc                   i }i }| D ]  }|j                  d      dk7  r|j                  di       }|j                  d      }|>t        |      }|j                  dd      |j                  dd      t        |j                  dd	            }}}|j                  d
      }	|dk(  r|dk(  r||vs|||   d   k  s|||	d||<   |dk(  s||vs|||   d   k  s|||	d||<    g }
|j                         D ]v  \  }}|j                  |      }||d   |d   k  s&|d   |d   z
  }|
j	                  t        dd|d    d| d|d    d|dd| d|d    d|d   dd|d   d             x |
S )zFor each (cudaLaunchKernel, GPU kernel) pair matched by External id,
    the GPU kernel must start at or after its cudaLaunchKernel.phXargsExternal idcat nametsr   correlationcuda_runtimecudaLaunchKernel)r5   r4   corrkernel_check_gpu_kernel_causalityzGPU kernel 'z' (External id=z, correlation=r9   z	) starts .1fz,us before its cudaLaunchKernel (External id=z
), gpu_ts=z	, cpu_ts=r   )r$   intfloatitemsappendr   )r*   cpu_launchesgpu_kernelsevr0   ext_idr2   r4   r5   r9   
violationsgpulaunchskews                 r   r;   r;   /   s    %'L#%K M66$<3vvfb!-(>Vub)266&"+=uRVVDRS_?U2Txx&. T-?%?\)R,v2Ft2L-L.0$'MV$H_[(BV1DT1J,J-/t&LF#!M$ J"((* !!&)>t9vd|#$<#d)+D;&s6{m?6( K''*6{m9T#J G==CH E''-f~&6 7""%d)C	&,s9K	M	$ r   c                   ddh}g }| D ]  }|j                  d      dk7  r|j                  di       }|j                  d      }||vr@|j                  d      }|t        |      dk  sbt        |j                  d	d            }|j                  t	        d
d| d|dd|j                  d       d|j                  d       d|
              |S )z^Stream Wait Events and Event Synchronize must have
    wait_on_cuda_event_record_corr_id >= 0.Stream Wait Eventz
Event Syncr.   r/   r0   cuda_sync_kind!wait_on_cuda_event_record_corr_idr   r5   $_check_stream_wait_corr_id_populated'z' event at ts=r<   zus on device=device stream=streamz/ has invalid wait_on_cuda_event_record_corr_id=r   )r$   r=   r>   r@   r   )r*   TARGET_KINDSrE   rC   r0   	sync_kindraw_corrr5   s           r   rM   rM   ]   s     (6LJ 66$<3vvfb!HH-.	L(88?@s8}q0rvvdA'BDI;nRH =""&((8"4!5Xdhhx>P=Q RIIQV	( r   c                p   g }| D ]  }|j                  d      dk(  s|j                  d      dk(  s-|j                  di       j                  d      dk(  sR|j                  di       }|j                  t        |j                  dd	            t        |j                  d
d	            |j                  d      |j                  d      d        |sg S t        t              }| D ]  }|j                  d      dk(  s|j                  d      dk(  s-|j                  di       }|j                  d      |j                  d      f}t        |j                  dd	            }||   j                  ||t        |j                  d
d	            z   |j                  dd      d        g }|D ]  }|d   |d   f}|j                  |g       D ]f  }	|	d   |d   cxk  r	|	d   k  sn |	d   |d   z
  }
|j                  t        dd|d    d|d    d|d   dd|	d    d|	d   dd|
dd             h  |S )z{For each Stream Synchronize on (device, stream), no kernel on that
    stream should still be running when the sync starts.r.   r/   r2   	cuda_syncr0   rK   zStream Syncr5   r   durrQ   rO   )r5   rW   rQ   rO   r:   r4   r3   )r5   endr4   rX   _check_stream_sync_overlapzStreamSynchronize on device=rP   z at ts=r<   zus but kernel 'z' (ends zus) is still running (zus overlap)r   )r$   r@   r>   r   r#   r   )r*   stream_syncsrC   r0   kernels_by_streamkeyr5   rE   synckoverlaps              r   rY   rY   z   sd    L FF4LCu,vr"&&'78MI66&"%DtQ0 q!12"hhx0"hhx0	 	1<T1B 66$<3266%=H#<66&"%D88H%txx'9:CrvvdA'Bc"))bffUA&6 77FF62. J H~tH~."&&sB/ 	Awd.ah.E(T$Z/!!">:4>:J K&&*8n%5WT$Z<L M++,V9+Xahs^ L118[J
	  r   >   cudaEventRecordcudaEventRecord_ptszcudaEventRecordWithFlagscudaEventRecordWithFlags_ptszc                \   i }| D ]  }|j                  d      dk(  s|j                  d      dv s,|j                  d      t        v sD|j                  di       }t        |j                  dd            }d	D ]3  }|j                  |      }|t        |      }||vs	|||   k  s/|||<   5  g }| D ]  }|j                  d      dk7  r|j                  di       }|j                  d
      dk7  r?|j                  d      }|t        |      dk  rat        |      }t        |j                  dd            }	|j                  |      }
|
%|j	                  t        dd|	dd| d             |
|	kD  s|
|	z
  }|j	                  t        dd|	dd| d|dd|
dd	              |S )zpwait_on_cuda_event_record_corr_id must point to a cudaEventRecord
    with cudaEventRecord.ts <= stream_wait.ts.r.   r/   r2   )r7   cuda_driverr4   r0   r5   r   )r1   r6   rK   rJ   rL   "_check_stream_wait_corr_id_in_pastzStream Wait Event at ts=r<   zus references corr_id=z) but no matching cudaEventRecord in tracer   z'us references cudaEventRecord (corr_id=z) z"us in the future (event_record_ts=))r$   _CUDA_EVENT_RECORD_NAMESr>   r=   r@   r   )r*   event_record_tsrC   r0   r5   fieldcidrE   refsw_ts	record_tslags               r   rf   rf      s    )+O 2FF4LCu!@@v"::66&"%DrvvdA'B7 2hhuo?c(C/1R/#:N5N/1,22 J "66$<3vvfb!88$%)<<hh:;;#c(Q,#hbffT1o&#'',	B25+ >##&%'PR e#CB25+ >44753s) D,,5c?!=	3"F r   >   
Group sizeIn msg nelemsOut msg nelemsCollective namedtypec                ^   g }| D ]  }|j                  d      dk7  s|j                  d      dk7  r,|j                  di       }t        t        |j                               z
  }|sa|j	                  t        ddt        |j                  dd	            d
dt        |                     |S )zlrecord_param_comms events must carry: Collective name, dtype,
    In msg nelems, Out msg nelems, Group size.r.   r/   r4   record_param_commsr0   _check_nccl_metadataz'record_param_comms' at ts=r5   r   r<   zus missing metadata: r   )r$   _NCCL_REQUIRED_FIELDSsetkeysr@   r   r>   sorted)r*   rE   rC   r0   missings        r   rw   rw      s     J 66$<3"&&.4H"Hvvfb!'#diik*::45eBFF4O6LS5Q R--3G_,=? r   c                J   t        t              }| D ]  }|j                  d      dk7  r|j                  dd      }d|vr/|j                  di       }|j                  d      xs |j                  d      }|ht        |      }d	|v r#|j	                  d	d
      d   j                         n|}|||   vs||   j                  |        g }|j                         D ]B  \  }}t        |      d
kD  s|j                  t        dd| dt        |       d|              D |S )z;Per Sequence number, at most one distinct backward op name.r.   r/   r4   r3   z$autograd::engine::evaluate_function:r0   zSequence numberseq_num:   !_check_backward_seq_id_uniquenesszSequence number z shared by z backward ops: r   )
r   r#   r$   r=   splitstripr@   r?   lenr   )	r*   
seq_to_opsrC   r4   r0   seqoprE   opss	            r   r   r     s;   '24'8J '66$<3vvfb!1=vvfb!hh()@TXXi-@;#h/2d{TZZQ#))+Z_$sO""2&' J$$& 
Ss8a<A*3%{3s8* E  #u&
 r   z-list[Callable[[list[dict]], list[Violation]]]_RULESc                    t        |       }g }t        D ]  }|j                   ||              t        |      dk(  |fS )a  
    Run all validation rules against a Chrome trace JSON file.

    Args:
        trace_path: Path to ``.pt.trace.json`` or ``.pt.trace.json.gz``.

    Returns:
        A ``(passed, violations)`` tuple.  ``passed`` is ``True`` when no
        violations were found.
    r   )r,   r   extendr   )
trace_pathr*   all_violationsrules       r   validate_tracer   >  sI     *%F&(N ,d6l+,~!#^33r   )r&   r	   r   
list[dict])r*   r   r   zlist[Violation])r   r	   r   ztuple[bool, list[Violation]])r   
__future__r   dataclassesr   r    collectionsr   typingr   collections.abcr   	dataclassr   r,   r;   rM   rY   rh   rf   rx   rw   r   r   r   r   r   r   r   <module>r      s    #    #   ( 3 3 36+\:5p 7t ,D  (&%95 4r   