^jHaddlmZddlZddlmZddlmZddlmZmZm Z m Z m Z m Z m Z ddlmZddlmZddlmZdd lmZmZmZmZmZerdd lmZmZej8eZd Zd Z d Z!dZ"dZ#GddeZ$GddeZ%GddeZ&GddeZ'GddeZ(GddeZ)GddeZ*GddeZ+d Z,Gd!d"eZ-d$d#Z.y)%) annotationsN) TYPE_CHECKING)tabulate) DebugHandler fetch_allformat_fetch_summary format_jsonNavLinkResponseRoute)tcpstore_client)build_db) JobConfig) CollectiveDatabaseGroup MembershipNCCLCall)FrontendServerHTTPRequestHandlerz {% extends "base.html" %} {% block header %}

{% block title %}Index{% endblock %}

{% endblock %} {% block content %} Hi {% endblock %} aQ {% extends "base.html" %} {% block header %}

{% block title %}py-spy Stack Traces{% endblock %}

{% endblock %} {% block content %}
{% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ resp.text }}
{% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}{{ title }}{% endblock %}

{% endblock %} {% block content %} {% if fetch_summary %}
{{ fetch_summary }}
{% endif %}

Groups

{{ groups | safe }}

Memberships

{{ memberships | safe }}

Collectives

{{ collectives | safe }}

NCCL Calls

{{ ncclcalls | safe }} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}torch.profiler{% endblock %}

{% endblock %} {% block content %}
{% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} {% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}TCPStore Keys{% endblock %}

{% endblock %} {% block content %} 
    {% for k, v in zip(keys, values) -%}
{{ k }}: {{ v | truncate(100) }}
    {% endfor %}
{% endblock %} c,eZdZddZddZddZd dZy) IndexHandlerc0td|jgS)N/r _handleselfs r/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/distributed/debug/_debug_handlers.pyrouteszIndexHandler.routessc4<<())ctddgS)NrHomer rs r nav_linkszIndexHandler.nav_linkssV$%%r!cdtiSNz index.html)INDEX_TEMPLATErs r templateszIndexHandler.templatess n--r!c8|jjdSr')frontendrender_template)rreqs rrzIndexHandler._handles||++L99r!Nreturnz list[Route]r/z list[NavLink]r/zdict[str, str]r-rr/bytes__name__ __module__ __qualname__r r%r)rr!rrrs*&.:r!rc4eZdZddZddZd dZd dZd dZy) StacksHandlerc0td|jgS)N/stacksrrs rr zStacksHandler.routessi.//r!ctddgS)Nr<zPython Stack Tracesr$rs rr%zStacksHandler.nav_linkss #89::r!cttd|j\}}|jjdd||S)Ndump_tracebacktimeoutz raw_resp.htmlStackstitleaddrsrespsr fetch_timeoutr+r,rr-rErFs rrzStacksHandler._handles> !14;M;MN u||++ 85,  r!ctd|j\}}g}t||}|r"|j||jdt t ||D]Y\}\}}|jd|d|d|j|j dk(r |jnd|j [d j|S) Nr?r@ === Rank :  ===Error:  rrHrappend enumeratezip status_codetextjoinrrErFpartssummaryiaddrresps rdumpzStacksHandler.dumps !14;M;MN u&ue4  LL ! LL (UE):; OA|d LL9QCr$t4 5 LL!--4 GDDTDTCU:V   yyr!cy)Nstacksr8rs r dump_filenamezStacksHandler.dump_filenamesr!Nr.r0r2r/z str | Noner/strr5r6r7r r%rr_rbr8r!rr:r:s0;  r!r:c<eZdZddZd dZd dZd dZd dZd dZy) PySpyHandlerc0td|jgS)N /pyspy_dumprrs rr zPySpyHandler.routessmT\\233r!ctddgS)Nrjz py-spy Stacksr$rs rr%zPySpyHandler.nav_linkss 788r!cdtiS)Npyspy_dump.html)PYSPY_DUMP_TEMPLATErs rr)zPySpyHandler.templatess!#677r!c|j}d|vr |rd|nd}td||j\}}|jj d||S)N nonblockingznonblocking=1& nonblocking=1 pyspy_dumpr@rmrErF) get_raw_queryrrHr+r,)rr-queryrErFs rrzPySpyHandler._handlese!!#  %05nUG,?E ud>P>PQ u||++ ,  r!ctdd|j\}}g}t||}|r"|j||jdt t ||D]Y\}\}}|jd|d|d|j|j dk(r |jnd |j [d j|S) Nrrrqr@rKrLrMrNrOrPrQrRrYs rr_zPySpyHandler.dumps /43E3E u&ue4  LL ! LL (UE):; OA|d LL9QCr$t4 5 LL!--4 GDDTDTCU:V   yyr!cy)Nrrr8rs rrbzPySpyHandler.dump_filenamesr!Nr.r0r1r2rcrd r5r6r7r r%r)rr_rbr8r!rrhrhs 498   r!rhc~eZdZd dZddZddZeddZ ddZddZ ddZ ddZ dd Z dd Z dd Zy )FlightRecorderHandlerctd|jtd|jtd|jtd|jgS)N /fr_trace/fr_trace_json/fr_trace_nccl/fr_trace_nccl_json)r _handle_fr_trace_handle_fr_trace_json_handle_fr_trace_nccl_handle_fr_trace_nccl_jsonrs rr zFlightRecorderHandler.routes sO +t44 5 "D$>$> ? "D$>$> ? ')H)H I   r!c^tddtddtddtddgS)Nr|zFlightRecorder CPUr}(JSON)r~FlightRecorder NCCLrr$rs rr%zFlightRecorderHandler.nav_linkss9 K!5 6 $h / $&; < )8 4   r!cdtiSN fr_trace.htmlFR_TRACE_TEMPLATErs rr)zFlightRecorderHandler.templates!233r!crt}|jg}d|_d|_i}t |D]?\}}|j dk7r|||d|j }d|vrg|d<||d|d<A|s tdtt|jd }t|||S) N)argsTrO)rank host_nameentriesrz.jsonzAll workers failed to respondversion) r parse_argsallow_incomplete_ranksverboserTrVjson RuntimeErrornextitervaluesr) rErFconfigrdetailsrr^r_rs r _build_dbzFlightRecorderHandler._build_dbs  b )&*# #E* /JD$3&"4[))+D $"$Y*.Gd4&& ' />? ?tGNN,-.y9w//r!c |j||}|jddt||t|jt j dt|jtj dt|jtj dt|jtj dS)NrFlightRecorderhtmlheaderstablefmtrD fetch_summarygroups memberships collectives ncclcalls) rr,rrrr_fieldsrrrrrrrserverrErFdbs r_render_tablesz$FlightRecorderHandler._render_tables8s^^E5 )%% ".ue<BIIu}}vN  (:(:V! (:(:Vr||X5E5EPVW&  r!ctd|j\}}|j|j|t |S)N fr_trace_jsonr@rrHrr+listrIs rrz&FlightRecorderHandler._handle_fr_traceJs5 $:L:LM u""3<<U DDr!cttd|j\}}|jjdd||S)Nrr@json_resp.htmlrrCrGrIs rrz+FlightRecorderHandler._handle_fr_trace_jsonNs@ $:L:LM u||++ " ,  r!ctdd|j\}}|j|j|t |S)Ndump_nccl_trace_jsononlyactive=truer@rrIs rrz+FlightRecorderHandler._handle_fr_trace_ncclWs= "$5t?Q?Q u""3<<U DDr!cvtdd|j\}}|jjdd||S)Nrrr@rrrCrGrIs rrz0FlightRecorderHandler._handle_fr_trace_nccl_json]sH "$5t?Q?Q u||++ ' ,  r!c&g}td|j\}}t||}|r"|j||jd|j ||}|j ddt |jtjddt |jtjdd t |jtjdd t |jtjdg  td d |j\}}|j ||}|j dd dt |jtjddt |jtjdd t |jtjdd t |jtjdg dj#|S#t $r|jdY-wxYw)Nrr@rKz=== FR Trace ===--- Groups ---plainr--- Memberships ------ Collectives ------ NCCL Calls ---rrz=== FR Trace NCCL ===z1 === FR Trace NCCL === Failed to fetch NCCL tracerQ)rrHrrSrextendrrrrrrrrrr ExceptionrX) rrZrErFr[r nccl_addrs nccl_respsnccl_dbs rr_zFlightRecorderHandler.dumphs $:L:LM u&ue4  LL ! LL  ^^E5 ) " EMMGL%1C1CgV%1C1CgV$x/?/?'R   P%.&(94CUCU& "J nnZ !6@R@RS u||++ O5,  r!ctd|j\}}g}t||}|r"|j||jdt t ||D]r\}\}}|jd|d|d|j dk(r%|jt|jU|jd|j td j|S) Nrr@rKrLrMrNrOrPrQ) rrHrrSrTrUrVr rWrXrYs rr_zWaitCountersHandler.dumps !6@R@RS u&ue4  LL ! LL (UE):; ;OA|d LL9QCr$t4 53& [34 wt'7'7&89:  ; yyr!cy)N wait_countersr8rs rrbz!WaitCountersHandler.dump_filenamesr!Nr.r0r2rcrdrfr8r!rrrs7<  r!rc<eZdZddZd dZd dZd dZd dZd dZy)TCPStoreHandlerc0td|jgS)N /tcpstorerrs rr zTCPStoreHandler.routessk4<<011r!ctddgS)NrTCPStorer$rs rr%zTCPStoreHandler.nav_linkss Z011r!cdtiS)N tcpstore.html)TCPSTORE_TEMPLATErs rr)zTCPStoreHandler.templatesrr!ctd}|j}|j|j|Dcgc] }t |}}|j j d||Scc}w)NrKprefixr)keysr)r list_keyssort multi_getreprr+r,)rr-storervrs rrzTCPStoreHandler._handlesbr*  #(??4#89a$q'99||++O$v+VV:sA2c&td}|j}|j|j|Dcgc] }t |}}t ||Dcgc] \}}|d|}}}dj |Scc}wcc}}w)NrKrrMrQ)r rrrrrUrX)rrrrrkrZs rr_zTCPStoreHandler.dumpsr*  #(??4#89a$q'99),T6):;AA3b;;yy:;s B#B cy)Ntcpstorer8rs rrbzTCPStoreHandler.dump_filenamerr!Nr.r0r1r2rcrdrxr8r!rrrs!224W r!rc`eZdZdZd dZd dZd dZddZddZ ddZ ddZ dd Z y )TorchCommsFlightRecorderHandlerz1Handler for TorchComms FlightRecorder trace data.cZtd|jtd|jgS)N/torchcomms_fr_trace/torchcomms_fr_trace_json)r _handle_torchcomms_fr_trace _handle_torchcomms_fr_trace_jsonrs rr z&TorchCommsFlightRecorderHandler.routess. ($*J*J K -t/T/T U  r!c2tddtddgS)Nrz TorchComms FRrrr$rs rr%z)TorchCommsFlightRecorderHandler.nav_linkss" *O < / :  r!cdtiSrrrs rr)z)TorchCommsFlightRecorderHandler.templatesrr!ctdd|j\}}|j|j|t |S)Ntorchcomms_fr_trace_jsonrr@rrIs rrz;TorchCommsFlightRecorderHandler._handle_torchcomms_fr_traces= &(94CUCU u""3<<U DDr!cvtdd|j\}}|jjdd||S)Nrrr@rTorchComms FlightRecorderrCrGrIs rrz@TorchCommsFlightRecorderHandler._handle_torchcomms_fr_trace_jsonsH &(94CUCU u||++ - ,  r!c tj||}|jddt||t |j t jdt |jtjdt |jtjdt |jtjdS)Nrrrrr)rzrr,rrrrrrrrrrrrs rrz.TorchCommsFlightRecorderHandler._render_tabless# , ,UE :%% -.ue<BIIu}}vN  (:(:V! (:(:Vr||X5E5EPVW&  r!c<td|j\}}g}t||}|r"|j||jdtj ||}|j ddt|jtjddt|jtjdd t|jtjdd t|jt jdg d j#|S) Nrr@rKz=== TorchComms FR Trace ===rrrrrrrQ)rrHrrSrzrrrrrrrrrrrrrX)rrErFrZr[rs rr_z$TorchCommsFlightRecorderHandler.dump+s !;TEWEWX u&ue4  LL ! LL  " , ,UE : - EMMGL%1C1CgV%1C1CgV$x/?/?'R  yyr!cy)Ntorchcomms_fr_tracer8rs rrbz-TorchCommsFlightRecorderHandler.dump_filenameBs$r!Nr.r0r1r2rrcrd) r5r6r7__doc__r r%r)rrrr_rbr8r!rrrsN;  4E   $ -6 ?M  $ .%r!raV {% extends "base.html" %} {% block header %}

{% block title %}TorchComms Health Check{% endblock %}

{% endblock %} {% block content %}

Health Status

{% if fetch_summary %}
{{ fetch_summary }}
{% endif %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ resp.text }}
{% endif %} {% endfor %} {% if dump_triggered %}

Flight Recorder Dump (triggered by unhealthy rank)

{% for i, (addr, resp) in enumerate(zip(dump_addrs, dump_resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to dump: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ resp.text }}
{% endif %} {% endfor %} {% endif %} {% endblock %} cReZdZdZd dZd dZd dZed dZddZ ddZ ddZ y )TorchCommsHealthCheckHandlerzBHealth check that detects watchdog timeouts and triggers FR dumps.c0td|jgS)N/torchcomms_health_checkrrs rr z#TorchCommsHealthCheckHandler.routesjs0$,,?@@r!ctddgS)NrzTorchComms Healthr$rs rr%z&TorchCommsHealthCheckHandler.nav_linksms24GHIIr!cdtiS)Ntorchcomms_health_check.html)HEALTH_CHECK_TEMPLATErs rr)z&TorchCommsHealthCheckHandler.templatesps.0EFFr!c|D]7}|jdk(s |j}|j ddr7yy#t$rtj dY[wxYw)aReturn True if any rank explicitly reports itself as unhealthy. Only responses with a 200 status code are inspected. Non-200 responses (connection errors, timeouts, etc.) are intentionally ignored so that a single unreachable rank does not trigger an expensive Flight Recorder dump across the entire job. A dump is only triggered when a rank is reachable *and* positively reports ``{"healthy": false}``. rOzGfailed to parse health check response as JSON; treating rank as healthyhealthyTF)rVrrlogger exceptionget)rFr^datas r_any_unhealthyz+TorchCommsHealthCheckHandler._any_unhealthyssr D3&99;Dxx 40 !$$3 s>AAc td|j\}}d}g}g}|j|rtd|j\}}d}|jj dt |||||||S)Ntorchcomms_health_checkr@Ftorchcomms_fr_dump_fileTr)rrErFdump_triggered dump_addrs dump_resps)rrHrr+r,r)rr-rErFr r!r"s rrz$TorchCommsHealthCheckHandler._handles !:DDVDVW u " %'   u %%.)43E3E& "J "N||++ *.ue<)!!,  r!ctd|j\}}g}t||}|r"|j||jdt t ||D]Y\}\}}|jd|d|d|j|j dk(r |jnd|j [|j|r|jd|jd td |j\}} t t || D]Y\}\}}|jd |d|d |j|j dk(r |jnd|j [d j|S)Nrr@rKrLrMrNrOrPz3=== Unhealthy rank detected, triggering FR dump ===rz --- Rank z ---rQ) rrHrrSrTrUrVrWrrX) rrErFrZr[r\r]r^r!r"s rr_z!TorchCommsHealthCheckHandler.dumps{ !:DDVDVW u&ue4  LL ! LL (UE):; OA|d LL9QCr$t4 5 LL!--4 GDDTDTCU:V     u % LL  LLN O%.)43E3E& "J $-SZ-H#I r1s" ;IQT   8 $4$?B ( :< :L>&<&R[L[|VlV$,@l:M%lM%`BU)<U)p r!