Ë
    ÄÓ^j”D  ã                   ó  — d Z ddlZddlmZ ddlmZ ddlmZmZm	Z	m
Z
mZmZmZ ddlmZ  G d„ de«      Z	 	 	 	 d#d	eee	   z  d
edededz  dedededeeedz        fd„Z G d„ d«      Zej0                   edd«      ej2                   edd«      ej4                   edd«      ej6                   eddd«      ej8                   eddd«      iZd	eeedz        deeedz        fd „Z	 d$d	eeedz        d!edz  ddfd"„Zy)%z¡
This visualizer requires matplotlib to be installed.

Example usage:

ops = get_schedule_ops("InterleavedZeroBubble", 4, 8)
visualize_schedule(ops, "test.png")
é    N)Ú
NamedTuple)Úmock)Ú_ActionÚ_ComputationTypeÚ_PipelineScheduleÚ_PipelineScheduleRuntimeÚget_schedule_classÚPipelineScheduleMultiÚPipelineScheduleSingle)ÚPipelineStagec                   ó,   — e Zd ZU eed<   eed<   eed<   y)ÚOpKeyÚstage_indexÚcomputation_typeÚmicrobatch_indexN)Ú__name__Ú
__module__Ú__qualname__ÚintÚ__annotations__r   © ó    ú|/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/distributed/pipelining/_schedule_visualizer.pyr   r      s   … ØÓØ&Ó&ØÔr   r   ÚscheduleÚ	pp_degreeÚnum_microbatchesÚnum_stages_per_rankÚadd_spacingÚ
with_commsÚdefer_pp_recvÚreturnc                 ó  — |r|rt        d«      ‚t        | t        «      rt        | «      }n!t	        | t
        «      r| }nt        d| › �«      ‚t        j                  t        d¬«      }d|_	        ||_
        d|_        t	        |t        «      r$|€d}|dk(  st        d|› �«      ‚|}	||z  |	_        n_t	        |t        «      rA|€d	}|d	k\  st        d
|› �«      ‚t!        |«      D �
cg c]  }
|‘Œ }	}
|	D ]  }||z  |_        Œ nt        d|› �«      ‚ ||	|«      }|j"                  €t        d«      ‚g }|rat%        |	||¬«      }|j'                  |j"                  «       t!        |«      D ])  }|j)                  t+        |j,                  |   «      «       Œ+ n.t!        |«      D ]   }|j)                  |j"                  |   «       Œ" |r-|D ��cg c]  }|D �cg c]  }|€Œ|‘Œ	 c}‘Œ }}}t/        |«      }|S c c}
w c c}w c c}}w )ar  
    Get all actions for a given schedule, pp_degree, and num_microbatches. The actions are returned in a list of lists
    where each inner list represents a rank and each element in the inner list represents an action.

    The schedule can be specified as a string which is passed into get_schedule_class() or a _PipelineSchedule instance.

    Args:
        defer_pp_recv: If True, RECV ops are deferred to right before the consuming
            compute op. If False (default), RECV ops are placed as early as possible to
            overlap P2P communication with computation. Only takes effect when with_comms=True.
    z2Cannot add spacing and view comms at the same timezInvalid schedule: T)Úinstancer   Né   z*Expected num_stages_per_rank to be 1, got é   z'Expected num_stages_per_rank >= 2, got z&Expected pipeline_order to not be None)r    )Ú
ValueErrorÚ
isinstanceÚstrr	   Ú
issubclassr   r   Úcreate_autospecr   Ú
group_rankÚ
group_sizeÚsubmodr   ÚAssertionErrorÚ
num_stagesr
   ÚrangeÚpipeline_orderr   Ú_prepare_schedule_with_commsÚappendÚlistÚpipeline_order_with_commsÚadd_schedule_op_spacing)r   r   r   r   r   r   r    Úschedule_classÚmock_pipeline_stageÚstagesÚ_ÚstageÚschedule_instanceÚall_actionsÚruntimeÚrankÚactions                    r   Úget_schedule_opsrA   "   si  € ñ( ‘zÜÐMÓNÐNä�(œCÔ Ü+¨HÓ5‰Ü	�HÔ/Ô	0Ø!‰äÐ-¨h¨ZÐ8Ó9Ð9ô ×.Ñ.¬}ÀtÔLÐà%&ÐÔ"Ø%.ÐÔ"Ø!%ÐÔô �.Ô"8Ô9ØÐ&Ø"#ÐØ" aÒ'Ü Ø<Ð=PÐ<QÐRóð ð %ˆØ/°)Ñ;ˆÕÜ	�NÔ$9Ô	:ØÐ&Ø"#ÐØ" aÒ'Ü Ø9Ð:MÐ9NÐOóð ô 05Ð5HÓ/IÖJ¨!Ò%ÐJˆÐJØò 	?ˆEØ2°YÑ>ˆEÕñ	?ô Ð-¨nÐ-=Ð>Ó?Ð?ñ ' vÐ/?Ó@ÐØ×'Ñ'Ð/ÜÐEÓFÐFð /1€KÙÜ*ØÐ$°Mô
ˆð 	×,Ñ,Ð->×-MÑ-MÔNÜ˜)Ó$ò 	NˆDØ×Ñœt G×$EÑ$EÀdÑ$KÓLÕMñ	Nô ˜)Ó$ò 	GˆDØ×ÑÐ0×?Ñ?ÀÑEÕFð	Gñ ð KV÷
ØBF $Ö=˜¨&Ñ*<ŠVÔ=ð
ˆñ 
ô .¨kÓ:ˆð ÐùòI Kùò> >ùó
s$   Ã%	G4Ç
	G>ÇG9ÇG9ÇG>Ç9G>c                   ó&   — e Zd Z	 	 ddededefd„Zy)Ú_ComputationTypeVisualÚcolorÚtextÚwidthc                 ó.   — || _         || _        || _        y ©N)rD   rF   rE   )ÚselfrD   rE   rF   s       r   Ú__init__z_ComputationTypeVisual.__init__€   s   € ð ˆŒ
ØˆŒ
Øˆ�	r   N)Ú r$   )r   r   r   r(   r   rJ   r   r   r   rC   rC      s+   „ ð Øñ	àðð ðð ô	r   rC   ÚblueÚForwardÚtealzBackward InputÚgreenzBackward WeightÚorangezFull Backwardr%   ÚpurplezOverlap F+Bé   c                 óD  ‡‡‡‡‡‡— | s| S t        d„ | D «       «      dz   Št        | «      }t        |«      D �cg c]  }g ‘Œ c}Š| D �cg c]  }t        j                  |«      ‘Œ }}i Šdt
        dt        dt        fˆfd„Šdt        dt        t
           fˆfd„Šdt        dt        dt        fˆˆˆfd	„Šdt        d
t        dt        dt        fˆˆfd„}d}d}t        j                  t        |«      d«      }|�rt        d|› �«       t        |«      D ]~  \  }	}
|
sŒ	||	   }
|
d   }t        d|	› d|›�«       |€'‰|	   j                  d«       |
j                  «        d}ŒM|||	   k\  sŒV ‰||«      sŒ` |||	|«      ||	<   |
j                  «        d}Œ€ t        |«      D ]  }	|||	   k\  sŒ‰|	   j                  d«       Œ! |D �
cg c]  }
|
sŒ|
‘Œ	 }}
|dz  }|dz  }|t        d„ t         j#                  «       D «       «      kD  rt%        d«      ‚|r�Œ‰S c c}w c c}w c c}
w )a˜  
    Add spacing to the schedule based on dependencies between ranks.

    Before adding an operation to the list, this function checks if there are
    dependencies from other ranks. If there are dependencies (other ranks have
    not finished processing the required microbatch), it adds None instead.

    For example, Forward microbatch 0 on rank 1 depends on rank 0 processing
    Forward microbatch 0 first.

    Args:
        schedule: The original schedule as a list of lists where each inner list
                 represents a rank and each element represents an action.

    Returns:
        A new schedule with proper spacing based on dependencies.
    c              3   óF   K  — | ]  }|D ]  }|�|j                   –— Œ Œ y ­wrH   )r   )Ú.0Úrank_actionsr@   s      r   ú	<genexpr>z*add_schedule_op_spacing.<locals>.<genexpr>±   s9   è ø€ ò 
àØ&ò
ð ØÐ!ð ×Õð
Øñ
ùs   ‚!r$   Údependency_keyÚtimestepr!   c                 ó    •— | ‰v xr |‰|    k\  S )zDCheck if a dependency operation has completed by the given timestep.r   )rX   rY   Úscheduled_opss     €r   Úis_dependency_readyz4add_schedule_op_spacing.<locals>.is_dependency_readyÁ   s$   ø€ ð ˜mÐ+ò :Ø˜M¨.Ñ9Ñ9ð	
r   r@   c                 óÎ  •— | j                   }| j                  }| j                  }|€t        d| › d�«      ‚|dk(  r|t        j
                  k(  rg S |‰dz
  k(  rA|t        j                  t        j                  fv rt        |dz
  t        j
                  |«      gS |t        j
                  k(  rt        |dz
  t        j
                  |«      gS |t        j                  t        j                  fv r<t        |dz   t        j                  |«      t        |dz   t        j                  |«      gS |t        j                  k(  rt        |t        j                  |«      gS t        d|› �«      ‚)z+Get the list of dependencies for an action.úAction ú has None microbatch_indexr   r$   úUnknown computation type: )r   r   r   r.   r   ÚFORWARDÚFULL_BACKWARDÚBACKWARD_INPUTr   ÚBACKWARD_WEIGHTÚRuntimeError)r@   Ú	stage_idxÚ	comp_typeÚmb_idxr/   s       €r   Úget_dependenciesz1add_schedule_op_spacing.<locals>.get_dependenciesÈ   sm  ø€ à×&Ñ&ˆ	Ø×+Ñ+ˆ	Ø×(Ñ(ˆð ˆ>Ü  7¨6¨(Ð2LÐ!MÓNÐNð ˜Š>˜iÔ+;×+CÑ+CÒCØˆIð ˜
 Q™Ò&¨9Ü×*Ñ*Ü×+Ñ+ð9
ñ ,
ô ˜) a™-Ô)9×)AÑ)AÀ6ÓJÐKÐKð Ô(×0Ñ0Ò0Ü˜) a™-Ô)9×)AÑ)AÀ6ÓJÐKÐKð Ü×*Ñ*Ü×+Ñ+ð
ñ 
ô
 �i !‘mÔ%5×%CÑ%CÀVÓLÜ�i !‘mÔ%5×%DÑ%DÀfÓMðð ð Ô(×8Ñ8Ò8Ü˜)Ô%5×%DÑ%DÀfÓMÐNÐNäÐ7¸	°{ÐCÓDÐDr   c                 ó6  •‡— | j                   t        j                  t        j                  t        j                  fv r ‰| «      }t        ˆˆfd„|D «       «      S | j                   t        j                  k(  r ‰| «      }t        ˆˆfd„|D «       «      S | j                   t        j                  k(  rQ| j                  €t        d| › d�«      ‚g }| j                  D ]  }|j                   ‰|‰«      «       Œ t        |«      S t        d| j                   › �«      ‚)zBCheck if an action is ready to be scheduled at the given timestep.c              3   ó0   •K  — | ]  } ‰|‰«      –— Œ y ­wrH   r   ©rU   Údepr\   rY   s     €€r   rW   zCadd_schedule_op_spacing.<locals>.is_action_ready.<locals>.<genexpr>ú   ó   øè ø€ ÒR¸cÑ*¨3°×9ÑRùó   ƒc              3   ó0   •K  — | ]  } ‰|‰«      –— Œ y ­wrH   r   rl   s     €€r   rW   zCadd_schedule_op_spacing.<locals>.is_action_ready.<locals>.<genexpr>þ   rn   ro   úOVERLAP_F_B action ú has None sub_actionsr`   )r   r   rb   rc   rd   Úanyra   ÚallÚOVERLAP_F_BÚsub_actionsr.   r3   re   )r@   rY   ÚdependenciesÚdep_listÚ
sub_actionri   Úis_action_readyr\   s    `   €€€r   rz   z0add_schedule_op_spacing.<locals>.is_action_readyñ   s  ù€ ð ×"Ñ"Ü×*Ñ*Ü×+Ñ+Ü×,Ñ,ð'
ñ 
ñ
 ,¨FÓ3ˆLÜÔRÀ\ÔRÓRÐRà×$Ñ$Ô(8×(@Ñ(@Ò@Ù+¨FÓ3ˆLÜÔRÀ\ÔRÓRÐRØ×$Ñ$Ô(8×(DÑ(DÒDØ×!Ñ!Ð)Ü$Ø)¨&¨Ð1FÐGóð ð $&ˆHØ$×0Ñ0ò G�
Ø—‘¡°
¸HÓ EÕFðGä�x“=Ð äÐ!;¸F×<SÑ<SÐ;TÐUÓVÐVr   r?   c                 óX  •— ‰
|   j                  | «       | j                  }t        |   j                  }||z   }|t        j
                  k(  rž| j                  €t        d| › d�«      ‚d}| j                  D ]p  }|j                  €t        d|› d�«      ‚t        |j                     j                  }||z  }||z   ‰	t        |j                  |j                  |j                  «      <   Œr |S | j                  €t        d| › d�«      ‚|‰	t        | j                  || j                  «      <   |S )z.Schedule an action and return completion time.rq   rr   r   zSub-action r_   r^   )r3   r   Úaction_type_to_color_mappingrF   r   ru   rv   r.   r   r   r   )r@   r?   rY   rg   Ú	comp_timeÚcompletion_timeÚcumulative_timery   Úsub_comp_timer[   Úspaced_schedules            €€r   Úschedule_actionz0add_schedule_op_spacing.<locals>.schedule_action  sl  ø€ à˜Ñ×$Ñ$ VÔ,Ø×+Ñ+ˆ	Ü0°Ñ;×AÑAˆ	Ø" YÑ.ˆàÔ(×4Ñ4Ò4à×!Ñ!Ð)Ü$Ø)¨&¨Ð1FÐGóð ð  ˆOØ$×0Ñ0ò /�
Ø×.Ñ.Ð6Ü(Ø% j \Ð1KÐLóð ô !=Ø×/Ñ/ñ!ç‘%ð ð   =Ñ0�ð ˜Ñ.ð ÜØ"×.Ñ.Ø"×3Ñ3Ø"×3Ñ3óòð/ð. Ðð ×&Ñ&Ð.Ü$ w¨v¨hÐ6PÐ%QÓRÐRð  ð Ü�f×(Ñ(¨)°V×5LÑ5LÓMñð Ðr   r   zCurrent timestep: zRank: z	, action=Nc              3   ó4   K  — | ]  }|j                   –— Œ y ­wrH   )rF   )rU   Úvisuals     r   rW   z*add_schedule_op_spacing.<locals>.<genexpr>W  s   è ø€ ò ,
Ø#ˆF�L�Lñ,
ùs   ‚z2No progress made in scheduling - possible deadlock)ÚmaxÚlenr0   ÚcollectionsÚdequer   r   Úboolr   r4   ÚdictÚfromkeysÚprintÚ	enumerater3   Úpopleftr|   Úvaluesre   )r   Ú	num_ranksr:   ÚopsÚrank_opsr‚   Úcurrent_timestepÚtimesteps_without_progressÚrank_completion_timesr?   Úop_queuer@   ri   rz   r\   r/   r[   r�   s               @@@@@@r   r6   r6   ™   s–  ý€ ñ( Øˆô 	ñ 
à (ô
ó 	
ð ñ	ð ô �H“€IÜ?DÀYÓ?OÖ2P¸!²2Ò2P€OØ2:Ö;¨3”×!Ñ! #Õ&Ð;€HÐ;ð ')€Mð
¬Eð 
¼Sð 
ÄTõ 
ð'E¤ð 'E¬T´%©[õ 'EðRW¤ð W´3ð W¼4÷ Wð4%¤ð %¬sð %¼cð %Äcö %ðP ÐØ!"ÐÜ ŸM™M¬%°	Ó*:¸AÓ>ÐÚ
ÜÐ"Ð#3Ð"4Ð5Ô6ä'¨Ó1ò 	/‰NˆD�(ÙØà ‘~ˆHØ˜a‘[ˆFÜ�F˜4˜& 
 6 )Ð,Ô-Øˆ~Ø Ñ%×,Ñ,¨TÔ2Ø× Ñ Ô"Ø-.Ñ*Ø!Ð%:¸4Ñ%@Ó@Á_ØÐ(õFñ />Ø˜DÐ"2ó/Ð% dÑ+ð × Ñ Ô"Ø-.Ñ*ð%	/ô* ˜)Ó$ò 	3ˆDØÐ#8¸Ñ#>Ó>Ø Ñ%×,Ñ,¨TÕ2ð	3ð
 .6ÖB º’HÐBˆÐBØ˜AÑÐØ" aÑ'Ð"à%¬ñ ,
Ü'C×'JÑ'JÓ'Lô,
ó )
ò 
ô ÐSÓTÐTóI ðL ÐùòC 3QùÚ;ùòn Cs   ¸	HÁHÇHÇHÚfilenamec                 ó:  — ddl m} ddlm} d|j                  d<   t        | «      }t        d„ | D «       «      }|j                  |dz   |dz   f¬«      \  }}d	}t        ||«      d
z   }	t        «       }
t        | «      D ]ð  \  }}d}|D ]ä  }|�Ït        j                  |j                  t        d«      «      }|
j                  |j                  «       |j                   }|j"                  }|j$                  �d}d}nd}d} ||||z
  dz
  f|d|d|¬«      }|j'                  |«       |j)                  ||dz  z   ||z
  dz
  dz   t+        |«      dd|	d|¬«       ||z  }n|dz  }t        ||«      }Œæ Œò |j-                  d|dz   «       |j/                  d|dz   «       |j1                  t3        |«      D �cg c]
  }||z
  dz
  ‘Œ c}«       |j5                  t3        |«      D �cg c]  }d|› �‘Œ	 c}|	¬«       |j7                  g «       |j9                  d«       |
D �cg c]2  } |dddt        |   j                   dt        |   j(                  ¬«      ‘Œ4 }}|j;                  |d|	¬«       |r|j=                  |d¬«       y|j?                  «        yc c}w c c}w c c}w )aq  
    Visualize the schedule using matplotlib.
    The schedule is a list of lists where each inner list represents a rank and each element in the inner list represents an action.
    The actions are represented as rectangles with different colors based on their computation type.
    The filename is optional and if provided, the plot will be saved to that file.

    Args:
        schedule: The schedule to visualize.
        filename: The filename to save the plot to. If not provided, the plot will be displayed.
        add_schedule_spacing: If True, add spacing to the schedule based on dependencies between ranks.

    r   N)Ú	RectanglezDejaVu Sanszfont.familyc              3   ó2   K  — | ]  }t        |«      –— Œ y ­wrH   )r†   )rU   r?   s     r   rW   z%visualize_schedule.<locals>.<genexpr>w  s   è ø€ Ò5 D”c˜$—iÑ5ùs   ‚r%   )Úfigsizeéÿÿÿÿé   ÚblackÚnormalr$   )Ú	facecolorÚ	edgecolorÚ	linewidthg      à?ÚcenterÚwhite)ÚhaÚvaÚfontsizerD   Úweightg      à¿zRank )r§   F)r   r   )r    r¡   Úlabelzupper right)ÚhandlesÚlocr§   Útight)Úbbox_inches) Úmatplotlib.pyplotÚpyplotÚmatplotlib.patchesr™   ÚrcParamsr†   r…   ÚsubplotsÚminÚsetr�   r|   Úgetr   rC   ÚaddrD   rF   rv   Ú	add_patchrE   r(   Úset_xlimÚset_ylimÚ
set_yticksr0   Úset_yticklabelsÚset_xticklabelsÚgridÚlegendÚsavefigÚshow)r   r—   Úpltr™   r�   Úmax_actionsÚfigÚaxÚmax_draw_positionÚ	font_sizeÚused_computationÚrank_idxÚactionsÚdraw_positionr@   Úcomp_type_colorrD   rF   r¢   Útext_weightÚrectÚirg   Úlegend_elementss                           r   Úvisualize_schedulerÐ   _  sæ  € õ" $Ý,ð 	ð ‡L�L�Ñô �H“€IÜÑ5¨HÔ5Ó5€Kð �l‰l K°!¡O°YÀ±]Ð#CˆlÓD�G€CˆØÐä�K Ó+¨aÑ/€IÜ“uÐÜ& xÓ0ò -FÑˆ�'ØˆØò +	FˆFØÐ!Ü">×"BÑ"BØ×+Ñ+Ô-CÀGÓ-Ló#�ð !×$Ñ$ V×%<Ñ%<Ô=Ø'×-Ñ-�Ø'×-Ñ-�ð ×%Ñ%Ð1Ø !�IØ"*‘Kà !�IØ"*�Kñ !Ø" I°Ñ$8¸1Ñ$<Ð=ØØØ#Ø%Ø'ô�ð —‘˜TÔ"ð —‘Ø! E¨A¡IÑ-Ø Ñ(¨1Ñ,¨sÑ2Ü˜“KØØØ&Ø!Ø&ð ô 	ð  Ñ&‘à Ñ"�Ü #Ð$5°}Ó EÑñW+	Fð-Fð\ ‡K�K�Ð'¨!Ñ+Ô,Ø‡K�K��i #‘oÔ&à‡M�M¼eÀIÓ>NÖO°(�9˜xÑ'¨#Ó-ÒOÔPØ×Ñ¬U°9Ó-=Ö>¨˜% ˜sšÒ>ÈÐÔSØ×Ñ�rÔð ‡G�GˆE„Nð *ö
ð ñ 	ØØØÜ2°9Ñ=×CÑCØÜ.¨yÑ9×>Ñ>ö	
ð
€Oð 
ð ‡I�I�o¨=À9€IÔMáØ�‰�H¨'ˆÕ2à�‰�
ùò/ PùÚ>ùò
s   Æ1JÇJÈ7J)NFFFrH   ) Ú__doc__r‡   Útypingr   Úunittestr   Ú&torch.distributed.pipelining.schedulesr   r   r   r   r	   r
   r   Ú"torch.distributed.pipelining.stager   r   r(   Útyper   r‰   r4   rA   rC   ra   rc   rd   rb   ru   r|   r6   rÐ   r   r   r   ú<module>r×      s®  ðñó Ý Ý ÷÷ ñ õ =ôˆJô ð '+ØØØñZØ�DÐ*Ñ+Ñ+ðZàðZð ðZð ˜t™ð	Zð
 ðZð ðZð ðZð 
ˆ$ˆw˜‰~Ñ
ÑóZ÷z	ñ 	ð ×ÑÑ4°V¸YÓGØ×#Ñ#Ñ%;¸FÐDTÓ%UØ×$Ñ$Ñ&<ØÐ"ó'ð ×"Ñ"Ñ$:Ø�/ 1ó%ð × Ñ Ñ"8¸À=ÐRSÓ"Tð
 Ð ðCØ�4˜ $™Ñ'Ñ(ðCà	ˆ$ˆw˜‰~Ñ
ÑóCðP  ñhØ�4˜ $™Ñ'Ñ(ðhà�D‰jðhð 
ôhr   