Ë
    ÄÓ^jº  ã            	       ó  — d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
Z
d dlmZ d dlmc mc mZ d dlmZmZmZ  e j,                  e«      Z G d„ d«      Zd	e
j4                  j6                  d
eeeeee   f   f   fd„Z y)é    N)Údefaultdict)ÚIterator)Úcontextmanager)ÚEnum)Ú_apply_to_modulesÚ_get_module_fsdp_stateÚclean_tensor_namec                   ó¾   — e Zd ZU  G d„ dee«      Z ee«      Ze	eef   e
d<    e«       Zee   e
d<   edd„«       Zeededed   fd	„«       «       Zed
eddfd„«       Zy)ÚSimpleProfilerc                   ó$   — e Zd ZdZdZdZdZdZdZy)úSimpleProfiler.TypeÚallÚ
all_gatherÚall_gather_objectÚ
reshardingÚH2DÚD2HN)	Ú__name__Ú
__module__Ú__qualname__ÚALLÚ	ALLGATHERÚALLGATHER_OBJÚ
RESHARDINGr   r   © ó    ún/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/distributed/fsdp/_debug_utils.pyÚTyper      s    „ ØˆØ ˆ	Ø+ˆØ!ˆ
ØˆØ‰r   r   ÚresultsÚ	profilingÚreturnNc                 ól   — | j                   j                  «        | j                  j                  «        y )N)r   Úclearr    )Úclss    r   ÚresetzSimpleProfiler.reset"   s"   € à�‰×ÑÔØ�‰×ÑÕr   Úprofile_typec              #   óÚ  K  — || j                   v rt        |› d�«      ‚| j                   j                  |«       t        j                  «       }	 d –— t        j                  «       }| j
                  |xx   ||z
  z  cc<   | j                   j                  |«       y # t        j                  «       }| j
                  |xx   ||z
  z  cc<   | j                   j                  |«       w xY w­w)Nzk is already being profiled. SimpleProfiler does not support profiling multiple instances at the same time. )r    ÚAssertionErrorÚaddÚtimeÚ	monotonicr   Úremove)r$   r&   ÚbeginÚends       r   ÚprofilezSimpleProfiler.profile'   sÇ   è ø€ ð ˜3Ÿ=™=Ñ(Ü Ø�.ð !"ð "óð ð 	�‰×Ñ˜,Ô'Ü—‘Ó ˆð	/Ûä—.‘.Ó"ˆCØ�K‰K˜Ó%¨¨u©Ñ4Ó%Ø�M‰M× Ñ  Õ.øô —.‘.Ó"ˆCØ�K‰K˜Ó%¨¨u©Ñ4Ó%Ø�M‰M× Ñ  Õ.üs    ‚AC+ÁB ÁA
C+ÂAC(Ã(C+Úmsgc                 óò   — t        j                  «       dk(  rPt        j                  «       t         j                  j                  k(  r!t
        j                  d|| j                  «       | j                  «        y )Nr   z%s %s)	ÚdistÚget_rankÚget_debug_levelÚ
DebugLevelÚINFOÚloggerÚinfor   r%   )r$   r0   s     r   Údump_and_resetzSimpleProfiler.dump_and_reset:   sI   € ô �=‰=‹?˜aÒ¤D×$8Ñ$8Ó$:¼d¿o¹o×>RÑ>RÒ$RÜ�K‰K˜  c§k¡kÔ2Ø�	‰	�r   )r!   N)r   r   r   Ústrr   r   r   Úfloatr   ÚdictÚ__annotations__Úsetr    Úclassmethodr%   r   r   r/   r9   r   r   r   r   r      sœ   … ôˆs�Dô ñ !,¨EÓ 2€GˆT�#�u�*ÑÓ2Ù›%€Iˆs�3‰xÓàòó ðð Øð/ 3ð /¨8°D©>ò /ó ó ð/ð" ð ð ¨ò ó ñr   r   Úmodelr!   c                 ó†   — d„ }d„ }dg}i }t        | ||| j                  «       D ��cg c]  \  }}|‘Œ	 c}}||«      S c c}}w )a÷  
    It is used for composable fully_shard() code path, it returns
      1. sharded module tree info: each line represents a submodule name that contains the
    submodule's FQN and its submodule class name, if the submodule is sharded by `fully_shard`,
    the submodule name will add a postfix with ' FULLY SHARDED'. Each increased tree
    level adds 4 spaces before the printed name. A printed sharded module tree info for a toy model
    is like this:
        [CompositeModel] FULLY SHARDED
            l1[Linear]
            u1[UnitModule] FULLY SHARDED
                u1.l1[Linear]
                u1.seq[Sequential]
                    u1.seq.0[ReLU]
                    u1.seq.1[Linear]
                    u1.seq.2[ReLU]
                u1.l2[Linear]
            u2[UnitModule] FULLY SHARDED
                u2.l1[Linear]
                u2.seq[Sequential]
                    u2.seq.0[ReLU]
                    u2.seq.1[Linear]
                    u2.seq.2[ReLU]
                u2.l2[Linear]
            l2[Linear]
      2. a dict mapping from the concated module FQN and class name to a list of its managed
    original parameters' FQNs. An example of the dict for the above toy sharded model is like this:
            {'[CompositeModel]': ['l1.weight', 'l1.bias', 'l2.weight', 'l2.bias'],
             'u1[UnitModule]': ['u1.l1.weight', 'u1.l1.bias', 'u1.seq.1.weight', 'u1.seq.1.bias', 'u1.l2.weight', 'u1.l2.bias'],
             'u2[UnitModule]': ['u2.l1.weight', 'u2.l1.bias', 'u2.seq.1.weight', 'u2.seq.1.bias', 'u2.l2.weight', 'u2.l2.bias']
            }
    All FQNs are prefixed starting from ``model``.

    Args:
        model (torch.nn.Module): Root module (which may or may not be passed to
                                 composable `fully_shard()`).
    c                 óf  — |dz  }t        |«      dkD  r|d   dk(  r|d d n|}|dz   | j                  j                  z   dz   }d|z  |z   }t        | «      }	|	€|dxx   |dz   z  cc<   y |	j                  j                  | d «      }
|
r|dxx   |d	z   dz   z  cc<   n|dxx   |dz   z  cc<   |
r�|
j                  }t        |t        j                  «      st        d
t        |«      › �«      ‚|j                  D �cg c]  }t        ||z   «      ‘Œ }}||v r||   j                  |«       y |||<   y y c c}w )Né   r   éÿÿÿÿú.ú[ú]ú ú
z FULLY SHARDEDzExpected FlatParameter, got )ÚlenÚ	__class__r   r   Ú_fully_sharded_module_to_handleÚgetÚ
flat_paramÚ
isinstanceÚflat_param_fileÚFlatParameterr(   ÚtypeÚ_fqnsr	   Úextend)ÚmoduleÚprefixÚ
tree_levelÚsharded_tree_infoÚsharded_module_name_to_fqnsÚ
num_spacesÚtrimed_prefixÚprefixed_module_nameÚprinted_prefixed_module_nameÚstateÚhandleÚparamÚnameÚglobal_fqnss                 r   Ú	module_fnzD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fnk   sr  € ð   !‘^ˆ
ä ›K¨!šO°°r±
¸cÒ0AˆF�3�B‰KÈð 	ð  -¨sÑ2°V×5EÑ5E×5NÑ5NÑNÐQTÑTÐØ'*¨ZÑ'7Ð:NÑ'NÐ$ä& vÓ.ˆØˆ=Ø˜aÓ Ð$@À4Ñ$GÑGÓ Øà×6Ñ6×:Ñ:¸6À4ÓHˆáØ˜aÓ Ø,Ð/?Ñ?À$ÑFñÔ ð ˜aÓ Ð$@À4Ñ$GÑGÓ áØ×%Ñ%ˆEÜ˜e¤_×%BÑ%BÔCÜ$Ð'CÄDÈÃKÀ=Ð%QÓRÐRà=B¿[¹[öØ59Ô! &¨4¡-Õ0ðˆKð ð $Ð'BÑBØ+Ð,@ÑA×HÑHÈÕUàDOÐ+Ð,@ÒAð ùòs   Ã7D.c                 ó   — | d   |fS )Nr   r   )rX   rY   s     r   Ú	return_fnzD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.return_fn�   s   € Ø  Ñ#Ð%@Ð@Ð@r   Ú )r   Únamed_parameters)r@   rc   re   rX   rY   ÚkeyÚ_s          r   Ú1_get_sharded_module_tree_with_module_name_to_fqnsrj   C   sa   € òP#PòJAð
 	ð$Ðð 9;ÐÜØØØØ ×1Ñ1Ó3×4‘��aŠÓ4ØØ#óð ùó 	5s   §=)!Úloggingr*   Úcollectionsr   Úcollections.abcr   Ú
contextlibr   Úenumr   ÚtorchÚtorch.distributedÚdistributedr2   Ú"torch.distributed.fsdp._flat_paramÚfsdpÚ_flat_paramrP   Ú$torch.distributed.fsdp._common_utilsr   r   r	   Ú	getLoggerr   r7   r   ÚnnÚModuleÚtupler:   r<   Úlistrj   r   r   r   ú<module>r|      sƒ   ðã Û Ý #Ý $Ý %Ý ã Ý  ß <Ó <÷ñ ð 
ˆ×	Ñ	˜8Ó	$€÷*ñ *ðZ\Ø�8‰8�?‰?ð\à
ˆ3��S˜$˜s™)�^Ñ$Ð$Ñ%ô\r   