Ë
    ÌÓ^j§E ã                   ó`	  — U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d d	l m!Z!m"Z" d d
l#m$Z$ d dl%Z%d dl&Z%d dl'Z%d dl(m)Z* d dl+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6 d dl7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZD d dlEmFZFmGZGmHZH dZIdZJdZKdZLdZMdZNe4r+d dlOZOdD ]"  \  ZPZQ eOj¤                  eP«      sŒd eS«       eQ<   Œ$ deTdeUdeUde%j¬                  deTdeTde*j®                  fd„ZX ej²                  eZ«      Z[e[j¹                  ejº                  «       g d¢Z^ddgZ_e?xs e@xs eCZ` G d „ d!e"«      Zai d" ead#d$«      “d% ead&d'«      “d( ead)d*«      “d+ ead,d-«      “d. ead/d0«      “d1 ead2d3«      “d4 ead5d6«      “d7 ead8d9«      “d: ead;d<«      “d= ead>d?«      “d@ eadAdB«      “dC eadDdE«      “dF eadGdH«      “dI eadJdK«      “dL eadMdN«      “dO eadPdQ«      “dR eadSdT«      “dU eadVdW«      i¥Zbe G dX„ dY«      «       ZcdZ„ Zdd[„ Zed\„ Zfd]„ Zgd^„ Zhd_„ Zid`„ Zjdekfda„Zlddbœdc„ZmddeUfde„Zndfe!dgeUdeUfdh„Zodi„ Zpdj„ Zqdk„ Zrdl„ Zsdm„ Ztdn„ Zudo„ Zvdp„ Zwdq„ Zxdr„ Zyd¸ds„Zzdt„ Z{du„ Z| e;dv„ «      Z}eke~dw<   dx„ Zdye€eTdzf   fd{„Z�d¸d|„Z‚d}„ Zƒde%j¬                  d~eUdeUdekfd€„Z„e<d�d‚d edƒ¬„«      dddfd…„«       Z…eBrd†Z†n eU e�j                  d‡dˆ«      «      Z†d‰d†dŠœZˆeArd‹eˆdŒ<   d¹d�ekfdŽ„Z‰deUfd�„ZŠed�„ «       Z‹dºdeUdeUd‘eUfd’„ZŒdeUdeTfd“„Z�daŽe�j                  dz  e~d”<   d¸d•eTdz  ddfd–„Z�d»d—„Z‘	 d¸d˜e%j
                  �j$                  d™e%j
                  �j&                  dšeUdz  de!fd›„Z”eAr+ e•dœ e–d�e%�j.                  �j1                  «       «      «      Z™ndœZ™ G dž„ dŸeD«      Zš G d „ d¡eš«      Z›d¢eœeTe�e!   f   d£ed¤e!fd¥„Zže�j>                  dekfd¦„«       Z d§„ Z¡de†e™fd¨„Z¢ G d©„ dªeD«      Z£ G d«„ d¬e,�jH                  «      Z¥ G d­„ d®e,�jH                  «      Z¦e	 d¼d¯„«       Z§ G d°„ d±e%�jP                  �jR                  jˆ                  «      Zª G d²„ d³e›«      Z« G d´„ dµeD«      Z¬ G d¶„ d·e¬«      Z­y)½é    N)ÚCallable)Úcontextmanager)Ú	dataclass)Ú	timedelta)ÚEnum)ÚpartialÚreduceÚwraps)ÚStringIO)ÚAnyÚ
NamedTuple)Úpatch)Ú
DeviceType)Ú_SymmetricMemory)Ú	trace_log)Ú_TORCHCOMM_AVAILABLE)Úcommon_utils)ÚFILE_SCHEMAÚfind_free_portÚIS_SANDCASTLEÚLazyValÚretry_on_connect_failuresÚskip_but_pass_in_sandcastleÚskip_but_pass_in_sandcastle_ifÚ	TEST_CUDAÚTEST_HPUÚTEST_WITH_ROCMÚTEST_WITH_TSANÚTEST_XPUÚTestCase)Ú_install_threaded_pgÚ_uninstall_threaded_pgÚProcessLocalGroupF))ÚglooÚTORCHCOMM_HAS_GLOO)ÚxcclÚTORCHCOMM_HAS_XCCL)ÚncclÚTORCHCOMM_HAS_NCCL)ÚrcclxÚTORCHCOMM_HAS_RCCLX)ÚncclxÚTORCHCOMM_HAS_NCCLXTÚbackendÚrankÚ
world_sizeÚdeviceÚ
store_pathÚ
group_nameÚreturnc                 ó"  — ddl m}m} t        j                  ||«      }t        |«      t        j                  d<   t        |«      t        j                  d<   t        j                  | ||› d�t        j                  |› d�|«      ddi¬	«      }	t        j                  t        j                  |› d
�|«      |	j                  «       |	j                  «       «      }
|
j                  |	j                  «       t        j                  j                   j"                   ||	«      «       |
j%                  t        j                  j                   j"                  «       |
j'                  |«        ||
| ||| t)        |«      D �ci c]  }||“Œ c}¬«       |
S c c}w )ur  Build a bare ProcessGroup whose backend is a torchcomms _BackendWrapper.

    Creates a torchcomms comm of the requested ``backend`` (e.g. ``nccl``,
    ``ncclx``) and wraps it in ``_BackendWrapper``, then registers it as the
    NCCL backend on a fresh ``ProcessGroup`` and publishes the group into
    ``c10d._world`` so name-based lookups (e.g. ``symm_mem.rendezvous(...,
    group=group_name)``) resolve to it.

    Callers are responsible for choosing a ``group_name`` (and matching
    ``store_path``) that is unique within the process â€” duplicates trip
    ``_register_pg_in_world`` and the underlying FileStore.
    r   )Ú_BackendWrapperÚ_register_pg_in_worldÚTORCHCOMM_RANKÚTORCHCOMM_SIZEÚ_commz_tc/Úpersistent_storeÚtrue)ÚnameÚstoreÚhintsz_pg/)Úbackend_namer>   r3   Úbackend_configÚrank_mapping)Ú"torch.distributed.distributed_c10dr6   r7   Úc10dÚ	FileStoreÚstrÚosÚenvironÚ
torchcommsÚnew_commÚPrefixStoreÚProcessGroupÚget_rankÚget_sizeÚ_register_backendÚ
get_deviceÚBackendTypeÚNCCLÚ_set_default_backendÚ_set_group_nameÚrange)r.   r/   r0   r1   r2   r3   r6   r7   Ú
file_storeÚtc_commÚpgÚrs               úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.pyÚsetup_torchcomms_pgr[   R   sb  € ÷(ô
 —‘ 
¨JÓ7€Jä#& t£9„B‡J�JÐÑ Ü#& z£?„B‡J�JÐÑ ä×!Ñ!ØØØˆ|˜5Ð!Ü×Ñ * ¨TÐ2°JÓ?Ø! 6Ð*ô€Gô 
×	Ñ	Ü×Ñ˜J˜< tÐ,¨jÓ9Ø×ÑÓØ×ÑÓó
€Bð
 ×ÑØ×ÑÓÜ×Ñ×%Ñ%×*Ñ*Ù˜Ó ôð ×ÑœD×-Ñ-×9Ñ9×>Ñ>Ô?à×Ñ�zÔ"ÙØ
ØØØØÜ$)¨*Ó$5Ö6˜q�a˜‘dÒ6õð €Iùò 7s   Å8
F©r(   r&   ÚhcclÚcudaÚxpuc                   ó"   — e Zd ZU eed<   eed<   y)ÚTestSkipÚ	exit_codeÚmessageN)Ú__name__Ú
__module__Ú__qualname__ÚintÚ__annotations__rF   © ó    rZ   ra   ra   œ   s   … ØƒNØ„Lrj   ra   Úbackend_unavailableéH   z5Skipped because distributed backend is not available.Úsmall_worldsizeéI   z Skipped due to small world size.Úodd_worldsizeéW   zSkipped due to odd world size.Úno_cudaéJ   zCUDA is not available.zmulti-gpu-1éK   zNeed at least 1 CUDA devicezmulti-gpu-2éM   zNeed at least 2 CUDA deviceszmulti-gpu-3éP   zNeed at least 3 CUDA deviceszmulti-gpu-4éQ   zNeed at least 4 CUDA deviceszmulti-gpu-5éR   zNeed at least 5 CUDA deviceszmulti-gpu-6éS   zNeed at least 6 CUDA deviceszmulti-gpu-7éT   zNeed at least 7 CUDA deviceszmulti-gpu-8éU   zNeed at least 8 CUDA devicesr(   éL   z#c10d not compiled with NCCL supportÚ
skipIfRocméN   zTest skipped for ROCmÚno_peer_accesséO   z'Test skipped because no GPU peer accessÚgenericéV   zHTest skipped at subprocess level, look at subprocess log for skip reasonÚimporterroréX   z"Test skipped due to missing importÚno_acceleratoréY   zaccelerator is not available.c                   óÀ   — e Zd Zi Zh d£ed<    e«       ed<   h d£ed<   h d£ed<   i Zh d£ed<   h d£ed	<   h d£ed
<   h d£ed<    e«       ed<   erdhed<   erdhed<   yy)ÚDistTestCases>   ÚmpiÚuccr(   r&   Úallgather_coalescedr	   >   r‰   r(   r&   zsendrecv anysourcezcpu barrier>   r‰   r$   r(   Úgpur^   ÚddpÚsubgroupÚpluginr]   Úhpur&   r_   N)rd   re   rf   Úskip_collectiveÚsetÚbackend_featurer   r   ri   rj   rZ   r‡   r‡   »   s“   „ ð €OÚ-K€OÐ)Ñ*Ù #£€O�HÑÚ,C€OÐ(Ñ)Ú%<€O�MÑ"ð €OÚ4€O�EÑÚ5€O�FÑÚ4€O�EÑÚ"9€O�JÑÙ #£€O�HÑÙØ"( ˆ˜ÑÙØ"( ˆ˜Òð rj   r‡   c                 ó   — | t         v S ©N)ÚDDP_RANK_DEVICES©r1   s    rZ   Úrequires_ddp_rankr—   Ñ   s   € ØÔ%Ð%Ð%rj   c                 ó.   ‡ — t        ‰ «      ˆ fd„«       }|S )z”Skips if the world size exceeds the number of GPUs, ensuring that if the
    test is run, each rank has its own GPU via ``torch.cuda.device(rank)``.c                  óœ  •— t         s2t        s,t        s&t        j                  t
        d   j                  «       t        t        j                  d   «      }t         rJt        j                  j                  «       |k  r)t        j                  t
        d|› �   j                  «       t        rJt        j                  j                  «       |k  r)t        j                  t
        d|› �   j                  «       t        rJt        j                  j                  «       |k  r)t        j                  t
        d|› �   j                  «        ‰| i |¤ŽS )Nrq   Ú
WORLD_SIZEú
multi-gpu-)r   r   r   ÚsysÚexitÚ
TEST_SKIPSrb   rg   rG   rH   Útorchr^   Údevice_countr�   r_   )ÚargsÚkwargsr0   Úfuncs      €rZ   Úwrapperzskip_if_no_gpu.<locals>.wrapperÙ   sã   ø€ å�X­Ü�H‰H”Z 	Ñ*×4Ñ4Ô5ÜœŸ™ LÑ1Ó2ˆ
ÝœŸ™×0Ñ0Ó2°ZÒ?Ü�H‰H”Z *¨Z¨LÐ 9Ñ:×DÑDÔEÝœŸ	™	×.Ñ.Ó0°:Ò=Ü�H‰H”Z *¨Z¨LÐ 9Ñ:×DÑDÔEÝœŸ	™	×.Ñ.Ó0°:Ò=Ü�H‰H”Z *¨Z¨LÐ 9Ñ:×DÑDÔEá�TÐ$˜VÑ$Ð$rj   ©r
   ©r£   r¤   s   ` rZ   Úskip_if_no_gpur§   Õ   s"   ø€ ô ˆ4ƒ[ó%ó ð%ð €Nrj   c                 ó.   ‡ — t        ‰ «      ˆ fd„«       }|S )Nc                  óÊ   •— t         j                  d   dk7  rEt        t         j                  d   «      dk  r&t        j                  t
        d   j                  «        ‰| i |¤ŽS )NÚBACKENDrˆ   rš   é   rm   ©rG   rH   rg   rœ   r�   rž   rb   ©r¡   r¢   r£   s     €rZ   r¤   z(skip_if_small_worldsize.<locals>.wrapperñ   sR   ø€ ä�J‰J�yÑ! UÒ*´´B·J±J¸|Ñ4LÓ0MÐPQÒ0QÜ�H‰H”ZÐ 1Ñ2×<Ñ<Ô=á�TÐ$˜VÑ$Ð$rj   r¥   r¦   s   ` rZ   Úskip_if_small_worldsizer®   ð   ó    ø€ Ü
ˆ4ƒ[ó%ó ð%ð €Nrj   c                 ó.   ‡ — t        ‰ «      ˆ fd„«       }|S )Nc                  óÐ   •— t         j                  d   dk7  rHt        t         j                  d   «      dz  dk(  r&t        j                  t
        d   j                  «        ‰| i |¤ŽS )Nrª   rˆ   rš   é   é   ro   r¬   r­   s     €rZ   r¤   z&skip_if_odd_worldsize.<locals>.wrapperü   sW   ø€ ä�J‰J�yÑ! UÒ*´´B·J±J¸|Ñ4LÓ0MÐPQÑ0QÐUVÒ0VÜ�H‰H”Z Ñ0×:Ñ:Ô;á�TÐ$˜VÑ$Ð$rj   r¥   r¦   s   ` rZ   Úskip_if_odd_worldsizer´   û   r¯   rj   c                 ó   ‡ ‡— ˆˆ fd„}|S )Nc                 ó4   •‡ — t        ‰ «      ˆˆ ˆfd„«       }|S )Nc                  ó´   •— ‰dk(  rKt         j                  j                  «       ‰k  r*t        j                  t
        d‰› �   j                  «       y  ‰| i |¤ŽS ©Nr(   r›   )rŸ   r^   r    rœ   r�   rž   rb   )r¡   r¢   r.   r£   Úns     €€€rZ   r¤   zCrequire_n_gpus_for_nccl_backend.<locals>.decorator.<locals>.wrapper  sM   ø€ à˜&Ò ¤U§Z¡Z×%<Ñ%<Ó%>ÀÒ%BÜ—‘œ j°°Ð$4Ñ5×?Ñ?Õ@á˜TÐ, VÑ,Ð,rj   r¥   )r£   r¤   r.   r¹   s   ` €€rZ   Ú	decoratorz2require_n_gpus_for_nccl_backend.<locals>.decorator  s    ù€ Ü	ˆt‹õ	-ó 
ð	-ð ˆrj   ri   )r¹   r.   rº   s   `` rZ   Úrequire_n_gpus_for_nccl_backendr»     s   ù€ õð Ðrj   c                  ó   — d„ } | S )Nc                 ó.   ‡ — t        ‰ «      ˆ fd„«       }|S )Nc                  ó�   •— 	 ddl m}m}  ‰| i |¤ŽS # t        $ r) t	        j
                  t        d   j                  «       Y y w xY w)Nr   )ÚAutoModelForMaskedLMÚ
BertConfigr‚   )Útransformersr¿   rÀ   ÚImportErrorrœ   r�   rž   rb   )r¡   r¢   r¿   rÀ   r£   s       €rZ   r¤   z?import_transformers_or_skip.<locals>.decorator.<locals>.wrapper  sA   ø€ ð>ßIá˜TÐ, VÑ,Ð,øÜò >Ü—‘œ MÑ2×<Ñ<Ö=ð>ús   ƒ “/AÁAr¥   r¦   s   ` rZ   rº   z.import_transformers_or_skip.<locals>.decorator  s    ø€ Ü	ˆt‹ó	>ó 
ð	>ð ˆrj   ri   )rº   s    rZ   Úimport_transformers_or_skiprÃ     s   € ò
ð Ðrj   c                 óô   — t         r"t        j                  j                  «       | k\  ryt        r"t        j
                  j                  «       | k\  ryt        r"t        j                  j                  «       | k\  ryy©NTF)r   rŸ   r^   r    r   r�   r   r_   )Úxs    rZ   Úat_least_x_gpurÇ   $  sS   € Ý”U—Z‘Z×,Ñ,Ó.°!Ò3ØÝ”E—I‘I×*Ñ*Ó,°Ò1ØÝ”E—I‘I×*Ñ*Ó,°Ò1ØØrj   c                 óV   — t        | d   dd «      }t        | «      dk(  s|€y ||«       y)Nr   Ú_handle_test_skipFT)ÚgetattrÚlen)r¡   ÚmsgrÉ   s      rZ   Ú_maybe_handle_skip_if_lt_x_gpurÍ   .  s5   € Ü  Q¡Ð)<¸dÓCÐÜ
ˆ4ƒy�A‚~Ð*Ð2ØÙ�cÔØrj   )Ú	allow_cpuc                ó   ‡ ‡— ˆˆ fd„}|S )zÁSkip if fewer than x accelerators available.

    Args:
        x: Minimum number of accelerators required.
        allow_cpu: If True, run the test on CPU-only machines (no accelerators).
    c                 ó4   •‡ — t        ‰ «      ˆˆ ˆfd„«       }|S )Nc                  ó<  •— t         j                  j                  «       r)t         j                  j                  «       ‰k\  r ‰| i |¤ŽS t        r)t         j
                  j                  «       ‰k\  r ‰| i |¤ŽS t        r)t         j                  j                  «       ‰k\  r ‰| i |¤ŽS ‰r2t         j                  j                  «       st        st        s ‰| i |¤ŽS t        d‰› �   }t        | |j                  «      s t        j                  |j                  «       y y )Nr›   )rŸ   r^   Úis_availabler    r   r�   r   r_   rž   rÍ   rc   rœ   r�   rb   )r¡   r¢   Ú	test_skiprÎ   r£   rÆ   s      €€€rZ   r¤   z4skip_if_lt_x_gpu.<locals>.decorator.<locals>.wrapper?  sä   ø€ ä�z‰z×&Ñ&Ô(¬U¯Z©Z×-DÑ-DÓ-FÈ!Ò-KÙ˜TÐ, VÑ,Ð,ÝœEŸI™I×2Ñ2Ó4¸Ò9Ù˜TÐ, VÑ,Ð,ÝœEŸI™I×2Ñ2Ó4¸Ò9Ù˜TÐ, VÑ,Ð,Ù¤%§*¡*×"9Ñ"9Ô";½xÍ8Ù˜TÐ, VÑ,Ð,Ü" Z°¨sÐ#3Ñ4ˆIÜ1°$¸	×8IÑ8IÔJÜ—‘˜×,Ñ,Õ-ð Krj   r¥   )r£   r¤   rÎ   rÆ   s   ` €€rZ   rº   z#skip_if_lt_x_gpu.<locals>.decorator>  s    ù€ Ü	ˆt‹õ	.ó 
ð	.ð ˆrj   ri   )rÆ   rÎ   rº   s   `` rZ   Úskip_if_lt_x_gpurÔ   6  s   ù€ õð" Ðrj   r¹   c                 ó   ‡ — ˆ fd„}|S )aR  
    Decorator to request a specific world size for a test. The test harness can
    read this attribute to set the number of ranks to spawn. If there are fewer
    than `n` CUDA devices available, the test should be skipped by the harness.

    Usage:
        @require_world_size(3)
        def test_something(self):
            ...
    c                 ó˜   •— ‰| _         t        j                  j                  «       } t	        j
                  |‰k\  d‰› d|› �«      | «      S )Nz	requires z GPUs, found )Ú_required_world_sizerŸ   r^   r    ÚunittestÚ
skipUnless)r£   Ú	availabler¹   s     €rZ   rº   z&requires_world_size.<locals>.decorator^  sR   ø€ Ø$%ˆÔ!Ü—J‘J×+Ñ+Ó-ˆ	ð
Œx×"Ñ"Ø˜‰N˜i¨ s¨-¸	°{ÐCó
à
óð 	rj   ri   )r¹   rº   s   ` rZ   Úrequires_world_sizerÛ   R  s   ø€ ôð Ðrj   ÚobjÚdefaultc                 óæ   — 	 t        | d«      r%t        | j                  «      r| j                  «       n| j                  }t	        | |«      }|j
                  }t        |«      S # t        $ r |cY S w xY w)z¤
    Returns the requested world size for the currently running unittest method on `obj`
    if annotated via `@require_world_size(n)`, else returns `default`.
    Ú_current_test_name)ÚhasattrÚcallablerß   Ú_testMethodNamerÊ   r×   rg   Ú	Exception)rÜ   rÝ   Ú	test_nameÚfnÚvalues        rZ   Úget_required_world_sizerç   h  st   € ð
ô �sÐ0Ô1´h¸s×?UÑ?UÔ6Vð ×"Ñ"Ô$à×$Ñ$ð 	ô
 �S˜)Ó$ˆØ×'Ñ'ˆÜ�5‹zÐøÜò ØŠðús   ‚AA" Á"A0Á/A0c                 ó   ‡ ‡— ˆ ˆfd„}|S )Nc                 ó4   •‡ — t        ‰ «      ˆˆ ˆfd„«       }|S )Nc                  ó2  •— ‰dk7  r ‰| i |¤ŽS t         j                  j                  «       r)t         j                  j                  «       ‰k\  r ‰| i |¤ŽS t        d‰› �   }t        | |j                  «      s t        j                  |j                  «       y y r¸   )
rŸ   r^   rÒ   r    rž   rÍ   rc   rœ   r�   rb   )r¡   r¢   rÓ   r.   r£   rÆ   s      €€€rZ   r¤   z9nccl_skip_if_lt_x_gpu.<locals>.decorator.<locals>.wrapper~  sˆ   ø€ à˜&Ò Ù˜TÐ, VÑ,Ð,Ü�z‰z×&Ñ&Ô(¬U¯Z©Z×-DÑ-DÓ-FÈ!Ò-KÙ˜TÐ, VÑ,Ð,Ü" Z°¨sÐ#3Ñ4ˆIÜ1°$¸	×8IÑ8IÔJÜ—‘˜×,Ñ,Õ-ð Krj   r¥   )r£   r¤   r.   rÆ   s   ` €€rZ   rº   z(nccl_skip_if_lt_x_gpu.<locals>.decorator}  s    ù€ Ü	ˆt‹õ	.ó 
ð	.ð ˆrj   ri   )r.   rÆ   rº   s   `` rZ   Únccl_skip_if_lt_x_gpurë   |  s   ù€ õð Ðrj   c                 ó  — | j                  «       }d|vrt        d«      ‚d|vrt        d«      ‚d|vrt        d«      ‚|d   }|j                  d«      dk(  r|n|j                  d«      d	   }||vrt        d
|› d|› �«      ‚y )NÚ	iterationz(Expected 'iteration' in ddp_logging_dataÚ	has_errorz(Expected 'has_error' in ddp_logging_dataÚerrorz$Expected 'error' in ddp_logging_dataz
Exception raised from éÿÿÿÿr   zDid not find expected z in ddp logging data error: )Ú_get_ddp_logging_dataÚAssertionErrorÚfindÚsplit)Ú	model_DDPÚ
err_substrÚddp_logging_dataÚlogging_errÚactuals        rZ   Úverify_ddp_error_loggedrú   �  s¹   € à ×6Ñ6Ó8ÐØÐ*Ñ*ÜÐGÓHÐHØÐ*Ñ*ÜÐGÓHÐHØÐ&Ñ&ÜÐCÓDÐDØ" 7Ñ+€Kð �?‰?Ð5Ó6¸"Ò<ñ 	à×ÑÐ8Ó9¸!Ñ<ð ð
 �[Ñ ÜØ$ V HÐ,HÈÈÐVó
ð 	
ð !rj   c                 ó.   ‡ — t        ‰ «      ˆ fd„«       }|S )aJ  
    Convenience decorator to set/unset TORCH_NCCL_BLOCKING_WAIT flag. Note that use of
    this decorator will override the setting of TORCH_NCCL_ASYNC_ERROR_HANDLING for
    the particular test. After the test, both TORCH_NCCL_BLOCKING_WAIT and
    TORCH_NCCL_ASYNC_ERROR_HANDLING will be restored to their original values.
    c                  óÜ  •— 	 t         j                  d   }t         j                  d= 	 t         j                  d   }dt         j                  d<   	  ‰| i |¤Ž}||�|t         j                  d<   |�|t         j                  d<   S S # t        $ r d }Y Œjw xY w# t        $ r d }Y Œgw xY w# dt         j                  d<   w xY w# |�|t         j                  d<   |�|t         j                  d<   w w xY w)NÚTORCH_NCCL_ASYNC_ERROR_HANDLINGÚTORCH_NCCL_BLOCKING_WAITÚ1)rG   rH   ÚKeyError)r¡   r¢   Ú cached_nccl_async_error_handlingÚcached_nccl_blocking_waitÚretr£   s        €rZ   r¤   z(with_nccl_blocking_wait.<locals>.wrapper«  s  ø€ ð	4Ü;=¿:¹:Ø1ñ<Ð,ô —
‘
Ð<Ð=ð	9Ü46·J±JØ*ñ5Ð%ð 69ŒB�J‰JÐ1Ñ2ð	SÙ˜Ð' Ñ'ˆCØð 0Ð;à4ô —
‘
Ð<Ñ=ð )Ð4Ø9R”—
‘
Ð5Ò6ð 5øô1 ò 	4à/3Ò,ð	4ûô ò 	-Ø(,Ò%ð	-ûð 69ŒB�J‰JÐ1Ò2ûð 0Ð;à4ô —
‘
Ð<Ñ=ð )Ð4Ø9R”—
‘
Ð5Ò6ð 5ús@   ƒ$B ¨B Á	B> ÂBÂBÂB#Â B& Â"B#Â#B& Â&B;Â>-C+r¥   r¦   s   ` rZ   Úwith_nccl_blocking_waitr  £  s%   ø€ ô ˆ4ƒ[ó Só ð SðD €Nrj   c                 ó   ‡ — ˆ fd„}|S )zK
    Runs a test for each distributed debug level specified in levels.
    c                 ó2   •‡ — t        ‰ «      ˆ ˆfd„«       }|S )Nc                  ó  •— t         j                  j                  dd «      }‰D ][  }|t         j                  d<   t        j                  «         ‰| i |¤Ž}t        j
                  «        |€ŒI|t         j                  d<   Œ] S )NÚTORCH_DISTRIBUTED_DEBUG)rG   rH   ÚgetrD   Úset_debug_level_from_envÚbarrier)r¡   r¢   Ú	old_levelÚlevelr  r£   Úlevelss        €€rZ   r¤   z:with_dist_debug_levels.<locals>.decorator.<locals>.wrapper×  sx   ø€ äŸ
™
Ÿ™Ð'@À$ÓGˆIØò F�Ø8=”—
‘
Ð4Ñ5Ü×-Ñ-Ô/Ù˜DÐ+ FÑ+�Ü—‘”ØÑ(Ø<E”B—J‘JÐ8Ò9ðFð ˆJrj   r¥   )r£   r¤   r  s   ` €rZ   rº   z)with_dist_debug_levels.<locals>.decoratorÖ  s    ù€ Ü	ˆt‹ô	ó 
ð	ð ˆrj   ri   )r  rº   s   ` rZ   Úwith_dist_debug_levelsr  Ñ  s   ø€ ô
ð$ Ðrj   c                  ó@   — t        t        j                  «        d«      S )Nz+c10d was not compiled with the Gloo backend)r   rD   Úis_gloo_availableri   rj   rZ   Úrequires_gloor  ë  ó!   € Ü)Ü×"Ñ"Ó$Ð$Ø5óð rj   c           	      ó  — t         sd„ S t        j                  «       st        d«      S t	        t
        j                  j                  j                  «       | k  d| › dt
        j                  j                  j                  «       › d|› �«      S )Nc                 ó   — | S r”   ri   )Úfs    rZ   ú<lambda>z'requires_nccl_version.<locals>.<lambda>ô  s   € ˜€ rj   ú+c10d was not compiled with the NCCL backendz0Requires NCCL version greater than or equal to: z	, found: z
, reason: )	r   rD   Úis_nccl_availabler   r   rŸ   r^   r(   Úversion)r  rÌ   s     rZ   Úrequires_nccl_versionr  ò  s   € ÝÙÐÜ×!Ñ!Ô#Ü*Ø9ó
ð 	
ô .Ü�J‰J�O‰O×#Ñ#Ó%¨Ñ/Ø>¸w¸iÀyÔQV×Q[ÑQ[×Q`ÑQ`×QhÑQhÓQjÐPkÐkuÐvyÐuzÐ{ó
ð 	
rj   c                  ó   — t        dd«      S )zK
    Require NCCL shrink support (NCCL available and version >= 2.27).
    )r²   é   z Need NCCL 2.27+ for shrink_group)r  ri   rj   rZ   Úrequires_nccl_shrinkr     s   € ô ! Ð*LÓMÐMrj   c                  ó@   — t        t        j                  «        d«      S )Nr  )r   rD   r  ri   rj   rZ   Úrequires_ncclr     r  rj   c                  ó@   — t        t        j                  «        d«      S )Nz*c10d was not compiled with the UCC backend)r   rD   Úis_ucc_availableri   rj   rZ   Úrequires_uccr#    ó!   € Ü)Ü×!Ñ!Ó#Ð#Ø4óð rj   c                  ó@   — t        t        j                  «        d«      S )Nz*c10d was not compiled with the MPI backend)r   rD   Úis_mpi_availableri   rj   rZ   Úrequires_mpir'    r$  rj   c                 óV   — | €t         } t        d„ | D «       «      }t        | d| › �«      S )aÆ  
    Decorator to skip tests if no accelerator communication backend (NCCL, XCCL, HCCL) is available.

    Args:
        backends (Optional[List[str]]): Specific accelerator backends to check (e.g., ["nccl", "xccl", "hccl"]).
                                       If None, checks all supported accelerator backends (NCCL, XCCL, HCCL).

    Returns:
        callable: A decorator that skips the test if no specified accelerator backend is available.
    c              3   óŽ   K  — | ]=  }	 t        j                  t         j                  d „ dœj                  |d„ «      «       –— Œ? y­w)c                  ó   — t         S r”   )r   ri   rj   rZ   r  z=requires_accelerator_dist_backend.<locals>.<genexpr>.<lambda>.  s   € œH€ rj   r\   c                   ó   — y©NFri   ri   rj   rZ   r  z=requires_accelerator_dist_backend.<locals>.<genexpr>.<lambda>/  s   � rj   N)rD   r  Úis_xccl_availabler	  )Ú.0r.   s     rZ   ú	<genexpr>z4requires_accelerator_dist_backend.<locals>.<genexpr>*  sG   è ø€ ò ð ð	&Ü×*Ñ*Ü×*Ñ*Ù$ñ	
÷ ‰#ˆg‘}Ó
%÷		(ñùs   ‚AAz5No accelerator communication backend available among )ÚACCELERATOR_DIST_BACKENDSÚanyr   )ÚbackendsÚbackend_availables     rZ   Ú!requires_accelerator_dist_backendr4    sH   € ð ÐÜ,ˆäñ ð  ôó Ðô *ØÐØ
?À¸zÐJóð rj   c                  ó¤   — t         j                  j                  «       xr$ t        j                  t
        j                  d«      } t        |  d«      S )Nr   z"multicast support is not available)rŸ   r^   rÒ   r   Úhas_multicast_supportr   ÚCUDAr   )r6  s    rZ   Úrequires_multicast_supportr8  9  sI   € ä�
‰
×ÑÓ!ò 	GÜ×2Ñ2´:·?±?ÀAÓFð ô *Ø!Ð!Ø,óð rj   c                  ó�   — t         r@t        r9ddg} | D ]/  }|t        j                  j	                  d«      j
                  v sŒ/ y yyy)NÚgfx942Úgfx950r   TF)r   r   rŸ   r^   Úget_device_propertiesÚgcnArchName)Ú	arch_listÚarchs     rZ   Ú#evaluate_platform_supports_symm_memr@  D  sL   € ÝÝØ! 8Ð,ˆIØ!ò  �Øœ5Ÿ:™:×;Ñ;¸AÓ>×JÑJÒJÙð ð ààrj   c                  ó   — t        «       S r”   )r@  ri   rj   rZ   r  r  S  s
   € Ô/Ó1€ rj   ÚPLATFORM_SUPPORTS_SYMM_MEMc                 ód   —  t        j                  t        t        d   j                  «      | «      S )z&Skips a test for ROCm multiprocess UTsr|   )rØ   ÚskipIfr   rž   rc   )r£   s    rZ   Úskip_if_rocm_multiprocessrE  W  s%   € àLŒ8�?‰?œ>¬:°lÑ+C×+KÑ+KÓLÈTÓRÐRrj   r?  .c                 ó   ‡ — ˆ fd„}|S )z4Skips a test for given ROCm archs - multiprocess UTsc                 óØ   •— d }t         rDt        j                  j                  d«      j                  j                  d«      d   }|‰v rd‰› �} t        j                  |d u|«      | «      S )Nr   ú:z0skip_if_rocm_arch_multiprocess: test skipped on )r   rŸ   r^   r<  r=  rô   rØ   rD  )r£   ÚreasonÚpropr?  s      €rZ   rº   z1skip_if_rocm_arch_multiprocess.<locals>.decorator_  se   ø€ ØˆÝÜ—:‘:×3Ñ3°AÓ6×BÑB×HÑHÈÓMÈaÑPˆDØ�t‰|ØKÈDÈ6ÐR�à:Œx�‰˜v¨TÐ1°6Ó:¸4Ó@Ð@rj   ri   )r?  rº   s   ` rZ   Úskip_if_rocm_arch_multiprocessrK  \  s   ø€ ôAð Ðrj   c                 ó   ‡ — ˆ fd„}|S )z:Skips a test for ROCm based on ROCm ver - multiprocess UTsc                 ó:  •— d }t         rut        t        j                  j                  «      }|j                  dd¬«      d   }t        d„ |j                  d«      D «       «      }|�‰�|t        ‰«      k  r	d|› d‰› d	�} t        j                  |d u|«      | «      S )
Nú-r³   ©Úmaxsplitr   c              3   ó2   K  — | ]  }t        |«      –— Œ y ­wr”   )rg   )r.  rÆ   s     rZ   r/  zLskip_if_rocm_ver_lessthan_multiprocess.<locals>.decorator.<locals>.<genexpr>s  s   è ø€ Ò&O°!¤s¨1§vÑ&Oùs   ‚ú.z-skip_if_rocm_ver_lessthan_multiprocess: ROCm z is available but z	 required)	r   rF   rŸ   r  Úhiprô   ÚtuplerØ   rD  )r£   rI  Úrocm_versionÚrocm_version_tupler  s       €rZ   rº   z9skip_if_rocm_ver_lessthan_multiprocess.<locals>.decoratorn  sª   ø€ ØˆÝÜœuŸ}™}×0Ñ0Ó1ˆLØ'×-Ñ-¨c¸AÐ-Ó>¸qÑAˆLÜ!&Ñ&O°|×7IÑ7IÈ#Ó7NÔ&OÓ!OÐà"Ð*Ø�?Ø%¬¨g«Ò6àHÐI[ÐH\Ð\nÐovÐnwð  xAð  B�à:Œx�‰˜v¨TÐ1°6Ó:¸4Ó@Ð@rj   ri   )r  rº   s   ` rZ   Ú&skip_if_rocm_ver_lessthan_multiprocessrW  k  s   ø€ ôAð Ðrj   c                  ó<   — t        t        j                  dk(  d«      S )NÚwin32z8This unit test case is not supported on Windows platform)r   rœ   Úplatformri   rj   rZ   Úskip_if_win32r[  €  s   € Ü)Ü�‰˜ÑØBóð rj   ÚmajorÚminorc                 ó    — | j                   dk7  ryt        j                  j                  �yt        j                  j                  | «      ||fk\  S )zç
    Returns True if the device's compute capability is (major, minor) or higher.
    Error out if the device is not a CUDA device.
    Returns False if device is a RoCM device.
    Returns True if device is a non-CUDA device.
    r^   TF)ÚtyperŸ   r  rS  r^   Úget_device_capability)r1   r\  r]  s      rZ   Úsm_is_or_higher_thanra  ‡  sD   € ð ‡{�{�fÒØä‡}�}×ÑÐ$àä�:‰:×+Ñ+¨FÓ3¸¸u°~ÑEÐErj   Ú	localhostr³   é   )Úminutesc                 óÚ   — t        «       }|rEt        |t        d¬«      z  «      }t        j                  j
                  j                  | ||||«      S t        j                  | |||||¬«      S )zL
    Creates a TCP store. Retries if the chosen port is already in use.
    r³   )Úmilliseconds)Úwait_for_workersÚ	use_libuv)r   rg   r   rŸ   ÚclassesÚ	dist_c10dÚTCPStorerD   )	Úaddrr0   Ú	is_masterÚtimeoutrg  Ú	jit_classrh  ÚportÚtimeout_milliseconds	            rZ   Úcreate_tcp_storerr  ˜  sr   € ô Ó€DÙÜ! '¬IÀ1Ô,EÑ"EÓFÐÜ�}‰}×&Ñ&×/Ñ/Ø�$˜
 IÐ/Bó
ð 	
ô �}‰}ØØØØØ-Øô
ð 	
rj   iô  Ú!DISTRIBUTED_TESTS_DEFAULT_TIMEOUTÚ300i�  )Útest_ddp_uneven_inputsÚtest_DistributedDataParalleléÈ   Útest_join_kwargsÚ	lazy_initc                 ó°   — t         j                  dk(  s| €!t        j                  j	                  d|¬«      S t        j                  j	                  | |¬«      S )NrY  z	127.0.0.1)Úhostnamery  ©Ú	interfacery  )rœ   rZ  rD   ÚProcessGroupGlooÚcreate_devicer|  s     rZ   r  r  Æ  s[   € Ü
‡|�|�wÒ )Ð"3Ü×$Ñ$×2Ñ2Ø ¨Ið 3ó 
ð 	
ô ×$Ñ$×2Ñ2Ø¨9ð 3ó 
ð 	
rj   c                 óZ   — t         j                  | j                  d«      d   t        «      S ©NrR  rð   )ÚTIMEOUT_OVERRIDEr	  rô   ÚTIMEOUT_DEFAULT)Útest_ids    rZ   Úget_timeoutr…  Ñ  s#   € Ü×Ñ §¡¨cÓ 2°2Ñ 6¼ÓHÐHrj   c               #   óN  K  — t        «       t        «       }} t        j                  t        j                  }}	 | |ct        _        t        _        t        j                  t        j                  f–— ||ct        _        t        _        y # ||ct        _        t        _        w xY w­wr”   )r   rœ   ÚstdoutÚstderr)Únew_outÚnew_errÚold_outÚold_errs       rZ   Úcaptured_outputr�  Õ  sl   è ø€ ä“z¤8£:ˆW€GÜ—z‘z¤3§:¡:ˆW€Gð2Ø!(¨'ÐŒŒ
”C”JÜ�j‰jœ#Ÿ*™*Ð$Ò$à!(¨'ÐŒŒ
”C•Jø ¨'ÐŒŒ
”C•Jüs   ‚5B%¸9B	 Á1B%Â	B"Â"B%Ú
num_inputsc                 ó¸  — ddt         dt         dt         dt         fd„}dt         fd„}t        |d¬«      t        |d	¬«      t        |d
¬«      t        |d¬«      t        |d	¬«      t        |d
¬«      fD ��cg c]N  }t        |«      D �cg c]  } ||| z  |z   ||z  «      ‘Œ c}t        |«      D �cg c]  } ||||z  «      ‘Œ c}f‘ŒP c}}S c c}w c c}w c c}}w )zæ
    Generate a number of basic test cases for sparse reduction.
    These cover tensors with a varying number of sparse dimensions and a varying
    number of dense dimensions. The only reduction operation we support is sum.
    r³   r/   r0   Úsparse_dimsÚ
dense_dimsc           	      óê  — t        j                  t        j                  | dz   «      d| dz   f«      }|gt        |«      D �cg c]  }d‘Œ c}z   }t        |dz
  «      D ]A  }t        j                  |t        j
                  d| dz   «      f«      }|j                  |«       ŒC t        j                  | dz   gt        |«      D �cg c]  }d‘Œ c}z   «      }t        j                  |||«      S c c}w c c}w )Nr³   r²   )	rŸ   ÚreshapeÚarangerU   ÚcatÚzerosÚappendÚonesÚsparse_coo_tensor)r/   r0   r�  r‘  ÚindicesÚ_ÚshapeÚvaluess           rZ   Úgeneratez,simple_sparse_reduce_tests.<locals>.generateç  sÖ   € ô —-‘-¤§¡¨T°A©XÓ 6¸¸DÀ1¹H¸ÓFˆØ�¬5°Ó+<Ö= a¢Ò=Ñ=ˆÜ�{ Q‘Ó'ò 	%ˆAÜ—i‘i ¬%¯+©+°a¸À¹Ó*BÐ CÓDˆGØ�L‰L˜Õ$ð	%ô —‘˜T A™X˜J´U¸:Ó5FÖ)G°ª!Ò)GÑGÓHˆÜ×&Ñ& w°¸Ó>Ð>ùò  >ùò *Hs   Á	C+Ã	C0
c           
      ó|   — t        t        j                  t        |«      D �cg c]  } | ||«      ‘Œ c}«      S c c}w r”   )r	   ÚoperatorÚaddrU   )rå   r0   r/   s      rZ   Úcompute_sumz/simple_sparse_reduce_tests.<locals>.compute_sumó  s2   € ÜÜ�L‰L¼EÀ*Ó<MÖN°D™2˜d JÕ/ÒNó
ð 	
ùÚNs   ¢9
)r�  r²   é   )r‘  )r³   r   )rg   r   rU   )r/   r0   rŽ  rž  r¢  rå   Úis          rZ   Úsimple_sparse_reduce_testsr¥  à  sò   € ñ
?”sð 
?¬ð 
?¼#ð 
?Ìsó 
?ð
¤Có 
ô �H¨!Ô,Ü�H¨!Ô,Ü�H¨!Ô,Ü�H¨Ô+Ü�H¨Ô+Ü�H¨Ô+ð
÷ð ô	 ˜zÓ*öàñ �: Ñ$ qÑ(¨*°zÑ*AÕBòô @EÀZÓ?PÖQ¸!‰[˜˜Z¨*Ñ4Õ5ÒQò	
óð ùòùò Rùós$   Á5CÂCÂ CÂ/CÃCÃ
Cc           
      óf  — t         j                  j                  «       }t        rt         j                  j                  «       }t
        rt         j                  j                  «       }t        |«      }d}| |kD  r|| z  }t        | «      D �ci c]  }|t        |||z  |dz   |z   «      “Œ }}|S c c}w )züMultigpu tests are designed to simulate the multi nodes with multi
    GPUs on each node. Nccl backend requires equal #GPUs in each process.
    On a single node, all visible GPUs are evenly
    divided to subsets, each process only uses a subset.
    r³   )	rŸ   r^   r    r   r�   r   r_   rU   Úlist)r0   r.   ÚnGPUsÚvisible_devicesÚnGPUs_per_processr¤  Úrank_to_GPUs          rZ   Úinit_multigpu_helperr¬    s¶   € ô �J‰J×#Ñ#Ó%€EÝÜ—	‘	×&Ñ&Ó(ˆÝÜ—	‘	×&Ñ&Ó(ˆÜ˜E“l€Oð ÐØ�EÒØ! ZÑ/Ðô �zÓ"öàð 	
Œ4� Ð$5Ñ 5¸¸Q¹ÐBSÑ8SÐTÓUÑUð€Kð ð Ðùò	s   ÂB.Útmp_dirÚinit_methodc                 ó”  — t        j                  «       at        j                  t        j
                  d<   t	        j                  t        j                  j                  t        j                  d«      «       t	        j                  t        j                  j                  t        j                  d«      «       t        j                  j                  t        j                  d«      }t	        j                  |«       | �| t        j
                  d<   y t        t        j                  j                  |d«      z   t        j
                  d<   y )NÚTEMP_DIRr  Útest_dirÚinit_dirÚINIT_METHODÚshared_init_file)
ÚtempfileÚTemporaryDirectoryr­  r=   rG   rH   ÚmkdirÚpathÚjoinr   )r®  Úinit_dir_paths     rZ   Úinitialize_temp_directoriesr»  (  sÀ   € ä×)Ñ)Ó+€GÜ$Ÿ\™\„B‡J�JˆzÑÜ‡H�HŒR�W‰W�\‰\œ'Ÿ,™,¨	Ó2Ô3Ü‡H�HŒR�W‰W�\‰\œ'Ÿ,™,¨
Ó3Ô4Ü—G‘G—L‘L¤§¡¨zÓ:€MÜ‡H�Hˆ]ÔàÐØ$/Œ�
‰
�=Ò!ä$/´"·'±'·,±,ØÐ-ó3
ñ %
Œ�
‰
�=Ò!rj   c                  ó:   — t         �t         j                  «        y y r”   )r­  Úcleanupri   rj   rZ   Úcleanup_temp_dirr¾  9  s   € ÜÐÜ�‰Õð rj   ÚprocessÚcompletion_queuern  c                 óŽ  — |€dnt        dt        d|dz  «      «      }t        j                  «       }	 	 |j                  |¬«      S # t        j
                  $ r= | j                  «       s*|j                  «       rt        d| j                  › �«      cY S Y nw xY w|�+t        j                  «       |z
  }||kD  rt        d|› d�«      S Œ”)z³Get result from the completion_queue associated with process.

    When the process finished without putting a result or the timeout expired an exception instance will be returnedéx   é
   é   ©rn  zExited with zProcess timed out after Ús)
ÚmaxÚminÚtimer	  ÚqueueÚEmptyÚis_aliveÚemptyÚRuntimeErrorÚexitcode)r¿  rÀ  rn  Úqueue_timeoutÚ
start_timeÚelapseds         rZ   Ú%retrieve_result_from_completion_queuerÓ  >  sÌ   € ð #˜?‘C´°B¼¸CÀÈAÁÓ8NÓ0O€MÜ—‘“€Jà
ð	GØ#×'Ñ'°Ð'Ó>Ð>øÜ�{‰{ò 	Gð ×#Ñ#Ô%Ð*:×*@Ñ*@Ô*BÜ# l°7×3CÑ3CÐ2DÐ$EÓFÒFùð	Gúð ÐÜ—i‘i“k JÑ.ˆGØ˜Ò Ü#Ð&>¸w¸iÀqÐ$IÓJÐJð s   ´A ÁABÂBrÄ  r²   c            	       ó6  ‡ — e Zd ZdZdZdefd„Zedefd„«       Zede	fd„«       Z
d„ Z	 dded	edd
fˆ fd„Zdˆ fd„Zdˆ fd„Zdefd„Zdd„Zdd„Z G d„ de«      Zede	fd„«       Zede	dededd
fd„«       Zdedd
fd„Zdd„Zdd„Zdd„Zedefd„«       Zˆ xZS )ÚMultiProcessTestCaserð   rÃ  r4   c                  ó   — yr,  ri   ©Úselfs    rZ   Ú_should_stop_test_suitez,MultiProcessTestCase._should_stop_test_suites  s   € Ørj   c                  ó   — y)NTri   r×  s    rZ   Údestroy_pg_upon_exitz)MultiProcessTestCase.destroy_pg_upon_exit{  s   € àrj   c                 ó   — t         S r”   ©ÚDEFAULT_WORLD_SIZEr×  s    rZ   r0   zMultiProcessTestCase.world_size  ó   € ä!Ð!rj   c                 óV   ‡— t        ‰«      ˆfd„«       }t        j                  || «      S )Nc                 ój   •— | j                   | j                  k(  r| j                  ‰«       y  ‰«        y r”   )r/   ÚMAIN_PROCESS_RANKÚ_join_processes©rØ  rå   s    €rZ   r¤   z1MultiProcessTestCase.join_or_run.<locals>.wrapper„  s(   ø€ à�y‰y˜D×2Ñ2Ò2Ø×$Ñ$ RÕ(á•rj   ©r
   ÚtypesÚ
MethodType©rØ  rå   r¤   s    ` rZ   Újoin_or_runz MultiProcessTestCase.join_or_runƒ  ó.   ø€ Ü	ˆr‹ó	ó 
ð	ô ×Ñ ¨Ó.Ð.rj   Úmethod_nameÚ
methodNameNc                 óò   •— |dk7  r|}t         ‰| �  |«       	 t        | |«      }t        | || j	                  |«      «       y # t
        $ r+}|dk7  rt        d| j                  › d|› �«      |‚Y d }~y d }~ww xY w©NÚrunTestzno such test method in ú: ©ÚsuperÚ__init__rÊ   Úsetattrré  ÚAttributeErrorÚ
ValueErrorÚ	__class__©rØ  rë  rì  rå   Úer÷  s        €rZ   ró  zMultiProcessTestCase.__init__‘  ó‹   ø€ ð
 ˜Ò"Ø$ˆKÜ‰Ñ˜Ô%ð		Ü˜˜{Ó+ˆBÜ�D˜+ t×'7Ñ'7¸Ó';Õ<øÜò 	Ø˜YÒ&ô !Ø-¨d¯n©nÐ-=¸RÀ
¸|ÐLóàðô 'ûð	úó   ™(A Á	A6Á!A1Á1A6c                 ó  •— t         ‰| �  «        i | _        g | _        g | _        | j
                  | _        t        j                  d¬«      5 }|j                  | _
        d d d «       i | _        y # 1 sw Y   i | _        y xY w)NF©Údelete)rò  ÚsetUpÚspecial_return_code_checksÚskip_return_code_checksÚ	processesrâ  r/   rµ  ÚNamedTemporaryFiler=   Ú	file_nameÚpid_to_pipe)rØ  r  r÷  s     €rZ   rÿ  zMultiProcessTestCase.setUp¤  ss   ø€ Ü‰‰Œð 13ˆÔ'ð .0ˆÔ$àˆŒØ×*Ñ*ˆŒ	Ü×(Ñ(°Ô6ð 	$¸!ØŸV™VˆDŒN÷	$ð ˆÕ÷	$ð ˆÕús   ÁA.Á.A>c                 ór   •— t         ‰| �  «        | j                  D ]  }|j                  «        Œ g | _        y r”   )rò  ÚtearDownr  Ú	terminate)rØ  Úpr÷  s     €rZ   r  zMultiProcessTestCase.tearDown¶  s3   ø€ Ü‰ÑÔØ—‘ò 	ˆAØ�K‰K�Mð	ð ˆ�rj   c                 óF   — | j                  «       j                  d«      d   S r�  ©Úidrô   r×  s    rZ   rß   z'MultiProcessTestCase._current_test_nameÀ  s   € à�w‰w‹y�‰˜sÓ# BÑ'Ð'rj   c                 ó  — g | _         t        t        | j                  «      «      D ]Ý  }t        j
                  j                  «       \  }} || j                  j                  dt        |«      z   || j                  «       | j                  |fdt        | dd«      i¬«      }|j                  «        t        j                  d||j                   «       || j"                  |j                   <   | j                   j%                  |«       Œß y )Núprocess Úfake_pgF)Útargetr=   r¡   r¢   úStarted process %s with pid %s)r  rU   rg   r0   rŸ   ÚmultiprocessingÚPiper÷  Ú_runrF   rß   r  rÊ   ÚstartÚloggerÚinfoÚpidr  r—  )rØ  Úprocr/   Úparent_connÚ
child_connr¿  s         rZ   Ú_start_processesz%MultiProcessTestCase._start_processesÄ  sÛ   € ØˆŒÜœ#˜dŸo™oÓ.Ó/ò 	+ˆDÜ&+×&;Ñ&;×&@Ñ&@Ó&BÑ#ˆK˜ÙØ—~‘~×*Ñ*Ø¤# d£)Ñ+àØ×+Ñ+Ó-Ø—N‘NØð	ð œw t¨Y¸Ó>ðôˆGð �M‰MŒOÜ�K‰KÐ8¸$ÀÇÁÔLØ,7ˆD×Ñ˜WŸ[™[Ñ)Ø�N‰N×!Ñ! 'Õ*ñ%	+rj   c                 óÖ   — 	 t         j                  j                  d«       t         j                  j	                  d«      j
                  }| j                  |«       y # t        $ r Y ŒFw xY w)NÚspawn)rŸ   r  Úset_start_methodrÎ  Úget_contextÚProcessr  )rØ  r  s     rZ   Ú_spawn_processesz%MultiProcessTestCase._spawn_processesÚ  s[   € ð	Ü×!Ñ!×2Ñ2°7Ô;ô ×$Ñ$×0Ñ0°Ó9×AÑAˆØ×Ñ˜dÕ#øô	 ò 	Ùð	ús   ‚A Á	A(Á'A(c                   ó   — e Zd ZdZy)úMultiProcessTestCase.Eventr³   N)rd   re   rf   ÚGET_TRACEBACKri   rj   rZ   ÚEventr$  ã  s   „ Ø‰rj   r&  r/   c                 ó‚  — t         j                  d|«       	 t        j                  j	                  | |g«      }| |v rñ| j
                  rt         j                  d|«       y | j                  «       }t         j                  d||«       |t        j                  j                  k(  rŠt        j                  d¬«      5 }t        j                  |«       |j                  «        |j!                  d«       | j#                  |j%                  «       «       t         j                  d|«       d d d «       ||v ry �Œ# 1 sw Y   ŒxY w)Nz*Starting event listener thread for rank %sz:Pipe closed for process %s, stopping event listener threadzReceived event %s on process %szr+)Úmoder   zProcess %s sent traceback)r  Údebugr  Ú
connectionÚwaitÚclosedÚrecvr  rÕ  r&  r%  rµ  r  ÚfaulthandlerÚdump_tracebackÚflushÚseekÚsendÚread)Úparent_pipeÚsignal_piper/   Úready_pipesÚeventÚtmp_files         rZ   Ú_event_listenerz$MultiProcessTestCase._event_listeneræ  s  € ä�‰ÐAÀ4ÔHØÜ)×4Ñ4×9Ñ9¸;ÈÐ:TÓUˆKà˜kÑ)Ø×%Ò%Ü—L‘LØTØôð à#×(Ñ(Ó*�Ü—‘Ð=¸uÀdÔKàÔ0×6Ñ6×DÑDÒDä!×4Ñ4¸$Ô?ð GÀ8Ü$×3Ñ3°HÔ=à Ÿ™Ô(Ø Ÿ™ aÔ(Ø#×(Ñ(¨¯©«Ô9äŸ™Ð$?ÀÔF÷Gð ˜kÑ)Øñ5 ÷ Gð Gús   Â:A,D5Ä5D>rä   r  c                 óT   —  | |«      }||_         ||_        |j                  ||«       y r”   )r/   r  Úrun_test©Úclsr/   rä   r  r4  r¢   rØ  s          rZ   r  zMultiProcessTestCase._run  s)   € ñ �9‹~ˆØˆŒ	Ø"ˆŒØ�‰�i Õ-rj   c                 ó:  — t         j                  j                  d¬«      \  }}t        j                  t
        j                  ||| j                  fd¬«      }|j                  «        t        j                  dk7  r2t        j                  dk7  rt         j                  j                  d«       dt        j                  d<   t        j                   «        	  t#        | |«      «        |�|j=                  d «       |€t?        d«      ‚|jA                  «        |jC                  «        | jD                  r	 tG        jH                  «        y y # t$        j&                  $ rR}t(        j+                  d	| j                  ||«       t        j,                  t.        d
   j0                  «       Y d }~ŒÃd }~wt2        $ rŒ t(        j5                  dt7        j8                  «       | j                  t
        j:                  «       |j=                  t7        j8                  «       «       t        j,                  t
        j:                  «       Y �ŒZw xY w# |�|j=                  d «       |€t?        d«      ‚|jA                  «        |jC                  «        w xY w# t>        tJ        f$ r Y y w xY w)NF)ÚduplexT)r  r¡   ÚdaemonrY  Údarwinrÿ   ÚTORCH_SHOW_CPP_STACKTRACESz4Process %s skipping test %s for following reason: %sr€   z;Caught exception: 
%s exiting process %s with exit code: %sz-Expected event_listener_thread to not be None)&rŸ   r  r  Ú	threadingÚThreadrÕ  r9  r/   r  rœ   rZ  Ú_CÚ'_set_print_stack_traces_on_fatal_signalrG   rH   r   Úset_rng_seedrÊ   rØ   ÚSkipTestr  r  r�   rž   rb   rã   rï   Ú	tracebackÚ
format_excÚTEST_ERROR_EXIT_CODEr2  rò   r¹  ÚcloserÛ  rD   Údestroy_process_grouprö  )rØ  rä   r4  Úsignal_recv_pipeÚsignal_send_pipeÚevent_listener_threadÚses          rZ   r;  zMultiProcessTestCase.run_test  s!  € ä-2×-BÑ-B×-GÑ-GÈuÐ-GÓ-UÑ*ÐÐ*Ü )× 0Ñ 0Ü'×7Ñ7ØÐ/°·±Ð;Øô!
Ðð
 	×#Ñ#Ô%Ü�<‰<˜7Ò"¤s§|¡|°xÒ'?ô �H‰H×<Ñ<¸TÔBà36Œ�
‰
Ð/Ñ0Ü×!Ñ!Ô#ð	 Ø$ŒG�D˜)Ó$Ô&ð(  Ð+Ø ×%Ñ% dÔ+à$Ð,Ü$Ð%TÓUÐUØ!×&Ñ&Ô(à×ÑÔà×$Ò$ðô ×*Ñ*Õ,ð	 %øô9 × Ñ ò 	6Ü�K‰KØFØ—	‘	ØØô	ô �H‰H”Z 	Ñ*×4Ñ4×5Ñ5ûÜò 		@Ü�L‰LØNÜ×$Ñ$Ó&Ø—	‘	Ü$×9Ñ9ô	ð ×ÑœY×1Ñ1Ó3Ô4Ü�H‰HÔ)×>Ñ>×?ð		@ûð  Ð+Ø ×%Ñ% dÔ+à$Ð,Ü$Ð%TÓUÐUØ!×&Ñ&Ô(à×ÑÕûô #¤JÐ/ò ÙðúsK   ÃE Ä2J ÅI ÅAF(Æ#I Æ(BI È<I È?I É I ÉAJÊJÊJc                 ó˜  — g }t        | j                  «      D ]h  \  }}|j                  �Œ| j                  |j                     }	 |j                  t        j                  j                  «       |j                  ||f«       Œj |D ]x  \  }}	 |j                  d«      rK|j                  rt        j                  d|«       Œ;|j!                  «       }t        j#                  d||«       nt        j#                  d|«       Œz y # t        $ r t        j                  d|«       Y �Œw xY w# t        $ r t        j                  d|«       Y ŒÃw xY w)Nz>Encountered error while trying to get traceback for process %src  z5Pipe closed for process %s, cannot retrieve tracebackz)Process %s timed out with traceback: 

%sz6Could not retrieve traceback for timed out process: %s)Ú	enumerater  rÏ  r  r  r2  rÕ  r&  r%  r—  ÚConnectionErrorr  Ú	exceptionÚpollr,  r  r-  rï   )rØ  Úpipesr¤  r¿  Úpiper/   rI  s          rZ   Ú_get_timedout_process_tracebackz4MultiProcessTestCase._get_timedout_process_tracebackG  s0  € ØˆÜ# D§N¡NÓ3ò 
	‰JˆAˆwØ×ÑÑ'Ø×'Ñ'¨¯©Ñ4�ðØ—I‘IÔ2×8Ñ8×FÑFÔGØ—L‘L ! T Õ+ð
	ð  ò 	‰JˆD�$ðà—9‘9˜Q”<Ø—{’{ÜŸ™ØSØ ôð !à $§	¡	£�IÜ—L‘LØEÀtÈYõô —L‘LØPÐRVôøñ!	øô 'ò Ü×$Ñ$ØXØ÷ðûô4 #ò Ü× Ñ ØTØöðús*   Á<DÂ3D'Ã >D'ÄD$Ä#D$Ä'E	ÅE	c                 óÄ  — t        | j                  «       «      }t        j                  «       }d}	 	 t        | j                  «      D ]w  \  }}|j
                  t        j                  k(  sŒ$t        d|› d|j
                  › d�«       t        j                  j                  «       }|D ]  }|j                  «        Œ d} n |rn‘t        d„ | j                  D «       «      rntt        j                  «       |z
  }	|	|kD  rA| j                  «        t        d|› d�«       | j                  D ]  }|j                  «        Œ nt        j                  d	«       �Œ#t        j                  «       |z
  }
| j!                  ||
«       | j"                  j%                  «       D ]  }|j'                  «        Œ y # | j"                  j%                  «       D ]  }|j'                  «        Œ w xY w)
NFTúProcess z terminated with exit code z", terminating remaining processes.c              3   ó8   K  — | ]  }|j                   d u–— Œ y ­wr”   )rÏ  )r.  r	  s     rZ   r/  z7MultiProcessTestCase._join_processes.<locals>.<genexpr>…  s   è ø€ ÒF°!�q—z‘z¨Ô-ÑFùs   ‚zTiming out after z" seconds and killing subprocesses.gš™™™™™¹?)r…  r  rÉ  rS  r  rÏ  rÕ  rK  ÚprintrŸ   r  Úactive_childrenr  ÚallrY  ÚsleepÚ_check_return_codesr  r�  rL  )rØ  rå   rn  rÑ  Úsubprocess_errorr¤  r	  r^  ÚacrÒ  Úelapsed_timerX  s               rZ   rã  z$MultiProcessTestCase._join_processeso  s®  € Ü˜dŸg™g›iÓ(ˆÜ—Y‘Y“[ˆ
Ø Ðð&	Øä% d§n¡nÓ5ò ‘D�A�qð —z‘zÔ%9×%NÑ%NÓNÜØ& q cÐ)DÀQÇZÁZÀLÐPrÐsôô +0×*?Ñ*?×*OÑ*OÓ*Q˜Ø"1ò +˜BØŸL™L�Nð+à+/Ð(Ùðñ $ØäÑF°t·~±~ÔFÔFØäŸ)™)›+¨
Ñ2�Ø˜WÒ$Ø×8Ñ8Ô:ÜØ+¨G¨9Ð4VÐWôð "Ÿ^™^ò &˜ØŸ™�ð&àä—
‘
˜3”ñ= ô@  Ÿ9™9›;¨Ñ3ˆLØ×$Ñ$ R¨Ô6ð ×(Ñ(×/Ñ/Ó1ò �Ø—
‘
•ñø˜×(Ñ(×/Ñ/Ó1ò �Ø—
‘
•ñús   ±9F. Á+DF. Æ.1Gc           
      ó  — | j                   st        j                  d«       y| j                   d   }t        | j                   «      D ��cg c]&  \  }}|j                  t
        j                  k(  r||f‘Œ( }}}|r[d}|D ]I  \  }}| j                  |j                     j                  «       }	|d|› dt
        j                  › d|	› d�z  }ŒK t        |«      ‚t        | j                   «      D ]#  \  }}|j                  �Œt        d|› d	|› d
�«      ‚ || j                  v ryt        j                  «       D ]q  }
|j                  |
j                  k(  sŒt        r1t        j!                  d| j#                  «       |
j$                  «        yt'        j(                  |
j$                  «      ‚ d}|| j*                  v r| j*                  |   }| j-                  |j                  |d|› d|j                  › d|j                  › �¬«       yc c}}w )z¦
        Checks that the return codes of all spawned processes match, and skips
        tests if they returned a return code indicating a skipping condition.
        z<Note: no subprocesses were spawned, test was likely skipped.Nr   Ú r[  z exited with error code z and exception:
ú
ú terminated or timed out after ú secondsú6Skipping %s on sandcastle for the following reason: %szExpected exit code z	 but got z
 for pid: )rÌ   )r  r  ÚwarningrS  rÏ  rÕ  rK  r  r  r-  rÎ  r  rž   r�  rb   r   r  r  rc   rØ   rH  r   ÚassertEqual)rØ  rå   rd  Úfirst_processr¤  r	  Úerrored_processesrï   r¿  Úerror_messageÚskipÚexpected_return_codes               rZ   ra  z(MultiProcessTestCase._check_return_codes›  s#  € ð �~Š~Ü�N‰NØNôð àŸ™ qÑ)ˆô " $§.¡.Ó1÷
á��1Ø�z‰zÔ1×FÑFÒFð �ŠFð
Ðñ 
ñ
 ØˆEØ/ò ‘
��7à $× 0Ñ 0°·±Ñ =× BÑ BÓ D�ØØ˜q˜cÐ!9Ô:N×:cÑ:cÐ9dð e'Ø'4 o°Rð9ñ‘ðô ˜uÓ%Ð%ô ˜dŸn™nÓ-ò 	‰DˆAˆqØ�z‰zÑ!Ü"Ø˜q˜cÐ!@ÀÀÈhÐWóð ð	ð �×-Ñ-Ñ-Øä×%Ñ%Ó'ò 	:ˆDØ×%Ñ%¨¯©Ó7Ý ô
 —K‘KØPØŸ™›	ØŸ™ôñ
 ä"×+Ñ+¨D¯L©LÓ9Ð9ð	:ð"  !Ðð �×0Ñ0Ñ0Ø#'×#BÑ#BÀ2Ñ#FÐ à×ÑØ×"Ñ"Ø Ø%Ð&:Ð%;¸9À]×E[ÑE[ÐD\Ð\fÐgt×gxÑgxÐfyÐzð 	õ 	
ùóg
s   Á
+Hc                 ó    — | j                   dk(  S )Nr   ©r/   r×  s    rZ   rm  zMultiProcessTestCase.is_masterç  s   € à�y‰y˜A‰~Ðrj   ©rï  rï  ©r4   N)rd   re   rf   râ  rK  ÚboolrÙ  ÚpropertyrÛ  rg   r0   ré  rF   ró  rÿ  r  rß   r  r"  r   r&  Ústaticmethodr9  Úclassmethodr  r;  rY  rã  ra  rm  Ú__classcell__©r÷  s   @rZ   rÕ  rÕ  j  s8  ø„ ØÐð
 Ðð¨ó ð ð dò ó ðð ð"˜Cò "ó ð"ò/ð ?HñØðØ8;ðà	õõ&õ$ð( Có (ó+ó,$ô�ô ð ð¸ò ó ðð< ð.Øð.Ø#&ð.Ø36ð.à	ò.ó ð.ð7 #ð 7°tó 7ór&óP*óXJ
ðX ð˜4ò ó ôrj   rÕ  c                   ó>   ‡ — e Zd Zˆ fd„Zd„ Zdefd„Zdd„Zd„ Zˆ xZ	S )ÚDistributedTestBasec                 óŽ   •— t         ‰| �  «        t        | j                  «      t        j
                  d<   | j                  «        y )Nrš   )rò  rÿ  rF   r0   rG   rH   r"  ©rØ  r÷  s    €rZ   rÿ  zDistributedTestBase.setUpò  s/   ø€ Ü‰‰ŒÜ#& t§¡Ó#7Œ�
‰
�<Ñ Ø×ÑÕrj   c                 ó¾   — 	 t         j                  j                  «        	 t	        j
                  | j                  «       y # t        $ r Y Œ,w xY w# t        $ r Y y w xY wr”   )rŸ   ÚdistributedrM  rò   rG   Úremover  ÚOSErrorr×  s    rZ   r  zDistributedTestBase.tearDown÷  sU   € ð	Ü×Ñ×3Ñ3Ô5ð	Ü�I‰I�d—n‘nÕ%øô ò 	Ùð	ûô ò 	Ùð	ús"   ‚A ¡A Á	AÁAÁ	AÁAr4   c                 ó"   — d|v ryd|v ryd|v ryy©Nr^   r(   r�   r]   r_   r&   r$   ri   )rØ  r1   s     rZ   r.   zDistributedTestBase.backend  s$   € Ø�VÑØØ�f‰_ØØ�f‰_Øàrj   c                 ó  — |€| j                   }t        j                  |«      j                  «       }t        j                  j                  | j                  |«      }t        j                  j                  | j                  |«      || j                  |¬«       d| j                  |«      v sd| j                  |«      v r£t        j                  j                  «       }|re|j                  }t        j                  |› d| j                  › �«      }t        j                  |«       t        j                  j                  |«       nt!        d| j                  |«      › d�«      ‚t        j                  j"                  j%                  «       S )N©r.   r0   r/   r>   r(   r&   rH  úEExpected to find an accelerator when initializing process group with ú backend, but got None)r0   rŸ   Úget_device_moduler    r�  rE   r  Úinit_process_groupr.   r/   ÚacceleratorÚcurrent_acceleratorr_  r1   Úset_default_deviceÚset_device_indexrÎ  Údistributed_c10dÚ_get_default_group)rØ  r1   r0   Únum_visible_devicesr>   rŒ  Údevice_types          rZ   Ú	create_pgzDistributedTestBase.create_pg  sE  € ØÐØŸ™ˆJÜ#×5Ñ5°fÓ=×JÑJÓLÐÜ×!Ñ!×+Ñ+¨D¯N©NÐ<OÓPˆÜ×Ñ×,Ñ,Ø—L‘L Ó(Ø!Ø—‘Øð	 	-ô 	
ð �T—\‘\ &Ó)Ñ)¨V°t·|±|ÀFÓ7KÑ-KÜ×+Ñ+×?Ñ?ÓAˆKÙØ)×.Ñ.�ÜŸ™¨¨°Q°t·y±y°kÐ&BÓC�Ü×(Ñ(¨Ô0Ü×!Ñ!×2Ñ2°6Õ:ä"ðØ!Ÿ\™\¨&Ó1Ð2Ð2HðJóð ô × Ñ ×1Ñ1×DÑDÓFÐFrj   c                 ó¢   — t        j                  |«      j                  «       }t        | j                  «      D �ci c]	  }|||z  g“Œ c}S c c}w r”   ©rŸ   rŠ  r    rU   r0   ©rØ  r1   r’  r¤  s       rZ   Úrank_to_devicez"DistributedTestBase.rank_to_device$  óG   € Ü#×5Ñ5°fÓ=×JÑJÓLÐÜ6;¸D¿O¹OÓ6LÖM°��AÐ+Ñ+Ð,Ñ,ÒMÐMùÒMó   »Ar”   )
rd   re   rf   rÿ  r  rF   r.   r”  r˜  rz  r{  s   @rZ   r}  r}  ñ  s%   ø„ ô ò
ð ó óGö2Nrj   r}  Úsubtest_configÚtest_fnÚtest_kwargsc                 óð  — t        |j                  «       «      }|D �cg c]  }|d   ‘Œ	 }}|D �cg c]  }|d   ‘Œ	 }}t        j                  |Ž D ]Ž  }	t	        t        ||	d¬«      «      }
 | j                  di |
¤Ž5  t        j                  j                  «         ||i |¤|
¤Ž t        j                  j                  «        ddd«       t        j                  «        Œ� yc c}w c c}w # 1 sw Y   Œ*xY w)a\  
    Runs a test function given by ``test_fn`` as a subtest according to the
    configurations specified by ``subtest_config``. This amortizes the
    costly setup overhead (including process spawn and initializing the
    process group) over the subtests.

    Args:
        subtest_config (Dict[str, List[Any]]): A mapping from subtest
            keyword argument name to a list of its possible values.
        test_fn (Callable): A callable that runs the actual test.
        test_args: Positional arguments to pass to ``test_fn``.
        test_kwargs: Keyword arguments to pass to ``test_fn``.
    r   r³   T)ÚstrictNri   )r§  ÚitemsÚ	itertoolsÚproductÚdictÚzipÚsubTestrŸ   Ú_dynamoÚresetrD   r  )Úcls_instr›  rœ  Ú	test_argsr�  Úsubtest_config_itemsÚitemÚsubtest_config_keysÚsubtest_config_valuesr�  Úsubtest_kwargss              rZ   Úrun_subtestsr¯  )  så   € ô* 9=¸^×=QÑ=QÓ=SÓ8TÐØ:NÖ%O°$ d¨1£gÐ%OÐÐ%OØBVÖ-W¸$¨d°1«gÐ-WÐÐ-WÜ×#Ñ#Ð%:Ð;ò ˆäœcÐ"5°vÀdÔKÓLˆØˆX×ÑÑ/ Ñ/ñ 	"Ü�M‰M×ÑÔ!Ù�YÐ@ +Ð@°Ò@Ü�M‰M×ÑÔ!÷	"ô 	�‰�ñùò &PùÚ-W÷	"ð 	"ús   žC"°C'Á<AC,Ã,C5	c                  ón   — 	 t        j                  g d¢d¬«      j                  dk(  S # t        $ r Y yw xY w)a   
    If shell command `fi_info -p efa -t FI_EP_RDM` returns exit code 0 then we assume that the machine has
    Libfabric EFA interfaces and EFA software components installed,
    see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html.
    )Úfi_infoz-pÚefaz-tÚ	FI_EP_RDMF)Úcheckr   )Ú
subprocessÚrunÚ
returncodeÚFileNotFoundErrorri   rj   rZ   Úhas_efar¹  K  sA   € ðä�N‰NÚ;À5ôç‰jØñð	
øô ò ØØðús   ‚%( ¨	4³4c                  ó"   — t        «       rddgS dS )a‹  
    If the machine has Libfabric EFA interfaces and EFA software components installed it may cause
    'RuntimeError: In operator() at tensorpipe/common/ibv.h:172 "": Operation not supported' if tensorpipe
    uses InfiniBand transport, so we exclude it from tensorpipe transports,
    see https://github.com/pytorch/pytorch/issues/73885 and https://github.com/pytorch/pytorch/issues/65022
    ÚshmÚuvN)r¹  ri   rj   rZ   Útp_transportsr½  _  s   € ô $œIˆE�4ˆ=Ð/¨4Ð/rj   c                 ód   ‡ ‡‡— ‰ €t        t        |‰¬«      S d„ Št        ‰ «      ˆˆ ˆfd„«       }|S )z+
    Wrapper to use with a test method
    )rn  r0   c                 óü   ‡ ‡‡‡— t        «       Št        j                  «       }ˆfd„Šˆˆˆ fd„}g }t        ‰ «      D ]=  }t	        j
                  ||‰|f¬«      }|j                  «        |j                  |«       Œ? |S )Nc                  ó>   •— ‰ t         j                  j                  k(  S r”   ©rD   r�  Ú_world©Úworlds   €rZ   Úworld_is_validzaspawn_threads_and_init_comms.<locals>._run_test_method_with_multi_threads.<locals>.world_is_validx  ó   ø€ ØœD×1Ñ1×8Ñ8Ñ8Ð8rj   c                 ó~  •— t        j                  d| ‰|¬«       	  ‰«         ‰«       rt        j                  «        y y # t        $ rR}t        j                  j                  | t        j                  «       f«       t        j                  |«       Y d }~Œsd }~ww xY w#  ‰«       rt        j                  «        w w xY w)NÚthreaded©r.   r/   r0   r>   )rD   r‹  ÚBaseExceptionÚMultiThreadedTestCaseÚexception_queueÚputrœ   Úexc_infor#   Úexception_handlerM  )r/   Úworld_pgr>   ÚexÚcallbackrÅ  r0   s       €€€rZ   ÚworkerzYspawn_threads_and_init_comms.<locals>._run_test_method_with_multi_threads.<locals>.worker{  sž   ø€ Ü×#Ñ#Ø"¨¸*ÈEõð
1Ù”
ñ "Ô#Ü×.Ñ.Õ0ð $øô !ò ä%×5Ñ5×9Ñ9¸4ÄÇÁÃÐ:PÔQÜ!×2Ñ2Ø÷ñ ûðûñ "Ô#Ü×.Ñ.Õ0ð $ús*   œA  Á 	BÁ	ABÂB ÂBÂB ÂB<©r  r¡   )r!   rD   Ú	HashStorerU   rC  rD  r  r—  )	r0   rÒ  Úglobal_storerÓ  Úthreadsr/   ÚtrÄ  rÅ  s	   ``     @@rZ   Ú#_run_test_method_with_multi_threadszIspawn_threads_and_init_comms.<locals>._run_test_method_with_multi_threadst  sq   û€ Ü$Ó&ˆÜ—~‘~Ó'ˆô	9ö	1ð  ˆÜ˜*Ó%ò 	ˆDÜ× Ñ ¨°d¸EÀ<Ð5PÔQˆAØ�G‰GŒIØ�N‰N˜1Õð	ð
 ˆrj   c                 óX  •‡ ‡‡— t         j                  j                  j                  d«       	  ‰‰ˆˆˆˆ fd„«      }t        j                  |‰«       t         j                  j                  j                  d«       y # t         j                  j                  j                  d«       w xY w)NTc                  ó   •—  ‰‰g‰ ¢­i ‰¤ŽS r”   ri   )r¡   r£   r¢   rØ  s   €€€€rZ   r  z?spawn_threads_and_init_comms.<locals>.wrapper.<locals>.<lambda>™  s   ø€ ¡D¨Ð$?°Ò$?¸Ñ$?€ rj   F)rŸ   rE  Ú_distributed_c10dÚ_set_thread_isolation_moderË  Ú_join_threads)rØ  r¡   r¢   r×  rÙ  r£   r0   s   ``` €€€rZ   r¤   z-spawn_threads_and_init_comms.<locals>.wrapper“  sv   û€ ô 	�‰×"Ñ"×=Ñ=¸dÔCð	IÙ9ØÖ?óˆGô "×/Ñ/°¸Ô>ä�H‰H×&Ñ&×AÑAÀ%ÕHøŒE�H‰H×&Ñ&×AÑAÀ%ÕHús   ¯%A> Á>+B))r   Úspawn_threads_and_init_commsr
   )r£   rn  r0   r¤   rÙ  s   ` ` @rZ   rß  rß  i  sD   ú€ ð €|ÜÜ(°'Àjô
ð 	
òô> ˆ4ƒ[õ
Ió ð
Ið €Nrj   c                   ó  ‡ — e Zd ZdZ ej
                  «       ZdZd„ Z	 dde	de	ddfˆ fd„Z
d	„ Zd
„ Zdˆ fd„Zˆ fd„Zd„ Zed„ «       Zd„ Zed„ «       Zed„ «       Zedefd„«       Zede	fd„«       Zdddœd„Zdddœd„Zˆ xZS )rË  a5  
    Test runner that runs all tests with the in-proc process group using
    multiple threads with the threaded process group.

    Each test spawns world_size threads and run the test method in each thread.

    Difference from regular MultiProcess test runner:
    Must explicitly defines SetUp and call self._spawn_threads() to run the tests.
    Cannot use setUp / tearDown (must use perThreadSetup / perThreadShutdown)
        to set up / tear down each thread when running each test.
    No global state possible
        How bad of a limitation is this?
    rð   c                 óV   ‡— t        ‰«      ˆfd„«       }t        j                  || «      S )Nc                 ó€   •— | j                   | j                  k(  r| j                  | j                  ‰«       y  ‰«        y r”   )r/   ÚMAIN_THREAD_RANKrÞ  r×  rä  s    €rZ   r¤   z2MultiThreadedTestCase.join_or_run.<locals>.wrapper·  s.   ø€ à�y‰y˜D×1Ñ1Ò1Ø×"Ñ" 4§<¡<°Õ4á•rj   rå  rè  s    ` rZ   ré  z!MultiThreadedTestCase.join_or_run¶  rê  rj   rë  rì  r4   Nc                 óò   •— |dk7  r|}t         ‰| �  |«       	 t        | |«      }t        | || j	                  |«      «       y # t
        $ r+}|dk7  rt        d| j                  › d|› �«      |‚Y d }~y d }~ww xY wrî  rñ  rø  s        €rZ   ró  zMultiThreadedTestCase.__init__À  rú  rû  c                  ó   — y r”   ri   r×  s    rZ   ÚperThreadSetUpz$MultiThreadedTestCase.perThreadSetUpÓ  s   € àrj   c                  ó   — y r”   ri   r×  s    rZ   ÚperThreadTearDownz'MultiThreadedTestCase.perThreadTearDown×  s   € Ørj   c                 óx   •— t         ‰| �  «        | j                  | _        g | _        dt
        j                  d<   y)z‘
        setUp only set up things in the main thread, if you want to configure things
        in the spawned threads, use perThreadSetUp
        rÿ   rB  N)rò  rÿ  rã  r/   r×  rG   rH   r  s    €rZ   rÿ  zMultiThreadedTestCase.setUpÚ  s1   ø€ ô
 	‰‰ŒØ×)Ñ)ˆŒ	ØˆŒà36Œ�
‰
Ð/Ò0rj   c                 ó0   •— t         ‰| �  «        g | _        y)z—
        tearDown only set up things in the main thread, if you want to configure things
        in the spawned threads, use perThreadTearDown
        N)rò  r  r×  r  s    €rZ   r  zMultiThreadedTestCase.tearDownå  s   ø€ ô
 	‰ÑÔØˆ�rj   c                 óò  ‡— t         j                  j                  j                  d«       | j                  }t        «       Št        j                  «       | j                  _	        ˆfd„} |«       st        d«      ‚t        | j                  «      D ]e  }t        j                  | j                  j                  ||| j                  f¬«      }|j!                  «        | j"                  j%                  |«       Œg y)zk
        class method to spawn threads and run test, use this method in the SetUp of your TestCase
        Tc                  ó>   •— ‰ t         j                  j                  k(  S r”   rÁ  rÃ  s   €rZ   rÅ  z<MultiThreadedTestCase._spawn_threads.<locals>.world_is_valid÷  rÆ  rj   zInvalid worldrÔ  N)rŸ   rE  rÜ  rÝ  rß   r!   rD   rÕ  r÷  rÖ  rÎ  rU   r0   rC  rD  r  r  r×  r—  )rØ  rä   rÅ  r/   rØ  rÄ  s        @rZ   Ú_spawn_threadsz$MultiThreadedTestCase._spawn_threadsí  sº   ø€ ô 	�‰×"Ñ"×=Ñ=¸dÔCØ×+Ñ+ˆ	ä$Ó&ˆÜ&*§n¡nÓ&6ˆ�‰Ô#ô	9ñ ÔÜ˜Ó/Ð/ä˜$Ÿ/™/Ó*ò 	#ˆDÜ× Ñ Ø—~‘~×*Ñ*°)¸TÀ4Ç?Á?Ð1SôˆAð �G‰GŒIØ�L‰L×Ñ Õ"ñ	#rj   c                 ó  —  | |«      }||_         t        |d«      rWt        j                  «       |_        t
        j                  |j                  _        t
        j                  |j                  _	        |j                  |||«       y )NÚ_tls)r/   rà   rC  Úlocalrï  r    Ú
_precisionÚ	precisionÚ_rel_tolÚrel_tolÚrun_test_with_threaded_pg)r=  rä   r/   r0   r¢   rØ  s         rZ   r  zMultiThreadedTestCase._run  sb   € á�9‹~ˆØˆŒ	ô �4˜Ô Ü!Ÿ™Ó)ˆDŒIÜ"*×"5Ñ"5ˆD�I‰IÔÜ (× 1Ñ 1ˆD�I‰IÔà×&Ñ& y°$¸
ÕCrj   c                 óð  — t        j                  d||| j                  j                  ¬«       | j	                  «        	  t        | |«      «        t        j                  «        | j                  «        y# t        $ rN}| j                  j                  |t        j                  «       f«       t        j                  |«       Y d}~Œwd}~ww xY w# t        j                  «        | j                  «        w xY w)zd
        Run the current test associated with `test_name` using the threaded process group.
        rÈ  rÉ  N)rD   r‹  r÷  rÖ  ræ  rÊ   rÊ  rÌ  rÍ  rœ   rÎ  r#   rÏ  rM  rè  )rØ  rä   r/   r0   rÑ  s        rZ   rõ  z/MultiThreadedTestCase.run_test_with_threaded_pg  sÆ   € ô 	×ÑØØØ!Ø—.‘.×-Ñ-õ		
ð 	×ÑÔð		%Ø$ŒG�D˜)Ó$Ô&ô ×&Ñ&Ô(Ø×"Ñ"Õ$øô ò 	Ø× Ñ ×$Ñ$ d¬C¯L©L«NÐ%;Ô<Ü×.Ñ.Ø÷ñ ûð	ûô ×&Ñ&Ô(Ø×"Ñ"Õ$ús*   ¿A5 Á5	CÁ>ACÃC ÃCÃC Ã&C5c           
      óâ  — t         }	 t        |«      D ]f  \  }}|j                  t        d|«      «       |j	                  «       sŒ2t
        j                  j                  |t        t        d|› d�«      d ff«       Œh t        j                  «        g }| j                  j                  «       sF| j                  j                  «       }|j                  |«       | j                  j                  «       sŒFt        «        t        j                   j"                  j%                  d«       | j'                  |||«       y # t        «        t        j                   j"                  j%                  d«       w xY w)Nr   zRank failed to join in under ri  F)rƒ  rS  r¹  rÇ  rÌ  rË  rÌ  rÍ  ÚTimeoutErrorr#   r§  rÍ  r	  r—  r"   rŸ   rE  rÜ  rÝ  ra  )r=  r×  rå   rn  ÚidxÚthreadÚfailed_ranksÚfailures           rZ   rÞ  z#MultiThreadedTestCase._join_threads+  s-  € ä!ˆð	IÜ(¨Ó1ò ‘��VØ—‘œC  7›OÔ,Ø—?‘?Õ$Ü)×9Ñ9×=Ñ=àä ,Ü ,Ø&CÀGÀ9ÈHÐ$Uó!"ð !%ðð	õðô ×#Ñ#Ô%ØˆLØ×)Ñ)×/Ñ/Ô1Ø×-Ñ-×1Ñ1Ó3�Ø×#Ñ# GÔ,ð ×)Ñ)×/Ñ/Õ1ô #Ô$Ü�H‰H×&Ñ&×AÑAÀ%ÔHà×Ñ ¨g°rÕ:øô #Ô$Ü�H‰H×&Ñ&×AÑAÀ%ÕHús   ˆ<D9 ÁB,D9 Ä95E.c                 ó‚  — d}d}|D �]&  \  }}|d   }t        |t        j                  «      r2t        j	                  d|||«       |dk  sŒDt
        d   j                  }ŒXt        |t        «      r)d|› d|› d	�}	t        j                  |	«       t        |	«      ‚t        |t        «      rEdj                  t        j                  |Ž «      }	t        j                  d
|	|«       |d|› d|	› d�z  }Œæt        |t        «      sŒ÷t        |j                   «      t"        u s�Œ|dk  s�Œ|j                   }�Œ) t%        |«      dkD  rt        |«      ‚|dkD  rqt
        j'                  «       D ]Y  }
||
j                  k(  sŒt(        r#t        j	                  d||
j*                  «        y t        j                  |
j*                  «      ‚ y y )Nrf  rð   r³   z3Thread %s skipping test %s for following reason: %sr   r€   zThread rh  z	 seconds
z'Caught exception: 
%s exiting thread %sz exited with exception:
rg  rj  )Ú
isinstancerØ   rH  r  r  rž   rb   rø  rï   rÎ  rã   r¹  rI  Úformat_exceptionÚ
SystemExitr_  Úcoderg   rË   r�  r   rc   )r=  rû  rn  rå   Ú	error_msgÚ	skip_coder/   rÎ  ÚexcrÌ   rp  s              rZ   ra  z)MultiThreadedTestCase._check_return_codesI  s—  € ð ˆ	Øˆ	Ø*ó 	)‰NˆD�(Ø˜1‘+ˆCÜ˜#œx×0Ñ0Ô1Ü—‘ØIØØØô	ð ˜q“=Ü *¨9Ñ 5× ?Ñ ?‘IÜ˜C¤Ô.Ø ˜vÐ%DÀWÀIÈZÐX�Ü—‘˜SÔ!Ü" 3Ó'Ð'Ü˜C¤Ô+Ø—g‘gœi×8Ñ8¸(ÐCÓD�Ü—‘ÐGÈÈdÔSØ˜w t fÐ,EÀcÀUÈ"ÐMÑM‘	Ü˜C¤Õ,Ü˜Ÿ™“>¤SÓ(¨Y¸¬]Ø #§¡’Ið+	)ô0 ˆy‹>˜AÒÜ˜yÓ)Ð)à�qŠ=Ü"×)Ñ)Ó+ò >�Ø §¡Ó.Ý$äŸ™ØTØØ ŸL™Lôñ
 ä&×/Ñ/°·±Ó=Ð=ñ>ð rj   c                 ó   — t         S r”   rÝ  r×  s    rZ   r0   z MultiThreadedTestCase.world_size{  rß  rj   c                 óF   — | j                  «       j                  d«      d   S r�  r  r×  s    rZ   rß   z(MultiThreadedTestCase._current_test_name  s   € ð �w‰w‹y�‰˜sÓ# BÑ'Ð'rj   r   rs  c                óJ   — | j                   |k(  r| j                  |||«       yy)zÁ
        The reason why we have this util function instead of
        self.assertEqual is all threads are sharing one CPU RNG
        so the assertion result is only reliable on rank 0
        N)r/   rl  ©rØ  rÆ   ÚyrÌ   r/   s        rZ   ÚassertEqualOnRankz'MultiThreadedTestCase.assertEqualOnRank„  s'   € ð �9‰9˜ÒØ×Ñ˜Q  3Õ'ð rj   c                óH   — | j                   |k(  r| j                  ||«       y y r”   )r/   ÚassertNotEqualr  s        rZ   ÚassertNotEqualOnRankz*MultiThreadedTestCase.assertNotEqualOnRank�  s#   € Ø�9‰9˜ÒØ×Ñ  1Õ%ð rj   rt  ru  r”   )rd   re   rf   Ú__doc__rÊ  ÚQueuerÌ  rã  ré  rF   ró  ræ  rè  rÿ  r  rí  ry  r  rõ  rÞ  ra  rw  rg   r0   rß   r
  r  rz  r{  s   @rZ   rË  rË  £  sõ   ø„ ñð "�e—k‘k“m€OàÐò/ð ?HñØðØ8;ðà	õò&òõ	7ôò#ð. ñDó ðDò%ð. ñ;ó ð;ð: ñ/>ó ð/>ðb ð"˜Cò "ó ð"ð ð( Cò (ó ð(ð(¸ô (ð&¸1÷ &ð &rj   rË  c                   ó–   ‡ — e Zd Zdeej
                  ej                  f   deddfˆ fd„Z	dej                  dej                  fd„Z
ˆ xZS )ÚSaveForwardInputsModuleÚforward_inputsÚcast_forward_inputsr4   Nc                 ót   •— t         ‰| �  «        t        j                  dd«      | _        || _        || _        y )Néd   )rò  ró  ÚnnÚLinearÚlr  r  ©rØ  r  r  r÷  s      €rZ   ró  z SaveForwardInputsModule.__init__“  s2   ø€ ô
 	‰ÑÔÜ—‘˜3 Ó$ˆŒØ,ˆÔØ#6ˆÕ rj   rÆ   c                 óÀ   — || j                   | <   | j                  | j                  r3|j                  | j                  j                  j
                  «      «      S |«      S r”   )r  r  r  ÚtoÚweightÚdtype©rØ  rÆ   s     rZ   ÚforwardzSaveForwardInputsModule.forward�  sI   € Ø$%ˆ×Ñ˜DÑ!Ø�v‰v°4×3KÒ3K�a—d‘d˜4Ÿ6™6Ÿ=™=×.Ñ.Ó/ÓSÐSÐQRÓSÐSrj   ©rd   re   rf   r£  r  ÚModulerŸ   ÚTensorrv  ró  r  rz  r{  s   @rZ   r  r  ’  sT   ø„ ð7à˜RŸY™Y¨¯©Ð4Ñ5ð7ð "ð7ð 
õ	7ðT˜Ÿ™ð T¨%¯,©,÷ Trj   r  c                   ó–   ‡ — e Zd Zdeej
                  ej                  f   deddfˆ fd„Z	dej                  dej                  fd„Z
ˆ xZS )ÚSaveForwardInputsModelr  r  r4   Nc                 ót   •— t         ‰| �  «        t        ||«      | _        t        ||«      | _        || _        y r”   )rò  ró  r  Úc1Úc2r  r  s      €rZ   ró  zSaveForwardInputsModel.__init__£  s6   ø€ ô
 	‰ÑÔÜ)¨.Ð:MÓNˆŒÜ)¨.Ð:MÓNˆŒØ,ˆÕrj   rÆ   c                 ó`   — || j                   | <   | j                  | j                  |«      «      S r”   )r  r'  r&  r  s     rZ   r  zSaveForwardInputsModel.forward­  s)   € Ø$%ˆ×Ñ˜DÑ!Ø�w‰w�t—w‘w˜q“zÓ"Ð"rj   r   r{  s   @rZ   r$  r$  ¢  sQ   ø„ ð-à˜RŸY™Y¨¯©Ð4Ñ5ð-ð "ð-ð 
õ	-ð#˜Ÿ™ð #¨%¯,©,÷ #rj   r$  c              #   óX  K  — |st         j                  j                  | «       t         j                  j                  «       x}r|j                  nd}|€t        j                  |«      }dt        j                  d<   dt        j                  d<   |rp|rVt         j                  j                  j                  j                  j                  «       }t        j                  d|| |¬«       nt        j                  || |¬«       t         j                  j!                  «        t         j                  j"                  j$                  j'                  «        	 d –— t         j                  j!                  «        t         j                  j"                  j$                  j'                  «        |rt        j(                  «        y y # t         j                  j!                  «        t         j                  j"                  j$                  j'                  «        |rt        j(                  «        w w xY w­w)	NÚcpurb  ÚMASTER_ADDRÚ6789ÚMASTER_PORTÚfaker‡  )r.   r/   r0   )rŸ   rŒ  r�  r�  r_  rD   Úget_default_backend_for_devicerG   rH   ÚtestingÚ	_internalr�  r  Ú	FakeStorer‹  r¦  r§  ÚutilsÚcountersÚclearrM  )r/   r0   r.   Úinit_pgr  Úaccr“  r>   s           rZ   Ú_dynamo_dist_per_rank_initr8  ²  sŠ  è ø€ ñ Ü×Ñ×*Ñ*¨4Ô0ô "×-Ñ-×AÑAÓCÐC�SÐCˆ�ŠÈ%ð ð €Ü×5Ñ5°kÓBˆà +„B‡J�Jˆ}ÑØ &„B‡J�Jˆ}ÑÙÙÜ—M‘M×+Ñ+×7Ñ7×?Ñ?×IÑIÓKˆEÜ×#Ñ#ØØ%ØØö	ô ×#Ñ#¨G¸$È:ÕVÜ	‡M�M×ÑÔÜ	‡M�M×Ñ× Ñ ×&Ñ&Ô(ð)Ûä�‰×ÑÔÜ�‰×Ñ×$Ñ$×*Ñ*Ô,ÙÜ×&Ñ&Õ(ð øô 	�‰×ÑÔÜ�‰×Ñ×$Ñ$×*Ñ*Ô,ÙÜ×&Ñ&Õ(ð üs    ‚EH*ÅF> ÅA(H*Æ>A)H'È'H*c                   ó@   ‡ — e Zd ZdZeˆ fd„«       Zeˆ fd„«       Zˆ xZS )Ú#DynamoDistributedSingleProcTestCasez£
    Test harness for single-process dynamo distributed tests,
    initializes dist process group.

    Prefer this for simple tests, as it's easier to debug.
    c                 óÚ  •— t         ‰| �  «        | j                  j                  t	        j
                  t        j                  dddœ«      «       d| _        t        j                  j                  «       j                  }|› d| j                  › �| _        || j                  v rd n| j                  g| _        t        j                   t        j"                  |«      | j                  d¬«       y )Nrb  Ú12355)r+  r-  r   rH  r³   )r/   r0   )rò  Ú
setUpClassÚ_exit_stackÚenter_contextr   r£  rG   rH   r/   rŸ   rŒ  r�  r_  r1   Ú
device_idsrD   r‹  r/  )r=  r1   r÷  s     €rZ   r=  z.DynamoDistributedSingleProcTestCase.setUpClassá  s¶   ø€ ä‰ÑÔà�‰×%Ñ%Ü�J‰JÜ—
‘
à#.Ø#*ñóô	
ð ˆŒÜ×"Ñ"×6Ñ6Ó8×=Ñ=ˆØ�x˜q §¡ 
Ð+ˆŒ
Ø!'¨3¯:©:Ñ!5™¸C¿H¹H¸:ˆŒÜ×ÑÜ×/Ñ/°Ó7¸c¿h¹hÐSTö	
rj   c                 óJ   •— t        j                  «        t        ‰| �  «        y r”   )rD   rM  rò  ÚtearDownClass©r=  r÷  s    €rZ   rB  z1DynamoDistributedSingleProcTestCase.tearDownClassö  s   ø€ ä×"Ñ"Ô$Ü‰ÑÕrj   )rd   re   rf   r  ry  r=  rB  rz  r{  s   @rZ   r:  r:  Ù  s0   ø„ ñð ó
ó ð
ð( ó ó ô rj   r:  c            	       óH   — e Zd ZdZedefd„«       Zededededdfd„«       Z	y)	Ú"DynamoDistributedMultiProcTestCasea   
    Use this for tests that actually run on multiple GPUs.

    Decorate tests with @skip_if_lt_x_gpu(ngpu)

    Note: MultiProcTestCase spawns processes per test and is slow.
    Prefer MultiThreadedTestCase for most tests. Perhaps use this one
    sparingly for integration tests.
    r4   c                 ó>   — t         j                  j                  «       S r”   )rŸ   rŒ  r    r×  s    rZ   r0   z-DynamoDistributedMultiProcTestCase.world_size  s   € ä× Ñ ×-Ñ-Ó/Ð/rj   r/   rä   r  Nc                 ó¢   — t        j                  t        j                  «       «        | |«      }||_        ||_        |j                  ||«       y r”   )r   Ú
addHandlerÚloggingÚNullHandlerr/   r  r;  r<  s          rZ   r  z'DynamoDistributedMultiProcTestCase._run  sB   € ô 	×ÑœW×0Ñ0Ó2Ô3ñ �9‹~ˆØˆŒ	Ø"ˆŒØ�‰�i Õ-rj   )
rd   re   rf   r  rw  rg   r0   ry  rF   r  ri   rj   rZ   rE  rE  ü  sV   „ ñð ð0˜Cò 0ó ð0ð ð	.Øð	.Ø#&ð	.Ø36ð	.à	ò	.ó ñ	.rj   rE  c                   ó’  ‡ — e Zd ZU dZdZeed<   dZeed<   dZe	dz  ed<    e
d¬«      Ze
ed	<   d
Zeed<   d
Zeed<   ede	dz  fd„«       Zede	fd„«       Zed d„«       Zed„ «       Zede	ddfd„«       Zed„ «       Zed!d„«       Zede	defd„«       Zeˆ fd„«       Zed„ «       Zeˆ fd„«       Zd!ˆ fd„Zd„ Z	 d"de	de	ddfˆ fd„Zˆ xZS )#ÚMultiProcContinuousTestrð   éþÿÿÿr0   r/   NÚ	rdvz_filerÂ  )Úsecondsrn  FÚpoison_pillÚ_processes_spawnedr4   c                  ó   — y)z¤
        ProcessGroup backend str.
        To be customized by sub test classes, e.g. "nccl".
        Otherwise we return None -- lazily decided by tensor.
        Nri   )r=  s    rZ   Úbackend_strz#MultiProcContinuousTest.backend_str(  ó   € ð rj   c                 ó\   — t         j                  j                  «       }|€y|j                  S )Nr*  )rŸ   rŒ  r�  r_  )r=  Úcurr_devices     rZ   r“  z#MultiProcContinuousTest.device_type2  s+   € ä×'Ñ'×;Ñ;Ó=ˆØÐØØ×ÑÐrj   c                  ó   — y)z“
        ProcessGroup init options.
        To be customized by sub test classes, e.g. ProcessGroupNCCLOpTest
        Here we return None.
        Nri   )r=  Úhigh_priority_streams     rZ   ÚoptszMultiProcContinuousTest.opts9  rT  rj   c                 óJ  — |€t        d«      ‚t        |«      t        j                  d<   t	        j
                  ||«      }t	        j                  | j                  «       |||| j                  «       | j                  ¬«       t        j                  j                  «       | _        y )Nz!Expected rdvz_file to not be NoneÚ
LOCAL_RANK)r.   r0   r/   r>   Ú
pg_optionsrn  )rò   rF   rG   rH   rD   rE   r‹  rS  rY  rn  r�  r‘  rX   )r=  r/   r0   rN  r>   s        rZ   Ú_init_pgz MultiProcContinuousTest._init_pgB  s�   € àÐÜ Ð!DÓEÐEô $' t£9Œ�
‰
�<Ñ Ü—‘˜y¨*Ó5ˆä×ÑØ—O‘OÓ%Ø!ØØØ—x‘x“zØ—K‘Kõ	
ô ×&Ñ&×9Ñ9Ó;ˆ�rj   r„  c                 óÔ   — |j                  dd¬«      d   } | |«      }| j                  |_        | j                  |_        t        ||«      }t	        j
                  «         |di |¤Ž y )NrR  r³   rO  rð   ri   )Úrsplitr/   r0   rÊ   r   rG  )r=  r„  r¢   rä   rØ  rœ  s         rZ   Ú_run_test_given_idz*MultiProcContinuousTest._run_test_given_idV  sa   € ð —N‘N 3°�NÓ3°BÑ7ˆ	á�9‹~ˆØ—H‘HˆŒ	ØŸ.™.ˆŒÜ˜$ 	Ó*ˆô 	×!Ñ!Ô#ñ 	Ñ�&Órj   c                 ó�  ‡— d}d|cxk  r|k  sn t        d|› d|› �«      ‚|| _        || _        d }	 | j                  |||«       t        j                  d«       	 |j                  «       }
t        j                  d	|
› �«       |
€nK|�%|j                  t        j                  |«      «       ŒS	 | j!                  |
«       |j                  |
«       Œvt        j                  d«       |st5        j6                  «        y y # t        $ rO}t        |dd «      Št        ˆfd„t        j                  «       D «       d «      }	|	r|	j                  }n‚ Y d }~�Œd }~ww xY w# t"        $ râ}t%        |t        «      rjt        |dd «      Št        ˆfd
„t        j                  «       D «       d «      }	|	r4|j                  t        j                  |	j                  «      «       Y d }~�Œ‚d}t'        j(                  «       }dj+                  t-        j.                  |Ž «      }t1        d|› �«      }||_        |j                  |«       Y d }~�Œod }~ww xY w)NFr   z*Expected 0 <= rank < world_size, got rank=z, world_size=r  c              3   óB   •K  — | ]  }|j                   ‰k(  sŒ|–— Œ y ­wr”   ©rb   ©r.  Úvrb   s     €rZ   r/  z7MultiProcContinuousTest._worker_loop.<locals>.<genexpr>y  s   øè ø€ ÒL�q°1·;±;À)Ó3K”ÑLùó   ƒ˜zSetup completeTz	Got test c              3   óB   •K  — | ]  }|j                   ‰k(  sŒ|–— Œ y ­wr”   rc  rd  s     €rZ   r/  z7MultiProcContinuousTest._worker_loop.<locals>.<genexpr>œ  s   øè ø€ ÒT˜q¸1¿;¹;È)Ó;SœÑTùrf  rf  zException in worker process:
zTerminating ...)rò   r/   r0   r]  r   rÊ   Únextrž   r�  rc   r  r)  r	  rÍ  rØ   rH  r`  rÊ  rþ  rœ   rÎ  r¹  rI  rÿ  rÎ  Ú	__cause__rD   rM  )r=  r/   r0   rN  Ú
task_queuerÀ  Úraised_exceptionÚinit_skip_reasonrÑ  Ú
skip_entryr„  rÎ  Útb_strÚenhanced_exrb   s                 @rZ   Ú_worker_loopz$MultiProcContinuousTest._worker_loopf  s*  ø€ à Ðà�TÔ&˜JÔ&Ü Ø<¸T¸FÀ-ÐPZÈ|Ð\óð ð ˆŒØ#ˆŒð  Ðð	Ø�L‰L˜˜z¨9Ô5ô 	�‰Ð%Ô&ð Ø —n‘nÓ&ˆGÜ�L‰L˜9 W IÐ.Ô/àˆØð  Ð+Ø ×$Ñ$¤X×%6Ñ%6Ð7GÓ%HÔIØð2Ø×&Ñ& wÔ/Ø ×$Ñ$ WÔ-ð ôV 	�‰Ð&Ô'ñ
  Ü×&Ñ&Õ(ð  øô ò 		Ü  F¨DÓ1ˆIÜÛLœJ×-Ñ-Ó/ÔLØóˆJñ Ø#-×#5Ñ#5Ñ àõ !ûð		ûô> !ò 2Ü˜b¤*Ô-ä '¨¨F°DÓ 9�Iô "&ÛT¤J×$5Ñ$5Ó$7ÔTØó"�Jñ "Ø(×,Ñ,¬X×->Ñ->¸z×?QÑ?QÓ-RÔSå à#'Ð äŸ<™<›>�ØŸ™¤×!;Ñ!;¸XÐ!FÓG�ä*Ð-KÈFÈ8Ð+TÓU�Ø(*�Ô%Ø ×$Ñ$ [×1Ò1ûð12ús8   ³C? Â/"E Ã?	EÄAEÅEÅ	IÅ#A4I ÇAI É Ic                 óJ  — g | _         g | _        g | _        t        j                  d¬«      5 }|j
                  | _        d d d «       	 t        j                  j                  d«       t        t        |«      «      D �]	  }t        j                  j                  «       }t        j                  j                  «       }t        j                  j                  | j                  dt!        |«      z   d||| j                  ||f¬«      }|j#                  «        | j                   j%                  |«       | j                  j%                  |«       | j                  j%                  |«       t&        j)                  d||j*                  «       �Œ y # 1 sw Y   �ŒLxY w# t        $ r Y �Œ;w xY w)NFrý  r  r  T)r  r=   r@  r¡   r  )r  Útask_queuesÚcompletion_queuesrµ  r  r=   rN  rŸ   r  r  rÎ  rU   rg   r  r!  rp  rF   r  r—  r  r)  r  )r=  r0   r  r/   rj  rÀ  r¿  s          rZ   r"  z(MultiProcContinuousTest._spawn_processes¸  sX  € àˆŒØˆŒØ "ˆÔä×(Ñ(°Ô6ð 	#¸!ØŸF™FˆCŒM÷	#ð
	Ü×!Ñ!×2Ñ2°7Ô;ô
 œ#˜j›/Ó*ó 	NˆDÜ×.Ñ.×4Ñ4Ó6ˆJÜ$×4Ñ4×:Ñ:Ó<ÐÜ×+Ñ+×3Ñ3Ø×'Ñ'Ø¤# d£)Ñ+ØØ˜J¨¯©°zÐCSÐTð	 4ó ˆGð �M‰MŒOØ�M‰M× Ñ  Ô)Ø�O‰O×"Ñ" :Ô.Ø×!Ñ!×(Ñ(Ð)9Ô:Ü�L‰LÐ9¸4ÀÇÁÖMñ	N÷	#ñ 	#ûô ò 	âð	ús   ¬FÁF ÆFÆ	F"Æ!F"r“  c                 óF  — t        j                  | dd«      }t        |t        «      r't        j                  | «      }|j                  |«      }n| j                  }|dk(  rAt        j                  |«      j                  «       }|dk(  rt        j                  d|› d�«      ‚|S )zª
        Get world_size, handling both class variable and property definitions.
        Properties are instance-level and need special handling in class methods.
        r0   NrM  r   zNo z devices available)ÚinspectÚgetattr_staticrþ  rw  ÚobjectÚ__new__Úfgetr0   rŸ   rŠ  r    rØ   rH  )r=  r“  Úworld_size_attrÚtemp_instancer0   s        rZ   Ú_get_world_sizez'MultiProcContinuousTest._get_world_sizeØ  s–   € ô "×0Ñ0°°lÀDÓIˆÜ�o¤xÔ0ô #ŸN™N¨3Ó/ˆMØ(×-Ñ-¨mÓ<‰JàŸ™ˆJð ˜ÒÜ×0Ñ0°Ó=×JÑJÓLˆJØ˜QŠÜ×'Ñ'¨#¨k¨]Ð:LÐ(MÓNÐNàÐrj   c                 ó"   •— t         ‰| �  «        y)a  
        Class-scope test fixture. Run once for entire test class, before any test starts.
        Note: Process spawning is deferred to setUp to support instantiate_device_type_tests,
        which calls setUpClass during class creation before any tests run.
        N)rò  r=  rC  s    €rZ   r=  z"MultiProcContinuousTest.setUpClassð  s   ø€ ô 	‰ÑÕrj   c                 ó:  — | j                   ry| j                  j                  d| j                  «      }t	        |t
        «      r|j                  | «      }n7t	        |t        «      r|j                  | «      }nt        |«      r |«       }n|}| j                  |«      | _        t        | j                  «      r| j                  «       n| j                  }|�rt        j                  t        j                  t        j                   t        j"                  dœ}|j                  |«      }|�  |«       st%        j&                  d|› d�«      ‚t(        j+                  d| j,                  › d| j                  › d|› �«       | j/                  | j                  «       d	| _         y)
zå
        Lazily spawn worker processes on first test run.
        This supports instantiate_device_type_tests which calls setUpClass during
        class creation (before any tests run), when spawning would be premature.
        Nr“  )r(   r$   rˆ   r&   z	Backend 'z' is not availablezTesting class z on ú T)rQ  Ú__dict__r	  r“  rþ  ry  Ú__func__rw  ry  rá   r|  r0   rS  rD   r  r  r&  r-  rØ   rH  r  r  rd   r"  )r=  Údevice_type_attrr“  r.   Úbackend_checksÚcheck_fns         rZ   Ú_ensure_processes_spawnedz1MultiProcContinuousTest._ensure_processes_spawnedù  s]  € ð ×!Ò!Øð
 Ÿ<™<×+Ñ+¨M¸3¿?¹?ÓKÐÜÐ&¬Ô4Ø*×3Ñ3°CÓ8‰KÜÐ(¬(Ô3ð +×/Ñ/°Ó4‰KÜÐ&Ô'Ù*Ó,‰Kà*ˆKð ×,Ñ,¨[Ó9ˆŒô (0°·±Ô'@�#—/‘/Ô#ÀcÇoÁoˆØÐä×.Ñ.Ü×.Ñ.Ü×,Ñ,Ü×.Ñ.ñ	ˆNð &×)Ñ)¨'Ó2ˆHØÐ#©H¬JÜ×'Ñ'¨)°G°9Ð<NÐ(OÓPÐPä�‰Ø˜SŸ\™\˜N¨$¨s¯~©~Ð.>¸aÀ¸}ÐMô	
ð 	×Ñ˜SŸ^™^Ô,Ø!%ˆÕrj   c                 óÈ  •— | j                   st        ‰| �	  «        yt        j	                  d| j
                  › d�«       | j                  D ]  }|j                  d«       Œ | j                  D ]  }|j                  «        Œ 	 t        j                  | j                  «       t        j                  d| j                  › d�«       t        ‰| �	  «        y# t        $ r Y Œ=w xY w)z“
        Class-scope test fixture. Run once for entire test class, after all tests finish.
        Tear down the process group if spawned.
        NzJoining z workerszClass z	 finished)rQ  rò  rB  r  r)  r0   rr  rÍ  r  r¹  rG   r‚  rN  rƒ  r  rd   )r=  rj  r¿  r÷  s      €rZ   rB  z%MultiProcContinuousTest.tearDownClass*  sÂ   ø€ ð ×%Ò%Ü‰GÑ!Ô#Øä�‰�x §¡Ð/¨xÐ8Ô9àŸ/™/ò 	!ˆJØ�N‰N˜4Õ ð	!ð —}‘}ò 	ˆGØ�L‰L�Nð	ð	Ü�I‰I�c—m‘mÔ$ô 	�‰�f˜SŸ\™\˜N¨)Ð4Ô5Ü‰ÑÕøô	 ò 	Ùð	ús   ÂC Ã	C!Ã C!c                 óº  •— t         ‰| �  «        | j                  j                  «        | j                  | _        | j                  j                  r&t        j                  d| j                  «       › �«      ‚t        | j                  «      D ]M  \  }}t        j                  d|› d| j                  «       › �«       |j                  | j                  «       «       ŒO y)z5
        Test fixture. Run before each test.
        zPrevious test failed, skipping zSending Rank rð  N)rò  rÿ  r÷  r…  râ  r/   rP  rØ   rH  r  rS  rr  r  r)  rÍ  )rØ  r¤  rj  r÷  s      €rZ   rÿ  zMultiProcContinuousTest.setUpG  s®   ø€ ô 	‰‰Œð 	�‰×0Ñ0Ô2ð ×*Ñ*ˆŒ	ð �>‰>×%Ò%Ü×#Ñ#Ð&EÀdÇgÁgÃiÀ[Ð$QÓRÐRô ' t×'7Ñ'7Ó8ò 	&‰MˆAˆzÜ�L‰L˜=¨¨¨2¨d¯g©g«i¨[Ð9Ô:Ø�N‰N˜4Ÿ7™7›9Õ%ñ	&rj   c                 óV   ‡— t        ‰«      ˆfd„«       }t        j                  || «      S )Nc           	      ó  •— | j                   | j                  k(  �rgt        j                  d| j	                  «       › �«       d }t        t        | j                  | j                  «      «      D �]  \  }\  }}t        ||t        | j	                  «       «      ¬«      }|�Œ2t        |t        j                  «      r|}ŒOt        |t        «      rTt        j                  d|› d| j	                  «       › d| j                   j"                  › �«       d| j                   _        |}Œ³|| j	                  «       k7  rt'        d|› d| j	                  «       › �«      ‚t        j                  d	|› d
| j	                  «       › �«       �Œ |�|‚y  ‰«        y )NzWaiting for workers to finish rÅ  zDetected failure from Rank z in: z(, skipping rest of tests in Test class: TzExpected rv == self.id(), got z != zMain proc detected rank z
 finished )r/   râ  r  r)  r  rS  r¤  r  rs  rÓ  r…  rþ  rØ   rH  rÊ  rk  r÷  rd   rP  rò   )rØ  Údeferred_exceptionr¤  r	  rÀ  Úrvrå   s         €rZ   r¤   z>MultiProcContinuousTest._worker_run_main_wait.<locals>.wrapper]  sz  ø€ à�y‰y˜D×2Ñ2Ó2Ü—‘Ð=¸d¿g¹g»i¸[ÐIÔJð &*Ð"Ü09Ü˜Ÿ™¨×(>Ñ(>Ó?ó1ó Ñ,�AÑ,˜Ð+ô ?ØÐ+´[ÀÇÁÃÓ5Kô�Bð *Ð5à Ü! "¤h×&7Ñ&7Ô8Ø-/Ð*Ø Ü! "¤mÔ4ÜŸ™Ø9¸!¸¸EÀ$Ç'Á'Ã)Àð MEØEIÇ^Á^×E\ÑE\ÐD]ð_ôð 6:˜Ÿ™Ô2Ø-/Ð*Ø ð ˜TŸW™W›Y’Ü,Ø<¸R¸DÀÀTÇWÁWÃYÀKÐPóð ô —L‘LØ2°1°#°ZÀÇÁÃ	¸{ÐKöð5ð< &Ð1Ø,Ð,ð 2ñ •rj   rå  rè  s    ` rZ   Ú_worker_run_main_waitz-MultiProcContinuousTest._worker_run_main_wait\  s/   ø€ Ü	ˆr‹ó(	ó 
ð(	ôT ×Ñ ¨Ó.Ð.rj   rë  rì  c                 óò   •— |dk7  r|}t         ‰| �  |«       	 t        | |«      }t        | || j	                  |«      «       y # t
        $ r+}|dk7  rt        d| j                  › d|› �«      |‚Y d }~y d }~ww xY wrî  )rò  ró  rÊ   rô  rŒ  rõ  rö  r÷  rø  s        €rZ   ró  z MultiProcContinuousTest.__init__Ž  s‹   ø€ ð
 ˜Ò"Ø$ˆKÜ‰Ñ˜Ô%ð		Ü˜˜{Ó+ˆBÜ�D˜+ t×'AÑ'AÀ"Ó'EÕFøÜò 	Ø˜YÒ&ô !Ø-¨d¯n©nÐ-=¸RÀ
¸|ÐLóàðô 'ûð	úrû  )Fru  rt  )rd   re   rf   râ  r0   rg   rh   r/   rN  rF   r   rn  rP  rv  rQ  ry  rS  r“  rY  r]  r`  rp  r"  r|  r=  r…  rB  rÿ  rŒ  ró  rz  r{  s   @rZ   rL  rL    s�  ø… àÐà€J�Óà€Dˆ#ƒNà €Iˆs�T‰zÓ á"¨3Ô/€GˆYÓ/à€K�Óà$Ð˜Ó$àð˜C $™Jò ó ðð ð ˜Cò  ó ð ð òó ðð ñ<ó ð<ð& ð¨ð ¸4ò ó ðð ñO)ó ðO)ðb òNó ðNð> ð¨#ð °#ò ó ðð. óó ðð ñ.&ó ð.&ð` ó ó ð õ8&ò*,/ðf ?HñØðØ8;ðà	÷ñ rj   rL  c                   ó†   ‡ — e Zd ZU eZeed<   edefd„«       Z	e
defd„«       Zdeddfd„Ze
ˆ fd„«       Zdˆ fd	„Zd
„ Zˆ xZS )ÚC10dTorchCommsTestBaser0   r4   c                 ó"   — d| v ryd| v ryd| v ryyr…  ri   r–   s    rZ   r.   zC10dTorchCommsTestBase.backend¥  s$   € à�VÑØØ�f‰_ØØ�f‰_Øàrj   c                 ó`   — | j                   }t        |«      r |«       }| j                  |«      S r”   )r“  rá   r.   )r=  r“  s     rZ   rS  z"C10dTorchCommsTestBase.backend_str°  s)   € à—o‘oˆÜ�KÔ Ù%›-ˆKØ�{‰{˜;Ó'Ð'rj   r1   Nc                 ó¨   — t         t        t        t        t        t
        dœ}| j                  |«      }||v r||   s| j                  d|› d�«       y y y )N)r$   r&   r(   Úrcclr,   r*   ztorchcomms z backend is not available)r%   r'   r)   ÚTORCHCOMM_HAS_RCCLr-   r+   r.   ÚskipTest)rØ  r1   Úbackend_flagsr@   s       rZ   Ú_skip_if_backend_unavailablez3C10dTorchCommsTestBase._skip_if_backend_unavailable·  sY   € ä&Ü&Ü&Ü&Ü(Ü(ñ
ˆð —|‘| FÓ+ˆØ˜=Ñ(°¸|Ò1LØ�M‰M˜K¨ ~Ð5NÐOÕPð 2MÐ(rj   c                 óª  •— dt         j                  j                  _        dt        j
                  d<   t        t        «       «      t        j
                  d<   t        |«      t        j
                  d<   t        |«      t        j
                  d<   |t        j
                  d<   t        ‰| �%  |||«       | j                  «       }d|v sd	|v rˆt         j                  j                  «       }|rYt        j                  |j                  › d
|› �«      }t        j                  |«       t         j                  j!                  |«       y t#        d|› d�«      ‚y )NTrb  r+  r-  r8   r9   ÚTORCHCOMM_STORE_PATHr(   r&   rH  rˆ  r‰  )rŸ   r�  ÚconfigÚuse_torchcommsrG   rH   rF   r   rò  r]  rS  rŒ  r�  r1   r_  rŽ  r�  rÎ  )r=  r/   r0   rN  r.   rŒ  r1   r÷  s          €rZ   r]  zC10dTorchCommsTestBase._init_pgÄ  s  ø€ à26Œ×Ñ× Ñ Ô/Ø$/Œ�
‰
�=Ñ!Ü$'¬Ó(8Ó$9Œ�
‰
�=Ñ!Ü'*¨4£yŒ�
‰
Ð#Ñ$Ü'*¨:£Œ�
‰
Ð#Ñ$Ø-6Œ�
‰
Ð)Ñ*Ü‰Ñ˜˜z¨9Ô5à—/‘/Ó#ˆØ�WÑ ¨'Ñ 1Ü×+Ñ+×?Ñ?ÓAˆKÙÜŸ™¨×)9Ñ)9Ð(:¸!¸D¸6Ð&BÓC�Ü×(Ñ(¨Ô0Ü×!Ñ!×2Ñ2°6Õ:ä"Ø[Ð\cÐ[dÐdzÐ{óð ð !2rj   c                 óð   •— | j                   j                  }t        j                  d| j	                  «       |«       t        |«      r |«       }| j                  t        |«      «       t        ‰| �%  «        y )Nz&Setting up test: %s on device type: %s)
r÷  r“  r  r)  r  rá   r—  rF   rò  rÿ  )rØ  r“  r÷  s     €rZ   rÿ  zC10dTorchCommsTestBase.setUpÚ  sT   ø€ Ø—n‘n×0Ñ0ˆÜ�‰Ð=¸t¿w¹w»yÈ+ÔVÜ�KÔ Ù%›-ˆKØ×)Ñ)¬#¨kÓ*:Ô;Ü‰‰�rj   c                 ó¢   — t        j                  |«      j                  «       }t        | j                  «      D �ci c]	  }|||z  g“Œ c}S c c}w r”   r–  r—  s       rZ   r˜  z%C10dTorchCommsTestBase.rank_to_deviceâ  r™  rš  ru  )rd   re   rf   rÞ  r0   rg   rh   rx  rF   r.   ry  rS  r—  r]  rÿ  r˜  rz  r{  s   @rZ   r�  r�  ¢  sv   ø… Ø(€J�Ó(àð˜3ò ó ðð ð(˜Cò (ó ð(ðQ°3ð Q¸4ó Qð óó ðõ*öNrj   r�  r”   r,  )r³   ru  rÅ   )®r.  Ú	functoolsru  r¡  rI  r  r   rG   rÊ  rµ  rœ   rµ  rC  rÉ  rI  ræ  rØ   Úcollections.abcr   Ú
contextlibr   Údataclassesr   Údatetimer   Úenumr   r   r	   r
   Úior   Útypingr   r   Úunittest.mockr   rŸ   Útorch._dynamo.test_caseÚtorch.cuda.ncclÚtorch.distributedr�  rD   Útorch.nnr  Útorch._C._autogradr   Útorch._C._distributed_c10dr   Útorch._logging._internalr   rC   r   Útorch.testing._internalr   Ú$torch.testing._internal.common_utilsr   r   r   r   r   r   r   r   r   r   r   r   r    Ú5torch.testing._internal.distributed.multi_threaded_pgr!   r"   r#   r%   r'   r)   r”  r-   r+   rI   Ú_backendÚ_flagÚis_backend_builtÚglobalsrF   rg   r1   rL   r[   Ú	getLoggerrd   r  ÚsetLevelÚINFOr0  r•   ÚHAS_ACCELERATORra   rž   r‡   r—   r§   r®   r´   r»   rÃ   rÇ   rv  rÍ   rÔ   rÛ   rç   rë   rú   r  r  r  r  r  r   r#  r'  r4  r8  r@  rB  rh   rE  rT  rK  rW  r[  ra  rr  rƒ  Úgetenvr‚  r  r…  r�  r¥  r¬  r­  r¶  r»  r¾  r!  r  rÓ  rÈ  rÇ  r^   r    rÞ  rÕ  r}  r£  r§  r¯  Úcacher¹  r½  rß  rË  r!  r  r$  r8  r¦  Ú	test_caser:  rE  rL  r�  ri   rj   rZ   ú<module>r¼     sµ  ðô Û Û Û Û Û Û Û 	Û Û Û 
Û Û Û Û Û Û Ý $Ý %Ý !Ý Ý ß ,Ñ ,Ý ß "Ý ã Û Û Ý  Ý Ý )Ý 7Ý .Ý CÝ 0÷÷ ÷ õ ÷ñ ð Ð ØÐ ØÐ ØÐ ØÐ ØÐ ÙÛðò $‰ˆ�%ð 'ˆ:×&Ñ& xÕ0Ø#‰G‹I�eÒð$ð?Øð?à
ð?ð ð?ð �L‰Lð	?ð
 ð?ð ð?ð 
×Ñó?ðD 
ˆ×	Ñ	˜8Ó	$€Ø ‡��—‘Ô â4Ð Ø˜E�?Ð ØÒ3˜xÒ3¨8€ôˆzô ð
Ø™8Ø
ÐCóðð ‘x Ð$FÓGð	ð
 ‘X˜bÐ"BÓCðð ‰x˜Ð4Ó5ðð ‘8˜BÐ =Ó>ðð ‘8˜BÐ >Ó?ðð ‘8˜BÐ >Ó?ðð ‘8˜BÐ >Ó?ðð ‘8˜BÐ >Ó?ðð ‘8˜BÐ >Ó?ðð ‘8˜BÐ >Ó?ðð ‘8˜BÐ >Ó?ðð ‰H�RÐ>Ó?ðð  ‘(˜2Ð6Ó7ð!ð" ‘h˜rÐ#LÓMð#ð$ ‰xØ
ÐVóð%ð* ‘8˜BÐ DÓEð+ð, ‘h˜rÐ#BÓCñ-€
ð4 ÷*ð *ó ð*ò*&òò6òòòò ð°ó ð &+ô ð8˜3ó ð, ð ¨sð °só ò(ò"
ò,+ò\ò4ò
òNòòòóò:òñ $+Ù1ó$Ð ˜Dó ò
Sð
¨¨s°C¨x©ó óò*ðF §¡ð F°cð FÀ#ð FÈ$ó Fð" à	ØØÙ˜aÔ ØØØò
ó ð
ñ: à�Oá˜)˜"Ÿ)š)Ð$GÈÓOÓP€Oà!Ø$'ñÐ ñ Ø+.ÐÐ'Ñ(ñ
¨Tó 
ðI˜Có Ið ñ2ó ð2ñ( Sð (°cð (Àsó (ðX Sð °3ó ð2 /3€ˆ×	$Ò	$ tÑ	+Ó 2ñ
¨S°4©Zð 
À4ó 
ó"ð ñKØ×"Ñ"×*Ò*ðKà×+Ñ+×1Ò1ðKð �4‰ZðKð 	ó	Kñ6 Ù˜Q¡ A u§z¢z×'>Ò'>Ó'@Ó AÓBÑàÐô˜8ô ôN5NÐ.ô 5Nðpà˜˜d 3™i˜Ñ(ðð ðð
 óðD ‡‚ð�ò ó ðò&0ð 
�Ð3Eó7ôtl&˜Hô l&ô^T˜bŸišiô Tô #˜RŸYšYô #ð  à:?ò#)ó ð#)ôL  ¨%¯-ª-×*AÒ*A×*JÑ*Jô   ôF.Ð)<ô .ô8G˜hô GôTBNÐ4õ BNrj   