Ë
    ÃÓ^jr*  ã                   ó^  — d Z ddlZddlZddlmZ ddlmZmZ ddlm	Z	 ddl
Z
ddlZ
ddlmZ ddlmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lmZmZmZmZmZ ddlm Z  ddl!m"Z"  ejF                  e$«      Z%de
jL                  jN                  de(e)   fd„Z*de
jL                  jV                  de,e
jZ                  e
jL                  j\                  f   fd„Z/de
jL                  jV                  de)de0dz  fd„Z1de
jL                  jV                  de)de0dz  fd„Z2de
jL                  jV                  de)fd„Z3de
jL                  jV                  de4e0dz     fd„Z5de
jL                  jV                  dee	   de	fd„Z6 G d„ d«      Z7 e"d e7«       ¬«       	 	 d&d ed!e	f   d"ee	   d#e8d$e8ded!ee	   f   f
d%„Z9y)'a�  
This module implements CUDA graphs support for TorchDynamo backends.

CUDA graphs allow for capturing and replaying GPU operations, which can significantly
reduce CPU overhead in GPU-accelerated PyTorch models. This module provides:

- CUDA graph creation and management for both forward and backward passes
- Input mutation detection and handling
- Device compatibility checking
- Stack trace management for debugging
- Integration with TorchInductor's cudagraph trees

The backend supports two main modes:
1. cudagraphs: Full CUDA graph support with both forward and backward pass optimization
2. cudagraphs_inner: Lower-level CUDA graph implementation used for benchmarking

Key components:
- CudagraphsBackend: Main backend class for CUDA graph integration
- Mutation detection utilities to ensure graph safety
- Device mapping and compatibility checks
- Stack trace collection for debugging
é    N)Údefaultdict)ÚCallableÚSequence)ÚAny)Úconfig)Úaot_autograd)Ú	boxed_nop)ÚBoxedDeviceIndexÚ'check_multiple_devices_or_any_cpu_nodesÚformat_default_skip_messageÚget_mutation_stack_traceÚget_placeholder_infoÚ#log_cudagraph_skip_and_bump_counter)Ú	BoxedBoolÚcount_tangentsÚ%get_first_incompatible_cudagraph_nodeÚnum_fw_fixed_argumentsÚoutput_node)ÚStorageWeakRefé   )Úregister_backendÚgÚreturnc           	      ó¨  — dt         t        t        f   dt        fd„}t        t        «      }d}t	        «       }| j
                  D �]‰  }|j                  dk(  rkt         ||j                  «      t        j                  «      r;|t         ||j                  «      j                  «       «         j                  |«       |dz  }Œ~|j                  dk(  sŒŽt        |j                  d«      sŒ¥|j                  j                   }t#        |j$                  «      D ]·  \  }}|t'        |j(                  «      k  r|j(                  |   }	n2|j*                  |j,                  vrŒG|j,                  |j*                     }	d	}
|j.                  r|j.                  j0                  rd
}
|
sŒ‰||t         ||	j                  «      j                  «       «         z  }Œ¹ �ŒŒ |S )NÚmetar   c                 ó   — d| v r| d   S | d   S )NÚvalÚfake_result© )r   s    úl/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_dynamo/backends/cudagraphs.pyÚmeta_fkz%find_input_mutations.<locals>.meta_fk;   s   € Ø# t™mˆt�E‰{ÐD°°mÑ1DÐDó    r   Úplaceholderr   Úcall_functionÚ_schemaFT)ÚdictÚstrr   r   ÚsetÚnodesÚopÚ
isinstancer   ÚtorchÚTensorr   Ú_typed_storageÚaddÚhasattrÚtargetr%   Ú	enumerateÚ	argumentsÚlenÚargsÚnameÚkwargsÚ
alias_infoÚis_write)r   r!   ÚinputsÚ	input_idxÚmutated_inputsÚnÚschemaÚiÚargÚargumentÚmut_args              r    Úfind_input_mutationsrC   :   s€  € ðE”dœ3¤˜8‘nð E¬ó Eô œÓ€FØ€IÜ“U€NØ�W‰Wó ˆØ�4‰4�=Ò Ü™' !§&¡&›/¬5¯<©<Ô8Ø”~¡g¨a¯f©f£o×&DÑ&DÓ&FÓGÑH×LÑLÈYÔWØ˜‰N‰IØ�T‰T�_Ó$Ü˜1Ÿ8™8 YÔ/Øà—X‘X×%Ñ%ˆFÜ# F×$4Ñ$4Ó5ò ‘��3Ø”s˜1Ÿ6™6“{’?Ø Ÿv™v a™y‘Hà—x‘x q§x¡xÑ/Ø Ø Ÿx™x¨¯©Ñ1�HØ�Ø—>’>Ø—~‘~×.Ò.Ø"&˜Úð # fÜ&¡w¨x¯}©}Ó'=×'LÑ'LÓ'NÓOñ'ñ ‘Nòðð: Ðr"   Úgmc                 óê   — i }| j                   j                  D ]W  }|j                  j                  dd «      }t	        |t
        j                  «      sŒ:|j                  |vsŒI|||j                  <   ŒY |S )Nr   )Úgraphr)   r   Úgetr+   r,   r-   Údevice)rD   Údevice_node_mappingr=   Úts       r    Úget_device_node_mappingrK   a   sg   € ð >@ÐØ�X‰X�^‰^ò .ˆØ�F‰F�J‰J�u˜dÓ#ˆÜ�aœŸ™Õ&¨1¯8©8Ð;NÒ+NØ,-Ð §¡Ò)ð.ð Ðr"   Ú	aot_modelÚ	num_fixedc                 óž   — t        | j                  «      t        t        |«      «      z
  }|sy t	        | j                  «      }t        ||«      S ©N)rC   rF   r(   Úranger   r   )rL   rM   Úmutation_indicesÚplaceholderss       r    Ú3check_for_mutation_ignore_cuda_graph_managed_tensorrS   l   sD   € ô ,¨I¯O©OÓ<¼sÄ5ÈÓCSÓ?TÑTÐÙØä'¨	¯©Ó8€LÜ# LÐ2BÓCÐCr"   c                 óÀ   — t         j                  st        | |«      x}r|S t        t	        | «      «      x}r|S t        | «      x}rt        d|j                  › d�«      S y )Nzincompatible op (ú))r   Ú(cudagraph_backend_support_input_mutationrS   r   rK   r   r   r6   )rL   rM   Úmut_skipÚskipÚnodes        r    Úcheck_for_skiprZ   w   sz   € Ü×:Ò:ÜJØ�yó
ð 
ˆ8ð 
ð ˆOä6Ü 	Ó*óð €tð ð ˆä4°YÓ?Ð?€tÐ?Ü*Ð->¸t¿y¹y¸kÈÐ+KÓLÐLàr"   c                 ó¢   — t        t        t        | «      «      «      }|j                  dk7  rt	        d|j                  › �«      ‚|j
                  S )NÚcudazExpected CUDA device, got )ÚnextÚiterrK   ÚtypeÚAssertionErrorÚindex)rD   rH   s     r    Úget_device_indexrb   ‰   sD   € Ü”$Ô.¨rÓ2Ó3Ó4€FØ‡{�{�fÒÜÐ9¸&¿+¹+¸ÐGÓHÐHØ�<‰<Ðr"   c                 ób  — t        | «      }t        |j                  «      dk7  r!t        dt        |j                  «      › �«      ‚|j                  d   }t	        |d«      sg S |D �cg c]>  }t        |t        j                  j                  j                  «      r|j                  nd ‘Œ@ c}S c c}w )Nr   z(Expected output node to have 1 arg, got r   Ú__iter__)r   r4   r5   r`   r0   r+   r,   ÚfxrY   ÚNodeÚstack_trace)rD   Úoutputr5   r@   s       r    Úget_stack_tracesri   �   s�   € Ü˜‹_€FÜ
ˆ6�;‰;Ó˜1ÒÜØ6´s¸6¿;¹;Ó7GÐ6HÐIó
ð 	
ð �;‰;�q‰>€DÜ�4˜Ô$Øˆ	ð öàô ' s¬E¯H©H¯M©M×,>Ñ,>Ô?ˆ�ŠÀTÑ	Iòð ùò s   Á&AB,Údynamo_modelÚdynamo_inputsc           	      óÆ  ‡‡‡‡— ddl mŠ t        d«      Št        d «      Š	 ddt        j
                  j                  dt        t           dt        dt        fˆˆˆˆfd„}dt        j
                  j                  dt        t           dt        fˆˆˆfd	„}t        ||t        j                  |d¬
«      t        j                  j                  j                  ¬«      } || ‰«      S )Nr   )Úcudagraphify_implTrL   Ú
aot_inputsÚis_inferencer   c                 óª  •— t        | |«      }t        t        ‰
«      t        |«      «      }t        | |«      x}r%t	        j
                  ‰	«       t        d|› �«       |S ‰j                  t        | «      «        ‰||t        |«      ‰j                  d|t        | «      t        | j                  «      t        | j                  «      ¬«	      }d|_        |S )Núskipping cudagraphs due to F©Údevice_indexÚis_backwardro   Ústack_tracesrR   Úmutated_input_idxsT)r	   r   r4   rZ   r   Údisabler   r(   rb   rP   Úvalueri   r   rF   rC   Ú_boxed_call)rL   rn   ro   ÚinterpÚfixedÚskip_msgÚoutÚboxed_device_indexrm   Údo_cudagraphsrk   s          €€€€r    Úforward_cudagraphsz&cudagraphs.<locals>.forward_cudagraphs¥   sÇ   ø€ ô
 ˜9 jÓ1ˆÜ&¤s¨=Ó'9¼3¸z»?ÓKˆÜ% i°Ó7Ð7ˆ8Ð7Ü×Ñ˜mÔ,Ü/Ø-¨h¨ZÐ8ôð ˆMà×ÑÔ/°	Ó:Ô;ÙØØÜ�%‹LØ+×1Ñ1ØØ%Ü)¨)Ó4Ü-¨i¯o©oÓ>Ü3°I·O±OÓDô

ˆð ˆŒØˆ
r"   c                 ó  •‡ ‡— t        ‰ |«      }‰s‰ S t        ‰ «      }t        ‰ |«      x}ryt        d|› �«       ‰	j                  }|€d}t
        j                  j                  j                  |d¬«      Š‰€t        d«      ‚dt        t           dt        fˆ ˆfd„}d	|_        |S  ‰
||t        |«      t        ‰ «      d	dt        ‰ «      t!        ‰ j"                  «      t%        ‰ j"                  «      ¬
«	      }d	|_        |S )Nrq   r   F)Úcreate_if_none_existsz)cudagraph manager must exist for backwardr:   r   c                 ó4   •— ‰j                  «         ‰| «      S rO   )Úset_to_running_backward)r:   rL   Úmanagers    €€r    Úfnz3cudagraphs.<locals>.backward_cudagraphs.<locals>.fnÙ   s   ø€ à×/Ñ/Ô1Ù  Ó(Ð(r"   Trr   )r	   r   rZ   r   rx   r,   Ú	_inductorÚcudagraph_treesÚget_managerr`   Úlistr   ry   rP   rb   ri   r   rF   rC   )rL   rn   rz   r{   r|   Ú
device_idxr†   r}   r…   r~   rm   r   s   `       @€€€r    Úbackward_cudagraphsz'cudagraphs.<locals>.backward_cudagraphsÂ   s  ú€ ô ˜9 jÓ1ˆÙØÐä˜yÓ)ˆÜ% i°Ó7Ð7ˆ8Ð7Ü/Ø-¨h¨ZÐ8ôð
 ,×1Ñ1ˆJØÐ!Ø�
Ü—o‘o×5Ñ5×AÑAØ°%ð Bó ˆGð ˆÜ$Ð%PÓQÐQð)œ4¤™9ð )¬ö )ð
 "ˆBŒNØˆIáØØÜ�%‹LÜ)¨)Ó4ØØÜ)¨)Ó4Ü-¨i¯o©oÓ>Ü3°I·O±OÓDô

ˆð ˆŒØˆ
r"   )ro   )Úfw_compilerÚbw_compilerÚinference_compilerÚkeep_inference_input_mutations)F)Útorch._inductor.cudagraph_treesrm   r   r
   r,   re   ÚGraphModulerŠ   r   Úboolr   Ú	functoolsÚpartialÚ_dynamor   Ú%cudagraph_backend_keep_input_mutation)rj   rk   r€   rŒ   Úaot_cudagraphsr~   rm   r   s    `   @@@r    Ú
cudagraphsr™   Ÿ   sÌ   û€ ÝAä˜d“O€MÜ)¨$Ó/Ðð
 #ñÜ—8‘8×'Ñ'ðäœ‘Iðô ðô 
÷	ð ð:+Ü—8‘8×'Ñ'ð+Ü59¼#±Yð+ä	÷+ôZ "Ø&Ø'Ü$×,Ñ,Ð-?ÈdÔSÜ',§}¡}×';Ñ';×'aÑ'aô	€Nñ ˜,¨Ó6Ð6r"   c            	       ór   — e Zd ZdZed	d„«       Zedej                  j                  de	e
   de
de
fd„«       Zy)
ÚCudagraphsBackendr™   r   Nc                  ó   — ddl m}   | «        y )Nr   ©Úreset_cudagraph_trees)r‘   rž   r�   s    r    ÚresetzCudagraphsBackend.resetû   s   € åIáÕr"   Úmodelr:   r7   c                 óJ   — |rt         j                  d|«       t        | |«      S )Nz+cudagraphs backend ignoring extra kwargs %s)ÚlogÚwarningr™   )r    r:   r7   s      r    Ú__call__zCudagraphsBackend.__call__  s#   € ñ Ü�K‰KÐEÀvÔNÜ˜% Ó(Ð(r"   )r   N)Ú__name__Ú
__module__Ú__qualname__Úcompiler_nameÚstaticmethodrŸ   r,   re   r’   r   r   r¤   r   r"   r    r›   r›   ø   s[   „ Ø €Màò ó ð ð
 ð)Ø�x‰x×#Ñ#ð)Ø-5°c©]ð)ØFIð)à	ò)ó ñ)r"   r›   r™   )r6   Úcompiler_fnr    .r:   Úcopy_outputsÚcopy_inputsc                 óî  ‡‡‡‡‡	— t        |t        t        f«      st        dt	        |«      › �«      ‚‰r$|D �cg c]  }t        j                  |«      ‘Œ c}Šnt        |«      Št
        j                  j                  «        t
        j                  j                  «       }|j                  t
        j                  j                  «       «       t
        j                  j                  |«      5   | |Ž  ddd«       |j                  «        t
        j                  j                  «       j                  |«       t
        j                  j                  «        t
        j                  j                  «       Št
        j                  j                  ‰|¬«      5   | ‰Ž Š	ddd«       t        ‰	t        t        f«      s‰	fŠ	dt        dt         t           fˆˆˆˆˆ	fd„}|S c c}w # 1 sw Y   ŒîxY w# 1 sw Y   ŒRxY w)zBThis isn't registered as a backend, but is used in some benchmarksz+Expected inputs to be a list or tuple, got N)ÚstreamÚ
new_inputsr   c                  ó4  •— t        ‰«      t        | «      k7  r#t        dt        ‰«      › dt        | «      › �«      ‚‰r%t        ‰| «      D ]  \  }}|j                  |«       Œ ‰j	                  «        ‰r‰D �cg c]  }|j                  «       ‘Œ c}S ‰S c c}w )Nz	Expected z inputs, got )r4   r`   ÚzipÚcopy_ÚreplayÚclone)	r¯   ÚdstÚsrcÚxr¬   r«   rF   Ústatic_inputsÚstatic_outputss	       €€€€€r    Úrunzcudagraphs_inner.<locals>.run0  s”   ø€ Üˆ}Ó¤ Z£Ò0Ü ØœC Ó.Ð/¨}¼SÀ»_Ð<MÐNóð ñ Ü ¨zÓ:ò ‘��SØ—	‘	˜#•ðà�‰ŒÙØ'5Ö6 !�A—G‘G•IÒ6Ð6à!Ð!ùò 7s   Á9B)r+   rŠ   Útupler`   r_   r,   Ú
zeros_liker\   ÚsynchronizeÚStreamÚwait_streamÚcurrent_streamr®   Ú	CUDAGraphrF   r   r   )
r    r:   r«   r¬   r·   r®   rº   rF   r¸   r¹   s
     ``   @@@r    Úcudagraphs_innerrÂ     s„  ü€ ô �fœt¤U˜mÔ,ÜØ9¼$¸v»,¸ÐHó
ð 	
ñ Ø6<Ö=°œ×)Ñ)¨!Õ,Ò=‰ä˜V›ˆô 
‡J�J×ÑÔÜ�Z‰Z×ÑÓ €FØ
×Ñ”u—z‘z×0Ñ0Ó2Ô3Ü	�‰×	Ñ	˜6Ó	"ñ Ùˆv‰÷à
×ÑÔÜ	‡J�J×ÑÓ×+Ñ+¨FÔ3Ü	‡J�J×ÑÔô �J‰J× Ñ Ó"€EÜ	�‰×	Ñ	˜%¨Ð	Ó	/ñ /Ù Ð.ˆ÷/ä�n¤t¬U mÔ4Ø(Ð*ˆð"œð "¤¬#¡÷ "ñ "ð €JùòG >÷ð ú÷/ð /ús   ¹GÃ,GÆG+ÇG(Ç+G4)TT):Ú__doc__r”   ÚloggingÚcollectionsr   Úcollections.abcr   r   Útypingr   r,   Útorch.fxÚtorch._dynamor   Útorch._dynamo.backends.commonr   Ú torch._dynamo.backends.debuggingr	   Útorch._inductor.cudagraph_utilsr
   r   r   r   r   r   Útorch._inductor.utilsr   r   r   r   r   Ú torch.multiprocessing.reductionsr   Úregistryr   Ú	getLoggerr¥   r¢   re   ÚGraphr(   ÚintrC   r’   r&   rH   rf   rK   r'   rS   rZ   rb   rŠ   ri   r™   r›   r“   rÂ   r   r"   r    ú<module>rÓ      sç  ðñó. Û Ý #ß .Ý ã Û Ý  Ý 6Ý 6÷÷ ÷õ õ <å &ð €g×Ñ˜Ó!€ð$˜EŸH™HŸN™Nð $¨s°3©xó $ðNØ�‰×Ñðà	ˆ%�,‰,˜Ÿ™Ÿ™Ð
%Ñ&óðDØ�x‰x×#Ñ#ðDØ03ðDàˆ4�ZóDð˜eŸh™h×2Ñ2ð ¸sð ÀsÈTÁzó ð$˜Ÿ™×-Ñ-ð °#ó ð˜Ÿ™×-Ñ-ð °$°s¸T±zÑ2Bó ðV7˜UŸX™X×1Ñ1ð V7À(È3Á-ð V7ÐTWó V7÷r)ñ )ñ( �lÑ0AÓ0CÕ Dð Øñ	/Ø�C˜�HÑð/à�S‰Mð/ð ð/ð ð	/ð
 ˆc�8˜C‘=Ð Ñ!ô/r"   