
    ^jP                     :	   d dl Z d dlZd dlmZ d dlmZ d dlmZm	Z	mZ d dl
mZ d dlmZ d dlmZmZmZ d dlmZmZ d d	lmZ d d
lmZmZ d dlmZ d Zd Zd Zd Zd Z d Z!g dZ"ejF                  jI                  ddejJ                  jL                         ejF                  jO                  dd      d        Z(ejF                  jO                  dd      d        Z)ejF                  jO                  dd      d        Z*ejF                  jW                  d      d        Z,d Z-d Z.d Z/ed         Z0d! Z1d" Z2d# Z3d$ Z4d% Z5d& Z6d' Z7d( Z8d) Z9d* Z:d+ Z;d, Z<d- Z=d. Z> e       Z?d/ Z@d0 ZAd1 ZBd2 ZCd3 ZDd4 ZE ed5d6e>e= eej                  ej                        d7d7d7d7d7d7d78       ed9d6e1e/ eej                  ej                        d7d7d7d7d7d:d78       ed;d6ee eej                  ej                        d7d7d7d7d7<
       ed;d=ee eej                  ej                        d7d7d7d7d7<
       ed;d>e e eej                  ej                        d7d7d7d7d7<
       ed?d6e.e- eej                  ej                        d7d7d7d7d7d:d78       ed@d6e@e/ eej                  ej                        d7d7d7d7d7d:d78       edAd6eAe/ eej                  ej                        d7d7d7d7d7d:d78       edBd6e9e8 eej                  ej                        d7d7d7d7d7d7C       edDd6e:e8 eej                  ej                        d7d7d7d7d7d7C       edEd6e3e2 eej                  ej                        d7d7d7d7d7d7C       edQdFd6ee4dG e!ej                  ej                        d7d7d7d7d7dH edQdId6e7e5dG e!ej                  ej                        d7d7d7d7d7d7d7dJ edQdIdKe7e6dG e!ej                  ej                        d7d7d7d7d7d7d7dJ edQdLd6e<e;dGej                  j                         r e!ej                  ej                        ndM e       id7d7d7d7d7dH edNd6eCeB eej                  ej                        d7d7d7d7d7d7C       edQdOd6eEeDdG e!ej                        d7d7d7d7d7d7dPgZLy)R    Nmap)flex_attention)_create_empty_block_maskcreate_block_maskr   )make_tensor)inline_asm_elementwise)all_types_andcustom_typesempty_types)OpInfoSampleInput)mark_compile_region)InvokeQuantinvoke_quant_packedregister_hook_opc           
   +      K   t        j                  t        |||      }t         |ddddd       |ddddd      g |ddd       |ddd      f       y w)Ndevicedtyperequires_grad   皙?lowhigh   args	functoolspartialr   r   opinfor   r   r   kwargsmake_args         i/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/testing/_internal/hop_db.pysample_inputs_mapr)      sj       F%}H 	!Qs	+XaA3Q-OPqc*HQCa,HI s   AAc                     | d   j                         j                  d      |z  | d   |j                         z   j                         j	                  | d   j                               gS )Nr         ?r   )cosadd_sincos_viewsizexy0y1s      r(   inner_fr6   "   sQ    aDHHJOOC 2%!rvvx'='='?'D'DQqTYY['QRR    c                 $    d }t        || ||      S )Nc                     t        | ||      S Nr6   r2   s      r(   fzsimple_map.<locals>.f'   s    q"b!!r7   r   )xsr4   r5   r<   s       r(   
simple_mapr>   &   s    " q"b"r7   c                 $    d }t        || ||      S )Nc                 $    d }t        || ||      S )Nc                     t        | ||      S r:   r;   r2   s      r(   f2z"nested_map.<locals>.f1.<locals>.f2/   s    1b"%%r7   r   xxr4   r5   rB   s       r(   f1znested_map.<locals>.f1.   s    	& 2r2r""r7   r   r=   r4   r5   rE   s       r(   
nested_maprG   -   s    # r2r2r7   c                 $    d }t        || ||      S )Nc                 $    d }t        || ||      S )Nc                 $    d }t        || ||      S )Nc                     t        | ||      S r:   r;   r2   s      r(   rB   z5triple_nested_map.<locals>.f0.<locals>.f1.<locals>.f2:   s    q"b))r7   r   rC   s       r(   rE   z)triple_nested_map.<locals>.f0.<locals>.f19   s    * r2r2&&r7   r   rF   s       r(   f0ztriple_nested_map.<locals>.f08   s    	' 2r2r""r7   r   )r=   r4   r5   rL   s       r(   triple_nested_maprM   7   s    # r2r2r7   c                  :    t               t        |  t               dS )N)dtypesdtypesIfCUDAdtypesIfXPU)r   r   )rO   s    r(   cuda_only_dtypesrR   D   s    -$f-"} r7   )"custom_function_callautograd_function_applyrun_and_save_rng_staterun_with_rng_staterun_dtensor_rng_opgraphsafe_run_with_rng_state	out_dtypetrace_wrappedtag_activation_checkpointexecutorch_call_delegatewrapwrap_with_set_grad_enabledauto_functionalized_v2associative_scan
flat_applywrap_with_autocastwrap_activation_checkpointrun_const_graphauto_functionalizedr   map_implwith_effectsstrict_mode_export_tracepointcall_torchbindtriton_kernel_wrapper_mutation triton_kernel_wrapper_functionalhints_wrapperdynamo_bypassing_wrapperforeach_mapaoti_call_delegateprintinductor_compiled_codeinvoke_leaf_functionztestlib::mutating_custom_opz8(Tensor(a!) x, Tensor(b!) z) -> (Tensor, Tensor, Tensor))tagscpuc                     | j                  d       |j                  d       | j                         |j                         | |z   fS N   r-   cloner3   zs     r(   foo_impl_cpur}      5    FF1IFF1I779aggiQ&&r7   cudac                     | j                  d       |j                  d       | j                         |j                         | |z   fS rw   ry   r{   s     r(   foo_impl_cudar      r~   r7   xpuc                     | j                  d       |j                  d       | j                         |j                         | |z   fS rw   ry   r{   s     r(   foo_impl_xpur      r~   r7   c                 J    | j                         |j                         | |z   fS r:   rz   r{   s     r(   foo_impl_abstractr      s    779aggiQ&&r7   c           	   +   v   K   t        j                  t        |||      }t         |ddddd             y wNr   r   r   r   r!   r$   s         r(   sample_inputs_condr      9       F%}H hq!QCa8
99   79c                 Z    t        j                  | j                         dkD  d d | g      S )Nr   c                 $    | j                         fS r:   r,   r3   s    r(   <lambda>zsimple_cond.<locals>.<lambda>   s    aeegZ r7   c                 $    | j                         fS r:   )r.   r   s    r(   r   zsimple_cond.<locals>.<lambda>   s    AEEG: r7   )torchcondsumr   s    r(   simple_condr      s&    ::aeegk#79MPQsSSr7   c           	   +   v   K   t        j                  t        |||      }t         |ddddd             y wr   r!   r$   s         r(   sample_inputs_invoke_subgraphr      r   r   c                 ,    t        j                  |       S r:   r   r.   r   s    r(   fn_for_invoke_subgraphr      s    99Q<r7   c                     t        |       S r:   )r   r   s    r(   simple_invoke_subgraphr      s    !!$$r7   c           
   +      K   t        j                  t        ||d      }t         |ddddd       |ddddd             y w)NFr   r   r   r   r!   r$   s         r(    sample_inputs_auto_functionalizer      sM       F%uH Aqc*HQ1#A,N    AAc                 V    t         j                  j                  j                  | |      S r:   )r   opstestlibmutating_custom_opr{   s     r(   simple_auto_functionalizer      s    99//155r7   c              +      
K   t        j                  t        |||      
d }
fdt        d      D        \  }}}t	        ||      }	t        |||||	       y w)Nr   c                     | |z   S r:    scorebhmns        r(   	score_modz/sample_inputs_flex_attention.<locals>.score_mod   s    qyr7   c           	   3   :   K   | ]  } d d dddd         yw)r         r   r   Nr   .0_r'   s     r(   	<genexpr>z/sample_inputs_flex_attention.<locals>.<genexpr>   s#     I1x1c1#A66I      )r"   r#   r   ranger   r   )r%   r   r   r   r&   r   qkv
block_maskr'   s             @r(   sample_inputs_flex_attentionr      s[       F%}H JaIGAq!)!Q/J
aAy*
55s   AAc              +     K   t        j                  t        ||d      d }d }fdt        d      D        \  }}}	t	        |dddd|	      }
d
|j                  d      dz  z  }t        |||	||
j                         |i       \  }}}t        |||	|j                         |j                         t        j                  |      d |d |
j                         |i ddf       y w)NFr   c                     | S r:   r   r   s        r(   r   z8sample_inputs_flex_attention_backward.<locals>.score_mod       r7   c                     ||k\  S r:   r   )r   r   r   r   s       r(   mask_modz7sample_inputs_flex_attention_backward.<locals>.mask_mod   s    Avr7   c           	   3   :   K   | ]  } d d dddd         ywr   r      r   r   Nr   r   s     r(   r   z8sample_inputs_flex_attention_backward.<locals>.<genexpr>   #     JAx1c23Q77Jr   r   r   r   BHQ_LENKV_LENr   r+         ?r   r   )r"   r#   r   r   r   r1   flex_attention_hopas_tupler   detachr   	rand_like)r%   r   r   r   r&   r   r   r   r   r   r   scaleout	logsumexpr   r'   s                  @r(   %sample_inputs_flex_attention_backwardr      s        F%uH KqJGAq!"8qASU[\J!&&*##E*	1aJ//15"CA 	q#**,	 0 0 2EOOC4H$tZ0022r2
 s   C!C$c              +      K   t        j                  t        ||d      t        j                  dd|t        j
                        d }fd}fdt        d	      D        \  }}}	t        |d
d
dd|      }
d|j                  d      dz  z  }t        |||	||
j                         |i       \  }}}t        |||	|j                         |j                         t        j                  |      d |d |
j                         |i ddf       y w)NFr   r   r   )r   r   c                     | S r:   r   r   s        r(   r   zIsample_inputs_flex_attention_backward_explicit_buffers.<locals>.score_mod   r   r7   c                     |z   |k\  S r:   r   )r   r   r   r   mask_offsets       r(   r   zHsample_inputs_flex_attention_backward_explicit_buffers.<locals>.mask_mod   s    ;!##r7   c           	   3   :   K   | ]  } d d dddd         ywr   r   r   s     r(   r   zIsample_inputs_flex_attention_backward_explicit_buffers.<locals>.<genexpr>   r   r   r   r   r   r+   r   r   r   )r"   r#   r   r   fullint32r   r   r1   r   r   r   r   r   )r%   r   r   r   r&   r   r   r   r   r   r   r   r   r   r   r'   r   s                  @@r(   6sample_inputs_flex_attention_backward_explicit_buffersr      s       F%uH **RV5;;GK$ KqJGAq!"8qASU[\J!&&*##E*	1aJ//15"CA 	q#**,	 0 0 2EOOC4H$tZ0022r2
 s   D
Dc                 n    t         j                  j                  j                  | |||||||||	|
|||      S r:   )r   r   higher_orderflex_attention_backward)querykeyvaluer   r   grad_outgrad_logsumexpfw_graphjoint_graphr   r   kernel_optionsscore_mod_other_buffersmask_mod_other_bufferss                 r(   simple_flex_attention_backwardr     sK      99!!99 r7   c           
   +      K   t        j                  t        ||d      }t        t	        j
                  d       |ddddd             y w)NFr   r   r      r   r   )r"   r#   r   r   r   tensorr$   s         r(   sample_inputs_while_loopr   2  sI       F%uH QAqc* s   AAc                 T    d }d }t         j                  j                  ||| |f      S )Nc                     | dkD  S Nr   r   iter_tr3   s     r(   cond_fnz"simple_while_loop.<locals>.cond_fn=      zr7   c                 ,    | dz
  |j                         fS Nr   r   r   s     r(   body_fnz"simple_while_loop.<locals>.body_fn@      z1557""r7   )r   _higher_order_ops
while_loopr   r3   r   r   s       r(   simple_while_loopr   <  s-    # ""--gwLLr7   c                 f    d }d }t         j                  j                  ||| |ft                     S )Nc                     | dkD  S r   r   r   s     r(   r   z/simple_while_loop_stack_output.<locals>.cond_fnG  r   r7   c                 ,    | dz
  |j                         fS r   r   r   s     r(   r   z/simple_while_loop_stack_output.<locals>.body_fnJ  r   r7   )r   r   while_loop_stack_outputtupler   s       r(   simple_while_loop_stack_outputr  F  s6    # ""::61+uw r7   c           
   +      K   t        j                  t        ||d      }t         |ddddd       |ddddd             y w)NFr   r   r   r   r   r   r!   r$   s         r(   sample_inputs_local_map_hopr  R  sN       F%uH Aqc*Aqc* r   c                 T   d }t         j                  j                  |      }t         j                  j	                         st        d      ddlm}  |        |        |       f |        |        |       ffd|j                  d<   t         j                  j                  || |      S )Nc                 D    | j                         |j                         z   S r:   )r,   r.   )inp1inp2s     r(   body_gmz%simple_local_map_hop.<locals>.body_gm^  s    xxzDHHJ&&r7   z*Expected torch.distributed to be availabler   )	Replicate)in_placementsout_placementslocal_map_kwargs)r   fxsymbolic_tracedistributedis_availableAssertionError(torch.distributed.tensor.placement_typesr  metar   local_map_hop)r
  r  r  gmr  s        r(   simple_local_map_hopr  ]  s    ' 
	 	 	)B))+IJJB $+y{IK@%KikBD#BGG ""00T4@@r7   c           
   +      K   t        j                  t        |||      }t         |dddd       |ddddd             y wr   r!   r$   s         r(   sample_inputs_scanr  p  sL       F%}H A3Q'Aqc* s   AAc                 J    d }t         j                  j                  || |      S )Nc                 6    | |z  |z   }|| j                         fS r:   r   )carryr3   results      r(   
combine_fnzsimple_scan.<locals>.combine_fn{  s     Qu{{}$$r7   )r   r   scan)initr=   r!  s      r(   simple_scanr$  z  s$    % ""''
D"==r7   c                 .    d }t        || |       d   dz  S )Nc                 4    t        j                  |       |z  fS r:   r   )r3   ys     r(   fnzsimple_invoke_quant.<locals>.fn  s    		!q ""r7   r          @)quant_tracerr3   r(  s     r(   simple_invoke_quantr,    s"    # Aq!!$s**r7   c                 ,    d }t        ||       d   dz  S )Nc                 .    t        j                  |       fS r:   r   r   s    r(   r(  z&simple_invoke_quant_packed.<locals>.fn  s    		!r7   r   r)  )r   r+  s     r(   simple_invoke_quant_packedr/    s      r1%a(3..r7   c           	   +   v   K   t        j                  t        ||d      }t         |ddddd             y w)NTr   r   r   r   r!   r$   s         r(   sample_inputs_register_hookr1    s9       F%tH hq!QCa8
99r   c                      d }t        | |      S )Nc                     | dz  S )Nr   r   )grads    r(   hookz"simple_register_hook.<locals>.hook  s    axr7   r   )r3   r5  s     r(   simple_register_hookr6    s     At$$r7   c           	   +   v   K   t        j                  t        |||      }t         |ddddd             y wr   r!   r$   s         r(   sample_inputs_inline_asmr8    r   r   c                     t         j                  j                  rt        | ddt         j                        S t        | ddt         j                        S )Nzv_mov_b32_e32 $0, $1z=v, v)asm_strconstraintsr   zmov.f32 $0, $1;z=f,f)r   versionhipr	   float32r   s    r(   simple_inline_asmr?    sI    }}%*--	
 	
 "	$& r7   r"  simpleF)namevariant_test_nameopsample_inputs_funcrO   supports_outcheck_batched_gradcheck_batched_gradgradcheck_batched_forward_grad"check_inplace_batched_forward_gradsupports_autogradsupports_gradgradinvoke_subgraphTr   )
rA  rB  rC  rD  rO   rE  rF  rG  rH  rI  nestedtriple_nestedr   invoke_quantr   r   )rA  rB  rC  rD  rO   rE  rF  rG  rH  rI  rJ  r  auto_functionalizer   )rA  rB  rC  rD  )rE  rF  rG  rH  rI  r   )rE  rF  rG  rH  rI  rJ  rK  explicit_buffersr  rO   register_hookr	   )rE  rF  rG  rH  rI  rJ  r   )Mr"   r   #functorch.experimental.control_flowr   &torch._higher_order_ops.flex_attentionr   r   !torch.nn.attention.flex_attentionr   r   torch.testingr   .torch._higher_order_ops.inline_asm_elementwiser	   $torch.testing._internal.common_dtyper
   r   r   #torch.testing._internal.opinfo.corer   r   'torch._higher_order_ops.invoke_subgraphr   torch._higher_order_opsr   r   %torch._higher_order_ops.register_hookr   r)   r6   r>   rG   rM   rR   0FIXME_hop_that_doesnt_have_opinfo_test_allowlistlibrarydefineTagpt2_compliant_tagimplr}   r   r   register_faker   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r$  r*  r,  r/  r1  r6  r8  r?  boolhalffloat16r>  r  r  hop_dbr   r7   r(   <module>rh     s     3 
 & Q Y Y C G D BS
,#4 0J   !>		$	$   159' :' 16:' ;' 159' :' :;' <':T:  %6
6:<DM	A&> }+/:%: "-UZZ4 $#(+0 "!8UZZ4 $#(+0 ",UZZ4 $#(+0 ",UZZ4 $#(+0 ),UZZ4 $#(+0 "-UZZ4 $#(+0 "8UZZ4 $#(+0 ""%8UZZ4 $#(+0 "3UZZ4 $#(+0 &")3UZZ4 $#(+0 !"$;UZZ4 $#(+0  "7	
 5==%--
8  $#(+0  &")@	
 5==%--
8  $#(+0  &,)Q	
 5==%--
8  $#(+0  "6	   --/ U]]EMM:KM*  $#(+0  "6UZZ4 $#(+0  %"3	
 5==
)  $#(+0yi
r7   