Ë
    ÄÓ^j£À ã                   ó"  — U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZmZ d dlmZ d dlmZmZmZmZ d dlZd dlZd dlZd dlmZ d dlmZmZ d dlmZ d dlmZm Z m!Z! d d	l"m#Z#m$Z$m%Z% d
dl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9 ddl:m;Z; ddl<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZH ddlImJZJmKZKmLZLmMZM ddlNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZ ddl[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZj ejÖ                  dk(  ZlejÚ                  d„ «       ZnejÞ                  já                  eqd«      Zr eg d¢«      Zsdddddd d!d"d#d#d$œ
Zt eg d%¢«      Zud&d'd(d)d*d+d,d-d.d/d0œ
Zvd1d2d3œZwejð                  ejò                  gZzejö                  ejø                  ejð                  ejò                  ejú                  ejü                  ejþ                  e�j                   e�j                  e�j                  e�j                  gZ„e…e�j                     e‡d4<   d5„ Zˆd6„ Z‰	 	 	 dwd7e�j                  eRz  dz  fd8„Z‹d9„ ZŒd:eVd;e�j                  d<e�d=e�j                  d>e�j                  d?ePfd@„ZŽdAe�eRz  dBe�dCe�dDe�j                  dEe�e�z  f
dF„Z�dGeVdHe�dIe�fdJ„Z‘dGeVfdK„Z’e�j&                   G dL„ dM«      «       Z” G dN„ dOe7«      Z• G dP„ dQ«      Z–dR„ Z— G dS„ dTeY«      Z˜e˜�j3                  dU«        G dV„ dWe˜«      Zšeš�j3                  dX«       eš�j7                  «         G dY„ dZeš«      Zœ G d[„ d\eW«      Z� G d]„ d^e�«      Zž G d_„ d`ež«      ZŸdae1d?e e�j                  dz  e}f   fdb„Z¡e�j&                   G dc„ dd«      «       Z¢ G de„ df«      Z£ G dg„ dhe�«      Z¤ G di„ dje�«      Z¥ G dk„ dle«      Z¦ G dm„ dne4«      Z§ G do„ dp«      Z¨ G dq„ dr«      Z©e�j&                   G ds„ dt«      «       Zªe�j&                   G du„ dv«      «       Z«y)xé    N)ÚCallableÚSequence)ÚEnum)ÚAnyÚcastÚClassVarÚOptional)Údependencies)Úis_float_dtypeÚis_integer_dtype)Ú
OrderedSet)ÚCeilDivÚFloorDivÚModularIndexing)Úfree_symbol_is_typeÚsymbol_is_typeÚSymTé   )Úcountersé   )ÚconfigÚcpp_builderÚcpu_vec_isaÚirÚmetrics)Ú'set_kernel_post_grad_provenance_tracing)ÚLoopBody)ÚBaseSchedulerNodeÚBaseSchedulingÚExternKernelSchedulerNodeÚForeachKernelSchedulerNodeÚFusedSchedulerNodeÚ	SchedulerÚSchedulerNode)Ústride_at_vec_range)Úcache_on_selfÚget_bounds_index_exprÚget_fused_kernel_nameÚhas_free_symbolsÚis_multi_outputs_templateÚis_welford_reductionÚparallel_num_threadsÚPlaceholderÚsympy_index_symbolÚsympy_index_symbol_with_prefixÚsympy_productÚ
sympy_subs)ÚNullKernelHandlerÚopsÚOpsValueÚVé   )ÚBackendFeatureÚBracesBufferÚCSEÚCSEVariableÚDataTypePropagationÚDeferredLineÚDTYPE_TO_COMPUTATION_DTYPEÚIndentedBufferÚKernelÚ
KernelArgsÚOpOverridesÚOptimizationContext)Ú_get_dtype_from_loopbodiesÚ_get_loop_bodyÚcexprÚcexpr_indexÚcodegen_randÚCppCSEVariableÚDTYPE_TO_CPPÚget_promote_dtypeÚ
INDEX_TYPEÚLocalBufferContextÚmay_unify_binary_op_mask_typeÚpromote_argsÚ(template_fusion_with_epilogues_supportedÚunify_mask_base_typeÚvalue_to_cppÚwin32c                  ó   — t         rdS dS )Nz__declspec(dllexport)Ú ©Ú_IS_WINDOWS© ó    úf/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/codegen/cpp.pyÚget_export_declarationrZ   [   s   € å&1Ð"Ð9°rÐ9rX   Úschedule)ú+Ú*ú^ú||ÚminÚmaxr\   r]   r^   r`   ra   ÚargminÚargmaxr_   Úwelford)
ÚsumÚprodÚxor_sumr`   ra   rb   rc   ÚanyÚwelford_reduceÚwelford_combine)
ra   r`   re   rf   rg   ri   rj   rb   rc   rh   z
at::TensorÚlongÚdoubleÚboolzstd::stringzc10::ScalarTypezat::MemoryFormatz
at::Layoutz
at::Devicez
at::Scalar)
ÚTensorÚintÚfloatrm   ÚstrÚ
ScalarTypeÚMemoryFormatÚLayoutÚDeviceÚnumberzstd::vectorzstd::optional)ÚListr	   ÚVECTORIZABLE_DTYPESc                 óŽ  — |t         v rt        j                  }| dv ry| dk(  ry| dv r|t        |   }|t        j                  k(  r| dv rt        t        j
                     }t        |«      rd|› d�nd	|› d
�}t        |«      rd	|› d�nd	|› d�}| dv r|n|}| dv r|S d|› d|› d�S t        | «      rdt        |   › d�S t        | «      ‚)N)rg   re   rh   r   rf   r6   )ra   rc   r`   rb   ©rb   rc   ú-std::numeric_limits<ú>::infinity()ústd::numeric_limits<ú>::min()ú>::max())ra   rc   )ra   r`   úIndexValue<z>{0, ú}úWelford<ú>())	ÚDTYPE_LOWP_FPÚtorchÚfloat32rI   rm   rp   r   r+   ÚAssertionError)Úreduction_typeÚdtypeÚcdtypeÚmin_varÚmax_varÚinit_vars         rY   Úreduction_initrŽ   ¤   s  € Ø”Ñô —‘ˆØÐ2Ñ2ØØ˜ÒØØÐ;Ñ;Ü˜eÑ$ˆØ”E—J‘JÒ >Ð5IÑ#IÜ!¤%§+¡+Ñ.ˆFô ˜eÔ$ð $ F 8¨=Ñ9à'¨ x¨xÐ8ð 	ô ˜eÔ$ð # 6 (¨-Ñ8à'¨ x¨xÐ8ð 	ð
 -Ð0AÑA‘7Àwˆð  Ñ/ð ð	
ð ˜v˜h f¨X¨J°bÐ9ð	
ô
 ˜NÔ+Øœ, uÑ-Ð.¨cÐ2Ð2Ü
˜Ó
(Ð(rX   c                 ó°   — t         t        |      }t        | «      rd|› d�S | dv r0|t        j                  k(  rt         t        j
                     }d|› d�S |S )Nr‚   ú>rz   r€   )rI   r=   r+   r…   rm   rp   )rˆ   r‰   Úscalar_types      rY   Úreduction_acc_typer’   Æ   sa   € ÜÔ9¸%Ñ@ÑA€KÜ˜NÔ+Ø˜+˜ aÐ(Ð(ØÐ-Ñ-Ø”E—J‘JÒÜ&¤u§{¡{Ñ3ˆKØ˜[˜M¨Ð+Ð+ØÐrX   Úindexc           	      óp  — |t         j                  k(  }| dk(  r|r	d|› d|› d�S |rdnd}|› d|› d|› �S | dk(  r|› d	|› �S | d
k(  r|› d|› �S | dk(  r|› d|› �S | dv r| › d|› d|› d�S | dk(  r|rd|› d|› d|› d�S d|› d|› d�S | dk(  r6t        |t        «      r|\  }}	}
nt	        | |«      \  }}	}
d|› d|› d|	› d|
› d�	S | dv rkt        |d«      rD|j                  t         j                  k(  r'|j                  s|�| › d|› d|› d|› d�S | › d|› d|› d�S |�| › d|› d|› d|› d�S | › d|› d|› d�S t        | «      ‚)Nre   úcascade_sum_combine(ú, &ú)ú|r\   ú rf   ú * rg   ú ^ rh   ú || )r`   ra   z_propagate_nan(ú, ri   úwelford_combine(rj   ú, {ú})rz   r‰   z	_combine(z, static_cast<float>(ú), ú)))	r…   rm   Ú
isinstanceÚtupleÚreduction_projectÚhasattrr‰   Úis_vecr‡   )rˆ   ÚvarÚ
next_valueÚ
helper_valr“   Ú	src_dtypeÚis_boolÚconjunctionÚmeanÚm2Úweights              rY   Úreduction_combiner±   Ñ   s  € ð œ5Ÿ:™:Ñ%€GØ˜ÒÙØ)¨*¨°S¸¸ÀAÐFÐFá!(™#¨cˆKØ�U˜!˜K˜=¨¨*¨Ð6Ð6Ø˜ÒØ��c˜*˜Ð&Ð&Ø˜Ò"Ø��c˜*˜Ð&Ð&Ø˜ÒØ��d˜:˜,Ð'Ð'Ø˜Ñ'Ø Ð! °°°R¸
°|À1ÐEÐEØÐ)Ò)ÙØ% c U¨"¨Z¨L¸¸J¸<ÀqÐIÐIà% c U¨"¨Z¨L¸Ð:Ð:ØÐ*Ò*Ü�j¤%Ô(Ø)ÑˆD�"‘fä0°ÀÓLÑˆD�"�fØ! #  d¨4¨&°°2°$°b¸¸ÀÐDÐDØÐ-Ñ-ä�J Ô(Ø× Ñ ¤E§J¡JÒ.Ø×%Ò%àÐ Ø(Ð)¨°3°%Ð7LÈZÈLÐX[Ð\aÐ[bÐbcÐdÐdð &Ð& i°¨uÐ4IÈ*ÈÐUWÐXðð ÐØ$Ð% Y¨s¨e°2°j°\ÀÀEÀ7È!ÐLÐLà$Ð% Y¨s¨e°2°j°\ÀÐCÐCÜ
˜Ó
(Ð(rX   c                 óJ   — t        | «      r|› d�|› d�|› d�fS | dv r|› d�S |S )Nz.meanz.m2z.weightrz   z.index)r+   )rˆ   Úaccs     rY   r¥   r¥     sC   € Ü˜NÔ+Ø��eˆ}   S˜k¨c¨U°'¨?Ð:Ð:Ø	Ð/Ñ	/Ø��fˆ~ÐØ€JrX   ÚcodeÚiter_varÚnew_iter_varÚ
loop_startÚloop_endÚreturnc                 ób  — t        «       }t        j                  «       5 }|j                  dt        › d|› dt        |«      › d�|› dt        |«      › d|› d�z   «       |j                  |j                  «       «       t        | j                  «      D ]…  \  }}t        |t        t        f«      sJ ‚d}	t        |t        «      r|j                  }	|j                  }t        j                   d	|› z   d	z   |› |«      }
|	rt        |	|
«      }
|j                  |
«       Œ‡ 	 ddd«       |S # 1 sw Y   |S xY w)
aú  
    f(iter_var) is transformed to f(new_iter_var) under the inner loop
      \/
    for (new_iter_var = loop_start; new_iter_var < loop_end; new_iter_var++) {
        f(new_iter_var)
    }
    Please be careful while using this function,
    as the variable defined in f(iter_var) will be invalid outside the for loop.
    For example:
    auto tmp0 = in_ptr[x0]; ->
    for (new_x0 = start; new_x0 < end; new_x0++){
        auto tmp0 = in_ptr[new_x0];
    }
    The tmp0 is invalid outside the loop.
    zfor (r™   ú = ú;ú < ú; ú++)Nú\b)r8   Ú
contextlibÚ	ExitStackÚ	writelinerK   rF   Úenter_contextÚindentÚ	enumerateÚ_linesr£   rq   r<   ÚnameÚlineÚreÚsub)r´   rµ   r¶   r·   r¸   Útransformed_codeÚstackÚ_rÉ   Údeferred_nameÚnew_lines              rY   Úmove_code_under_inner_looprÑ     s?  € ô, $“~ÐÜ	×	Ñ	Ó	ð 1 5Ø×"Ñ"Ø”J�<˜q  ¨c´+¸jÓ2IÐ1JÈ!ÐLØ�˜c¤+¨hÓ"7Ð!8¸¸<¸.ÈÐLñMô	
ð 	×ÑÐ,×3Ñ3Ó5Ô6Ü  §¡Ó-ò 	1‰GˆAˆtÜØäÜ ðôð ð ð !ˆMÜ˜$¤Ô-Ø $§	¡	�Ø—y‘y�Ü—v‘v˜e¨¨
Ñ3°eÑ;À¸~ÐPTÓUˆHÙÜ'¨°xÓ@�Ø×&Ñ& xÕ0ñ	1÷1ð, Ð÷-1ð, Ðús   ŸC:D$Ä$D.Úacc_varÚacc_typerˆ   r‰   Úlenc                 óâ   — t        «       }t        j                  «       rd| › d|› d|› d�n
|› d| › d|› d�}|j                  |› «       |j	                  d|› d	�d
d| › d |||«      › d�dg«       |S )a˜  
    MSVC don't support dynamic array(VLA). So we use std::unique_ptr here.
    Ref: https://stackoverflow.com/questions/56555406/creating-dynamic-sized-array-using-msvc-c-compiler
    MSVC is the only one compiler without VLA. support. Since MSVC can't get good performance here.
    We just use unique_ptr make it works on MSVC.
    For other compilers, we continue to use VLA to get best performance.
    úauto z_arr = std::make_unique<z[]>(ú);r™   ú_arr[ú];úfor (int i = 0; i < ú; i++)ú{ú    z
_arr[i] = r¼   r�   )r>   r   Ú
is_msvc_clrÃ   Ú
writelines)rÒ   rÓ   rˆ   r‰   rÔ   Úinit_fnÚcode_bufferÚacc_decls           rY   Úreduction_prefix_arrayrã   >  s©   € ô !Ó"€Kô ×!Ñ!Ô#ð �ˆyÐ0°°
¸$¸s¸eÀ2ÑFàˆz˜˜7˜) 5¨¨¨RÐ0ð ð
 ×Ñ˜X˜JÔ(Ø×Ñà" 3 % vÐ.ØØ�7�)˜:¡g¨n¸eÓ&DÐ%EÀQÐGØð		
ôð ÐrX   ÚbufferrÈ   Únew_namec                 óD  — t        | j                  «      D ]ˆ  \  }}t        |t        t        f«      sJ ‚t        |t        «      r/t        j                  d|› z   dz   |› |j                  «      |_        Œ]t        j                  d|› z   dz   |› |«      | j                  |<   ŒŠ y )NrÀ   )rÆ   rÇ   r£   rq   r<   rÊ   rË   rÉ   )rä   rÈ   rå   ÚirÉ   s        rY   Úreplace_acc_namerè   _  sœ   € Ü˜VŸ]™]Ó+ò V‰ˆˆ4ÜØäÜðô
ð 	
ð 
ô �dœLÔ)ÜŸ™˜u¨$¨Ñ0°5Ñ8¸X¸JÈÏÉÓSˆD�Iä!Ÿv™v e°°Ñ&7¸%Ñ&?ÀHÀ:ÐPTÓUˆF�M‰M˜!ÒñVrX   c           
      ó”  — d}t        | j                  «      D ]®  \  }}t        |t        t        f«      sJ ‚t        |t        «      r|j
                  n|}t        j                  ||«      }|sŒU|j                  «       \  }}t        j                  ||› d|› d|› d�|«      }t        |t        «      r||_        Œ || j                  |<   Œ° y)zT
    Replaces `acc = cascade_sum_combine(value, ...)` with `acc = acc + value;`
    z/(.*?)\s*=\s*cascade_sum_combine\(([^,]+),.*?\);r»   ú + r¼   N)
rÆ   rÇ   r£   rq   r<   rÉ   rÊ   ÚsearchÚgroupsrË   )	rä   Úpatternrç   rÉ   ÚcontentÚmatchr³   ÚvalueÚnew_contents	            rY   Úreplace_cascade_sum_with_addrò   n  sÂ   € ð
 A€GÜ˜VŸ]™]Ó+ò /‰ˆˆ4ÜØäÜðô
ð 	
ð 
ô  *¨$´Ô=�$—)’)À4ˆÜ—	‘	˜' 7Ó+ˆÚØŸ™›‰JˆC�ÜŸ&™& ¨S¨E°°S°E¸¸U¸GÀ1Ð*EÀwÓOˆKÜ˜$¤Ô-Ø'�•	à#.�—‘˜aÒ ñ!/rX   c                   ó&   — e Zd ZU dZeed<   eed<   y)ÚParallelDepthz{
    A class representing parallel depth.
    Includes the starting depth of parallelism and the depth of parallelism.
    Úparallel_depthÚstart_depthN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ro   Ú__annotations__rW   rX   rY   rô   rô   ‡  s   … ñð
 ÓØÔrX   rô   c                   ó`   ‡ — e Zd Zededefd„«       Zdddeeez     fˆ fd„Z	d„ Z
d	„ Zd
„ Zˆ xZS )ÚOuterLoopFusedSchedulerNodeÚnode1Únode2c                 óž  — |j                   |j                   u sJ ‚t        d„ ||fD «       «      sJ ‚t        d„ ||fD «       «      rt | |j                   t        |«      t        u rt        |j                  «       «      n|gt        |«      t        u r t        |j                  «       «      z   |«      S |gz   |«      S  | |j                   ||g|«      S )Nc              3   óT   K  — | ]   }t        |«      t        t        t        fv –— Œ" y ­w©N)Útyperý   r$   r"   ©Ú.0Únodes     rY   ú	<genexpr>z3OuterLoopFusedSchedulerNode.fuse.<locals>.<genexpr>˜  s1   è ø€ ò 
ð ô �‹Jä+ÜÜ"ðôñ
ùó   ‚&(c              3   ó>   K  — | ]  }t        |«      t        u –— Œ y ­wr  ©r  rý   r  s     rY   r  z3OuterLoopFusedSchedulerNode.fuse.<locals>.<genexpr>¡  ó   è ø€ ÒT¸TŒt�D‹zÔ8Ô8ÑTùó   ‚)Ú	schedulerÚallrh   r  rý   ÚlistÚget_outer_nodes)Úclsrþ   rÿ   Úouter_loop_fusion_depths       rY   Úfusez OuterLoopFusedSchedulerNode.fuse“  s÷   € ð �‰ %§/¡/Ñ1Ð1Ð1Üñ 
ð  ˜ô
ô 
ð 	
ð 
ô ÑTÀeÈUÀ^ÔTÔTÙØ—‘ô ˜E“{Ô&AÑAô ˜×.Ñ.Ó0Ô1ð ðô ˜E“{Ô&AÑAô ˜×.Ñ.Ó0Ó1ñð (ó#ð ð ðñð (ó#ð ñ( �u—‘¨°¨Ð8OÓPÐPrX   r  r#   Úouter_fused_nodesc                 óè   •— || _         || _        g }| j                   D ]B  }t        |t        t        f«      sJ ‚|j                  t        |j                  «       «      «       ŒD t        ‰| �%  ||«       y r  )
r  r  r£   r$   r"   Úextendr  Ú	get_nodesÚsuperÚ__init__)Úselfr  r  r  Úflatten_snodesÚ_nodeÚ	__class__s         €rY   r  z$OuterLoopFusedSchedulerNode.__init__¸  su   ø€ ð ð 	Ôð (?ˆÔ$ØˆØ×+Ñ+ò 	;ˆEÜ˜e¤mÔ5GÐ%HÔIÐIÐIØ×!Ñ!¤$ u§¡Ó'8Ó"9Õ:ð	;ô 	‰Ñ˜ NÕ3rX   c                 ó   — | j                   S r  )r  ©r  s    rY   r  z+OuterLoopFusedSchedulerNode.get_outer_nodesÈ  s   € Ø×%Ñ%Ð%rX   c           
      ó8  ‡— dt         dt         dt        dt        dt        f
ˆfd„Št        t	        |«      dz
  «      D ]0  }||   j
                  }||dz      j
                  } ‰|||d«      rŒ0 y	 |D ]¦  }t        j                  t        j                  |j                  d | «      }t	        |j                  «      |kD  sŒMt        |t        j                  «      sŒht        |j                  |   t        j                  «      sŒ�|d
z  |j                  |   k  sŒ¦ y	 y)NÚleft_loop_nestÚright_loop_nestÚloop_fusion_depthÚcurrent_checking_depthr¹   c                 ó^  •‡‡— | j                   sJ ‚|j                   sJ ‚| j                   |   Š|j                   |   Šg d¢}t        ˆˆfd„|D «       «      sy|dk\  sJ ‚|dz
  x}dkD  rE|dz   }|t        | j                   «      k  sJ ‚|t        |j                   «      k  sJ ‚ ‰| |||«      syy)N)r¨   ÚsizeÚoffsetÚstepsc              3   óP   •K  — | ]  }t        ‰|«      t        ‰|«      k(  –— Œ y ­wr  )Úgetattr)r  Úattr_compareÚleft_loop_levelÚright_loop_levels     €€rY   r  zaOuterLoopFusedSchedulerNode.check_outer_fusion_loop_level_attr.<locals>._inner.<locals>.<genexpr>æ  s2   øè ø€ ò ð %ô ˜O¨\Ó:ÜÐ/°Ó>õ?ñùs   ƒ#&Fr6   r   T)Úloopsr  rÔ   )r!  r"  r#  r$  Úouter_loops_attr_compare_listr,  r-  Ú_inners        @@€rY   r0  zNOuterLoopFusedSchedulerNode.check_outer_fusion_loop_level_attr.<locals>._innerÔ  sç   ú€ ð "×'Ò'Ð'Ð'Ø"×(Ò(Ð(Ð(Ø,×2Ñ2Ð3IÑJˆOØ.×4Ñ4Ð5KÑLÐò-Ð)ô ô ð )Fôô ð à$¨Ò)Ð)Ð)Ø%6¸Ñ%:Ð:Ð!¸aÒ?à)?À!Ñ)CÐ&Ø-´°N×4HÑ4HÓ0IÒIÐIÐIØ-´°O×4IÑ4IÓ0JÒJÐJÐJÙØ"Ø#Ø%Ø*ô	ð !àrX   r6   r   Fé,  T)ÚLoopNestro   rm   ÚrangerÔ   Ú	loop_nestÚ	functoolsÚreduceÚoperatorÚmulÚrangesr£   ÚsympyÚInteger)	r  Úcpp_kernel_proxy_listr  Úidxr!  r"  Úcpp_kernel_proxyÚouter_rangesr0  s	           @rY   Ú"check_outer_fusion_loop_level_attrz>OuterLoopFusedSchedulerNode.check_outer_fusion_loop_level_attrË  s<  ø€ ð(	Ü$ð(	ä%ð(	ô  #ð(	ô %(ð	(	ô
 õ(	ôT œÐ2Ó3°aÑ7Ó8ò 		ˆCØ2°3Ñ7×AÑAˆNØ3°C¸!±GÑ<×FÑFˆOÙØØØ'Øõ	ñ ð		ð !6ò 	ÐÜ$×+Ñ+Ü—‘Ø ×'Ñ'Ð(@Ð)@ÐAóˆLô Ð$×+Ñ+Ó,Ð/FÓFÜ˜|¬U¯]©]Õ;ÜØ$×+Ñ+Ð,CÑDÜ—M‘Mõð ! 3Ñ&Ø"×)Ñ)Ð*AÑBóCñ ð)	ð, rX   c                 óD  — |d   j                   }t        |«      }|D �cg c]'  }|j                  j                  | j                  «      ‘Œ) c}|_        |d   }||j                  _        |j                  j                  d | j                   |j                  _        |S c c}w ©Nr   )Úkernel_groupÚOuterLoopFusedKernelr4  Úfrom_loop_levelr  ÚinnerÚkernelr.  )r  r<  rC  Úouter_loop_fused_kernelÚproxyÚouter_fused_proxys         rY   Úmerge_outer_fusion_kernelsz6OuterLoopFusedSchedulerNode.merge_outer_fusion_kernels!  s¦   € ð -¨QÑ/×<Ñ<ˆÜ"6°|Ó"DÐð /ö)
àð �O‰O×+Ñ+¨D×,HÑ,HÕIò)
ÐÔ%ð 2°!Ñ4ÐØ-DÐ×#Ñ#Ô*Ø,=×,GÑ,G×,MÑ,MØ*ˆd×*Ñ*ð-
Ð×#Ñ#Ô)ð !Ð ùò)
s   Ÿ,B)r÷   rø   rù   Úclassmethodr   r  r  r"   r$   r  r  r@  rK  Ú__classcell__©r  s   @rY   rý   rý   ’  s]   ø„ Øð"QØ%ð"QØ.?ò"Qó ð"QðH4àð4ð  Ð 2°]Ñ BÑCõ4ò &òTöl!rX   rý   c                   ó2   — e Zd Zddefd„Zd„ Zd„ Zd„ Zd„ Zy)	ÚRecordOptimizationContextÚ	func_namec                 ó.   — || _         d | _        d | _        y r  )rQ  Úcurrent_nodeÚopt_ctx)r  rQ  s     rY   r  z"RecordOptimizationContext.__init__4  s   € Ø"ˆŒØ26ˆÔØ37ˆ�rX   c                 óÒ  — t         j                  sJ ‚t         j                  j                  sJ ‚t         j                  j                  | _        | j                  €J ‚t        j                  | j                  j
                  v r-| j                  j
                  t        j                     | _        nt        «       | _        | j                  €J ‚| j                  | j                  _        | S r  )	r5   ÚinterpreterrS  rB   ÚkeyÚmetarT  rQ  Úops_namer  s    rY   Ú	__enter__z#RecordOptimizationContext.__enter__9  s­   € Ü�}Š}Ðˆ}Ü�}‰}×)Ò)Ð)Ð)äŸM™M×6Ñ6ˆÔØ× Ñ Ð,Ð,Ð,Ü×"Ñ" d×&7Ñ&7×&<Ñ&<Ñ<Ø×,Ñ,×1Ñ1Ô2E×2IÑ2IÑJˆD�Lä.Ó0ˆDŒLØ�|‰|Ð'Ð'Ð'Ø $§¡ˆ�‰ÔØˆrX   c                 óž   — | j                   sJ ‚| j                  sJ ‚| j                  | j                   j                  t        j                  <   y r  )rS  rT  rX  rB   rW  ©r  Úexc_typeÚexc_valÚexc_tbs       rY   Ú__exit__z"RecordOptimizationContext.__exit__G  s>   € Ø× Ò Ð Ð Ø�|Š|Ðˆ|Ø:>¿,¹,ˆ×Ñ×ÑÔ2×6Ñ6Ò7rX   c                 ó   — | j                   S r  )rT  r  s    rY   Úget_opt_ctxz%RecordOptimizationContext.get_opt_ctxL  s   € Ø�|‰|ÐrX   c                 ó6   — | j                   sJ ‚| j                   S r  )rS  r  s    rY   Úget_fx_nodez%RecordOptimizationContext.get_fx_nodeO  s   € Ø× Ò Ð Ð Ø× Ñ Ð rX   N)rT   )	r÷   rø   rù   rq   r  rZ  r`  rb  rd  rW   rX   rY   rP  rP  3  s#   „ ñ8 #ó 8ò
òGò
ó!rX   rP  c                  ór   — t        d„ | D «       «      rJ d«       ‚t        | «      x}�	t        |   S d| d   › d�S )Nc              3   óX   K  — | ]"  }t        |t        «      xr |j                  –— Œ$ y ­wr  )r£   rH   r§   )r  Úargs     rY   r  z$decltype_promoted.<locals>.<genexpr>U  s#   è ø€ ÒRÀc”:˜c¤>Ó2ÒA°s·z±zÓAÑRùs   ‚(*z*Promotion of vector types is not supportedú	decltype(r   r—   )rh   rJ   rI   )ÚargsÚdts     rY   Údecltype_promotedrk  T  sP   € ÜÑRÈTÔRÔRð Ø4óÐRô   Ó%Ð%ˆÐ2Ü˜BÑÐà˜4 ™7˜) 1Ð%Ð%rX   c                   óú  — e Zd ZdZed„ «       Zed„ «       Zed„ «       ZedZd„«       ZedZd„«       Z	ed„ «       Z
ed	„ «       Zed
„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Z ed„ «       Z!ed „ «       Z"ed!„ «       Z#ed"„ «       Z$ed#„ «       Z%ed$„ «       Z&ed%„ «       Z'ed&„ «       Z(ed'„ «       Z)ed(„ «       Z*ed)„ «       Z+ed*„ «       Z,ed+„ «       Z-ed,„ «       Z.ed-„ «       Z/ed.„ «       Z0ed/„ «       Z1ed0„ «       Z2ed1„ «       Z3ed2„ «       Z4ed3„ «       Z5ed4„ «       Z6ed5„ «       Z7ed6„ «       Z8ed7„ «       Z9ed8„ «       Z:ed9„ «       Z;ed:„ «       Z<ed;„ «       Z=ed<„ «       Z>ed=„ «       Z?ed>„ «       Z@ed?„ «       ZAed@„ «       ZBedA„ «       ZCedB„ «       ZDedC„ «       ZEedD„ «       ZFedE„ «       ZGedF„ «       ZHedG„ «       ZIedHeJj–                  dIeJj–                  fdJ„«       ZLedHeJj–                  dKeJj–                  dLeJj–                  dMeJj–                  dNeJj–                  f
dO„«       ZMedHeJj–                  dIeJj–                  fdP„«       ZNedHeJj–                  dIeJj–                  fdQ„«       ZOedR„ «       ZPedS„ «       ZQdTeRdUeRdVeSdWeTeReUf   dXdf
dY„ZVy)[ÚCppOverrideszMap element-wise ops to C++c                 ó,   — t        | |«      › d| › d|› d�S )Nú(rê   r—   ©rk  ©ÚaÚbs     rY   ÚaddzCppOverrides.addb  ó#   € ä# A qÓ)Ð*¨!¨A¨3¨c°!°°AÐ6Ð6rX   c                 ó,   — t        | |«      › d| › d|› d�S )Nro  ú - r—   rp  rq  s     rY   rË   zCppOverrides.subf  ru  rX   c                 ó,   — t        | |«      › d| › d|› d�S )Nro  rš   r—   rp  rq  s     rY   r8  zCppOverrides.mulj  ru  rX   Nc                 óÚ  — t        | t        «      sJ ‚|€| j                  }t        j                  j                  | ||«      }t        j                  j                  j                  t        j                  j                  |«      }|j                  d| |fd|i«       |rN|t        v rF|t        j                  k(  r3t        j                  s#	 t        j                  j                  | |||«       |S )NÚto_dtyper«   ©r£   rH   r‰   r5   rG  Úget_to_dtype_exprÚcseÚgenerateÚcomputeÚupdate_on_argsr„   r…   rp   r   Úemulate_precision_castsÚcache_dtype_convert©Úxr‰   r«   Úuse_compute_typesÚexprÚcsevars         rY   rz  zCppOverrides.to_dtypen  s½   € ä˜!œ^Ô,Ð,Ð,ØÐØŸ™ˆIÜ�x‰x×)Ñ)¨!¨U°IÓ>ˆÜ—‘—‘×&Ñ&¤q§x¡x×'7Ñ'7¸Ó>ˆØ×Ñ˜j¨1¨e¨*°{ÀIÐ6NÔOáØœÑ&ØœUŸ[™[Ò(Ü×2Ò2ð"ôF �H‰H×(Ñ(¨¨I°v¸uÔEØˆrX   c                 óØ  — t        | t        «      sJ ‚|€| j                  }t        j                  j                  | ||d¬«      }t        j                  j                  j                  t        j                  j                  |«      }|j                  d| |fd|i«       |t        v rE|t        j                  k(  r2t        j                  s"t        j                  j                  | |||«       |S )NT©ÚroundingÚround_to_intr«   r{  rƒ  s         rY   r‹  zCppOverrides.round_to_int¢  s¹   € ô ˜!œ^Ô,Ð,Ð,ØÐØŸ™ˆIÜ�x‰x×)Ñ)¨!¨U°IÈÐ)ÓMˆÜ—‘—‘×&Ñ&¤q§x¡x×'7Ñ'7¸Ó>ˆØ×Ñ˜n¨q°%¨j¸;È	Ð:RÔSà”]Ñ"ØœUŸ[™[Ò(Ü×2Ò2ä�H‰H×(Ñ(¨¨I°v¸uÔEØˆrX   c                 óT   — |t         v sJ |› dt        › d�«       ‚dt         |   › d| › d�S )Nz missing from z.DTYPE_TO_CPPzc10::bit_cast<ú>(r—   )rI   r÷   )r„  r‰   r«   s      rY   Úto_dtype_bitcastzCppOverrides.to_dtype_bitcast³  s=   € àœÑ$ÐU¨¨¨~¼h¸ZÀ}Ð&UÓUÐ$Ø¤¨UÑ 3Ð4°B°q°c¸Ð;Ð;rX   c                 ó   — d| › d�S )Nz	std::abs(r—   rW   ©r„  s    rY   ÚabszCppOverrides.abs¸  ó   € ð ˜1˜#˜QÐÐrX   c                 ó   — d| › d�S )Nz	std::sin(r—   rW   r�  s    rY   ÚsinzCppOverrides.sin½  r’  rX   c                 ó   — d| › d�S )Nz	std::cos(r—   rW   r�  s    rY   ÚcoszCppOverrides.cosÂ  r’  rX   c                 ó   — d| › d| › d�S )Nrh  z)(-r—   rW   r�  s    rY   ÚnegzCppOverrides.negÇ  ó   € à˜1˜#˜S   1Ð%Ð%rX   c                 ó   — d| › d�S )Nz	std::exp(r—   rW   r�  s    rY   ÚexpzCppOverrides.expË  s   € ð ˜1˜#˜QÐÐrX   c                 ó   — d| › d�S )Nz
std::exp2(r—   rW   r�  s    rY   Úexp2zCppOverrides.exp2Ñ  ó   € à˜A˜3˜aÐ Ð rX   c                 ó   — d| › d�S )Nzstd::expm1(r—   rW   r�  s    rY   Úexpm1zCppOverrides.expm1Õ  ó   € à˜Q˜C˜qÐ!Ð!rX   c                 ó   — d| › d�S )Nz	std::erf(r—   rW   r�  s    rY   ÚerfzCppOverrides.erfÙ  r’  rX   c                 ó   — d| › d�S )Nz
std::erfc(r—   rW   r�  s    rY   ÚerfczCppOverrides.erfcÞ  rž  rX   c                 ó   — d| › d�S )Nzcalc_erfinv(r—   rW   r�  s    rY   ÚerfinvzCppOverrides.erfinvâ  ó   € ð ˜a˜S Ð"Ð"rX   c                 ó   — d| › d�S )Nz
std::sqrt(r—   rW   r�  s    rY   ÚsqrtzCppOverrides.sqrtç  ó   € ð ˜A˜3˜aÐ Ð rX   c                 ó   — d| › d�S )Nz1 / std::sqrt(r—   rW   r�  s    rY   ÚrsqrtzCppOverrides.rsqrtì  s   € ð   ˜s !Ð$Ð$rX   c                 ó|   — t         j                  j                  }|dk(  r| › d| › d�S |€d| › d�S t        d|›�«      ‚)NÚaccuracyú + decltype(ú)(1)zstd::log1p(r—   ú8unrecognized config cpp.inject_log1p_bug_TESTING_ONLY = ©r   ÚcppÚinject_log1p_bug_TESTING_ONLYr‡   ©r„  Úbugs     rY   Úlog1pzCppOverrides.log1pñ  sW   € ä�j‰j×6Ñ6ˆØ�*ÒØ�S˜ Q C tÐ,Ð,Øˆ[Ø    1Ð%Ð%ä ØJÈ3È'ÐRóð rX   c                 ó   — d| › d�S )Nz	std::tan(r—   rW   r�  s    rY   ÚtanzCppOverrides.taný  r’  rX   c                 ó   — d| › d�S )Nz
std::tanh(r—   rW   r�  s    rY   ÚtanhzCppOverrides.tanh  r«  rX   c                 ó&   — t         rd| › d�S d| › d�S )z§
        On windows std::signbit only support float type.
        Ref: https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/signbit?view=msvc-170
        z std::signbit(static_cast<float>(r¢   zstd::signbit(r—   rU   r�  s    rY   ÚsignbitzCppOverrides.signbit  s-   € õ ð /¨q¨c°Ð4ð	
ð !   1Ð%ð	
rX   c                 ó   — d| › d|› d�S )Nz	std::pow(r�   r—   rW   rq  s     rY   ÚpowzCppOverrides.pow  s   € à˜1˜#˜R ˜s !Ð$Ð$rX   c                 ó   — d| › d�S )Nz	std::log(r—   rW   r�  s    rY   ÚlogzCppOverrides.log  r’  rX   c                 ó   — d| › d�S )Nzstd::nearbyint(r—   rW   r�  s    rY   ÚroundzCppOverrides.round  s   € ð !   1Ð%Ð%rX   c                 ó   — d| › d�S )Nzstd::floor(r—   rW   r�  s    rY   ÚfloorzCppOverrides.floor"  ó   € ð ˜Q˜C˜qÐ!Ð!rX   c                 ó   — d| › d|› d�S )Núfloor_divide_integral(r�   r—   rW   rq  s     rY   ÚfloordivzCppOverrides.floordiv'  s   € ð (¨ s¨"¨Q¨C¨qÐ1Ð1rX   c                 ó   — d| › d�S )Nz
std::ceil(r—   rW   r�  s    rY   ÚceilzCppOverrides.ceil,  r«  rX   c                 ó   — d| › d�S )Nzstd::trunc(r—   rW   r�  s    rY   ÚtrunczCppOverrides.trunc1  rÇ  rX   c                 ó   — | › d|› �S ©Nú / rW   rq  s     rY   ÚtruncdivzCppOverrides.truncdiv6  s   € ð ��C˜�sˆ|ÐrX   c                 ó   — d| › d|› d�S )Nz
std::fmod(r�   r—   rW   rq  s     rY   ÚfmodzCppOverrides.fmod<  s   € ð ˜A˜3˜b   1Ð%Ð%rX   c                 ó   — d| › d�S )Nzstd::isinf(r—   rW   r�  s    rY   ÚisinfzCppOverrides.isinfA  rÇ  rX   c                 ó   — d| › d�S )Nzstd::isnan(r—   rW   r�  s    rY   ÚisnanzCppOverrides.isnanF  rÇ  rX   c                 ó   — d| › d�S )Nzstd::lgamma(r—   rW   r�  s    rY   ÚlgammazCppOverrides.lgammaK  r¨  rX   c                 ó   — d| › d�S )Nz
std::acos(r—   rW   r�  s    rY   ÚacoszCppOverrides.acosP  r«  rX   c                 ó   — d| › d�S )Nzstd::acosh(r—   rW   r�  s    rY   ÚacoshzCppOverrides.acoshU  rÇ  rX   c                 ó   — d| › d�S )Nz
std::cosh(r—   rW   r�  s    rY   ÚcoshzCppOverrides.coshZ  r«  rX   c                 ó   — d| › d�S )Nz
std::sinh(r—   rW   r�  s    rY   ÚsinhzCppOverrides.sinh_  r«  rX   c                 ó   — d| › d�S )Nz
std::asin(r—   rW   r�  s    rY   ÚasinzCppOverrides.asind  r«  rX   c                 ó   — d| › d�S )Nzstd::asinh(r—   rW   r�  s    rY   ÚasinhzCppOverrides.asinhi  rÇ  rX   c                 ó   — d| › d|› d�S )Nzstd::atan2(r�   r—   rW   ©r„  Úys     rY   Úatan2zCppOverrides.atan2n  ó   € ð ˜Q˜C˜r !  AÐ&Ð&rX   c                 ó   — d| › d�S )Nz
std::atan(r—   rW   r�  s    rY   ÚatanzCppOverrides.atans  r«  rX   c                 ó   — d| › d�S )Nzstd::atanh(r—   rW   r�  s    rY   ÚatanhzCppOverrides.atanhx  rÇ  rX   c                 ó   — d| › d|› d�S )Nzstd::copysign(r�   r—   rW   rè  s     rY   ÚcopysignzCppOverrides.copysign}  s   € ð   ˜s " Q C qÐ)Ð)rX   c           	      ó¸  — d| › d�d| › d�f}t        d„ |D «       «      rt        d„ |D «       «      S t        «       }t        j                  j
                  j                  t        j                  | j                  ¬«      }t        j                  j
                  j                  | j                  | j                  ¬«      }|j                  d|› d�«       |j                  d	|› d
| › d|› d�«       t        j                  j                  j                  |«       ||f}t        ||«      D ]/  \  }}t        j                  j
                  j                  ||«       Œ1 ||fS )Núfrexp(ú)[0]ú)[1]c              3   ór   K  — | ]/  }t         j                  j                  j                  |«      d u–— Œ1 y ­wr  ©r5   rG  r}  Útry_get©r  Ú	cache_keys     rY   r  z%CppOverrides.frexp.<locals>.<genexpr>†  ó(   è ø€ ÒW¸yŒq�x‰x�|‰|×#Ñ# IÓ.°dÔ:ÑWùó   ‚57c              3   ón   K  — | ]-  }t         j                  j                  j                  |«      –— Œ/ y ­wr  r÷  rù  s     rY   r  z%CppOverrides.frexp.<locals>.<genexpr>‡  ó#   è ø€ ÒU¸YœŸ™Ÿ™×-Ñ-¨i×8ÑUùó   ‚35)r‰   Úshapezint32_t r¼   rÖ   z = std::frexp(r–   r×   )r  r¤   r8   r5   rG  r}  Únewvarr…   Úint32r   r‰   rÃ   r  ÚspliceÚzipÚput)r„  Ú
cache_keysr´   ÚexponentÚmantissaÚcse_varsrú  Úcse_vars           rY   ÚfrexpzCppOverrides.frexp‚  s&  € ð ˜a˜S Ð%¨°¨s°$Ð'7Ð7ˆ
ÜÑWÈJÔWÔWÜÑUÈ*ÔUÓUÐUä‹~ˆÜ—8‘8—<‘<×&Ñ&¬U¯[©[ÀÇÁÐ&ÓHˆÜ—8‘8—<‘<×&Ñ&¨Q¯W©W¸A¿G¹GÐ&ÓDˆØ�‰˜ ( ¨1Ð-Ô.Ø�‰˜˜x˜j¨°q°c¸¸X¸JÀbÐIÔJÜ	�‰×Ñ×Ñ Ô%Ø˜hÐ'ˆÜ"% j°(Ó";ò 	1ÑˆI�wÜ�H‰H�L‰L×Ñ˜Y¨Õ0ð	1à˜Ð!Ð!rX   c                 ó   — d| › d|› d�S )Nzstd::hypot(r�   r—   rW   rè  s     rY   ÚhypotzCppOverrides.hypot”  rë  rX   c                 ó   — d| › d�S )Nzstd::log10(r—   rW   r�  s    rY   Úlog10zCppOverrides.log10™  r¡  rX   c                 ó   — d| › d�S )Nz
std::log2(r—   rW   r�  s    rY   Úlog2zCppOverrides.log2�  rž  rX   c                 ó   — d| › d|› d�S )Nzstd::ldexp(r�   r—   rW   )r„  Úns     rY   ÚldexpzCppOverrides.ldexp¡  rë  rX   c                 ó   — d| › d|› d�S )Nzstd::nextafter(r�   r—   rW   rè  s     rY   Ú	nextafterzCppOverrides.nextafter¦  s   € ð !   2 a S¨Ð*Ð*rX   c                 ó¢   — t         j                  j                  }|dk(  ry|dk(  r| › d�S |dk(  r| › d| › d�S |€	d| › d	| › d
�S t        d|›�«      ‚)NÚcompile_errorúcompile error!Úruntime_errorú	; throw 1r¯  r°  r±  z	std::max(ú, decltype(ú)(0))ú7unrecognized config cpp.inject_relu_bug_TESTING_ONLY = ©r   r´  Úinject_relu_bug_TESTING_ONLYr‡   r¶  s     rY   ÚreluzCppOverrides.relu«  s|   € ä�j‰j×5Ñ5ˆØ�/Ò!Ø#Ø�OÒ#Ø�S˜	�?Ð"Ø�JÒØ�S˜ Q C tÐ,Ð,Øˆ[Ø˜q˜c ¨Q¨C¨uÐ5Ð5ä ØIÈ#ÈÐQóð rX   c                 ó   — d| › d|› d�S )Nzmin_propagate_nan(r�   r—   rW   rq  s     rY   ÚminimumzCppOverrides.minimum»  ó   € ð $ A 3 b¨¨¨1Ð-Ð-rX   c                 ó   — d| › d|› d�S )Nzmax_propagate_nan(r�   r—   rW   rq  s     rY   ÚmaximumzCppOverrides.maximumÀ  r$  rX   c                 ó   — | › d|› d|› �S )Nú ? ú : rW   )rr  rs  Úcs      rY   ÚwherezCppOverrides.whereÅ  s   € ð ��C˜�s˜#˜a˜SÐ!Ð!rX   c                 ó   — d| › d|› d�S )Nzmod(r�   r—   rW   rq  s     rY   ÚmodzCppOverrides.modÊ  s   € à�a�S˜˜1˜#˜QÐÐrX   c                 ó(   — t        | t        |   «      S r  )rQ   rI   )Úvalr‰   s     rY   ÚconstantzCppOverrides.constantÎ  s   € ä˜C¤¨eÑ!4Ó5Ð5rX   c                 ó  — t        t        j                  j                  | «      «      }t        j                  j                  j                  t        j                  j                  |t        | «      ¬«      }t        j                  ||«      S )N©Úbounds)
rE   r5   rG  Úrename_indexingr}  r~  r  r'   r3   rz  )r†  r‰   Úidx_strr¨   s       rY   Ú
index_exprzCppOverrides.index_exprÒ  sb   € äœŸ™×0Ñ0°Ó6Ó7ˆÜ�h‰h�l‰l×#Ñ#Ü�H‰H×Ñ˜gÔ.CÀDÓ.Ið $ó 
ˆô �|‰|˜C Ó'Ð'rX   c                 ó.   — t         j                  | |«      S r  )rm  r6  ©r†  r‰   s     rY   Ú
value_exprzCppOverrides.value_exprÚ  s   € ô ×&Ñ& t¨UÓ3Ð3rX   c                 ó"  — t        «       }t        j                  j                  j	                  «       }|j                  d|› d�«       t        j                  j                  |«      5  |j                  «       5   |«       }|j                  d|› d�«       d d d «       d d d «       |j                  d«       t        j                  j                  j                  |«       t        |d|› d�«      }| › d|› d|› �S # 1 sw Y   ŒexY w# 1 sw Y   ŒixY w)	NrÖ   ú = [&]úreturn r¼   rh  z())r(  z() : )r8   r5   rG  r}  r  rÃ   Úswap_buffersrÅ   r  r  rQ   )ÚmaskÚbodyÚotherr´   Úbody_varÚresultÚ
other_codes          rY   ÚmaskedzCppOverrides.maskedà  sê   € ä‹~ˆô —8‘8—<‘<×&Ñ&Ó(ˆØ�‰˜˜x˜j¨Ð/Ô0Ü�X‰X×"Ñ" 4Ó(ñ 	0¨$¯+©+«-ñ 	0Ù“VˆFØ�N‰N˜W V H¨AÐ.Ô/÷	0÷ 	0ð 	�‰�sÔÜ	�‰×Ñ×Ñ Ô%ô " %¨9°X°J¸cÐ)BÓCˆ
Ø��s˜8˜* E¨*¨Ð6Ð6÷	0ð 	0ú÷ 	0ð 	0ús$   Á'DÁ8C9ÂDÃ9D	Ã>DÄDc                 ó   — | › d|› �S )Nú && rW   rq  s     rY   Úlogical_andzCppOverrides.logical_andñ  ó   € ð ��D˜˜ˆ}ÐrX   c                 ó   — d| › �S )Nú!rW   ©rr  s    rY   Úlogical_notzCppOverrides.logical_notö  ó   € à�1�#ˆwˆrX   c                 ó   — | › d|› �S )Nrœ   rW   rq  s     rY   Ú
logical_orzCppOverrides.logical_orú  rH  rX   c                 ó   — | › d|› �S )Nú != rW   rq  s     rY   Úlogical_xorzCppOverrides.logical_xorÿ  rH  rX   c                 ó   — d| › d| › d|› d�S )Nrh  ú)(ú & r—   rW   rq  s     rY   Úbitwise_andzCppOverrides.bitwise_and  ó   € à˜1˜#˜R ˜s # a S¨Ð*Ð*rX   c                 ó   — d| › d| › d�S )Nrh  z)(~r—   rW   rK  s    rY   Úbitwise_notzCppOverrides.bitwise_not  r™  rX   c                 ó   — d| › d| › d|› d�S )Nrh  rT  ú | r—   rW   rq  s     rY   Ú
bitwise_orzCppOverrides.bitwise_or  rW  rX   c                 ó   — d| › d| › d|› d�S )Nrh  rT  r›   r—   rW   rq  s     rY   Úbitwise_xorzCppOverrides.bitwise_xor  rW  rX   c                 óä  — t        «       }|j                  d«       |j                  «       5  t        | j                     }|j                  d|› d|› d�«       |j                  d|› d|› d|› d�«       |j                  «       5  |j                  d	| › d
�«       d d d «       |j                  d	| › d|› d| › d|› d�	«       d d d «       |j                  d«       |S # 1 sw Y   ŒBxY w# 1 sw Y   Œ(xY w)Nú[&]()úconstexpr decltype(ú) max_shift = sizeof(z) * CHAR_BIT;ú$if ((static_cast<std::make_signed_t<ú>>(ú) < 0) || (ú >= max_shift))úreturn decltype(z)(0);z#)(static_cast<std::make_unsigned_t<z) << r×   ú()©r8   rÃ   rÅ   rI   r‰   ©rr  rs  r´   Úscalar_ts       rY   Úbitwise_left_shiftzCppOverrides.bitwise_left_shift  s  € ä‹~ˆØ�‰�wÔØ�[‰[‹]ñ 	Ü# A§G¡GÑ,ˆHØ�N‰NØ% a SÐ(=¸h¸ZÀ}ÐUôð �N‰NØ6°x°jÀÀAÀ3ÀkÐRSÐQTÐTcÐdôð —‘“ñ <Ø—‘Ð!1°!°°EÐ:Ô;÷<à�N‰NØ" 1 #Ð%HÈÈ
ÐRUÐVWÐUXÐX]Ð^_Ð]`Ð`bÐcô÷	ð 	�‰�tÔØˆ÷<ð <ú÷	ð 	ús$   ¬AC&ÂCÂ&C&ÃC#	ÃC&Ã&C/c           
      óê  — t        «       }|j                  d«       |j                  «       5  t        | j                     }|j                  d|› d|› d|› d�«       |j                  d|› d|› d|› d	�«       |j                  «       5  |j                  d
| › d| › d�«       d d d «       |j                  d
| › d| › d|› d�«       d d d «       |j                  d«       |S # 1 sw Y   Œ?xY w# 1 sw Y   Œ(xY w)Nr`  ra  rb  z ) * CHAR_BIT - std::is_signed_v<z>;rc  rd  re  rf  rg  rT  z >> max_shift);ú >> r×   rh  ri  rj  s       rY   Úbitwise_right_shiftz CppOverrides.bitwise_right_shift(  s  € ä‹~ˆØ�‰�wÔØ�[‰[‹]ñ 
	AÜ# A§G¡GÑ,ˆHØ�N‰NØ% a SÐ(=¸h¸ZÐGgÐhpÐgqÐqsÐtôð �N‰NØ6°x°jÀÀAÀ3ÀkÐRSÐQTÐTcÐdôð —‘“ñ KØ—‘Ð!1°!°°B°q°c¸ÐIÔJ÷Kà�N‰NÐ-¨a¨S°°1°#°T¸!¸¸BÐ?Ô@÷
	Að 	�‰�tÔØˆ÷	Kð Kú÷
	Að 
	Aús$   ¬AC)ÂCÂ#C)ÃC&	Ã"C)Ã)C2Úseedr'  c                 ó   — d| › d|› d�S ©Nznormalized_rand_cpu(r�   r—   rW   ©rp  r'  s     rY   ÚrandzCppOverrides.rand:  s   € à% d V¨2¨f¨X°QÐ7Ð7rX   Úbase_offsetÚthreads_per_roundÚtidÚvecc                 ó   — d| › d|› d�S rr  rW   )rp  ru  rv  rw  rx  s        rY   Ú
rand_eagerzCppOverrides.rand_eager>  s   € ð & d V¨2¨k¨]¸!Ð<Ð<rX   c                 ó   — d| › d|› d�S )Nz
randn_cpu(r�   r—   rW   rs  s     rY   ÚrandnzCppOverrides.randnK  s   € à˜D˜6  F 8¨1Ð-Ð-rX   c           	      ó    — d| › d|› d|› d|› d�	S )Nzrandint64_cpu(r�   r—   rW   )rp  r'  ÚlowÚhighs       rY   Ú	randint64zCppOverrides.randint64O  s#   € à ˜v R¨ x¨r°#°°b¸¸¸aÐ@Ð@rX   c                 ó   — d| › d| › d| › d�S )Nrh  z)(1) / (decltype(z)(1) + std::exp(-r¢   rW   r�  s    rY   ÚsigmoidzCppOverrides.sigmoidS  s   € à˜1˜#Ð.¨q¨cÐ1BÀ1À#ÀRÐHÐHrX   c           
      óN  — t        «       }d| › d�}d| › d�}|j                  d«       |j                  «       5  |j                  d| › d|› d|› d�«       |j                  d	| › d
|› d|› d�«       |j                  d«       d d d «       |j                  d«       |S # 1 sw Y   ŒxY w)Nrh  ú)(0)r±  r`  úauto left = z > 0 ? r)  r¼   úauto right = z < 0 ? úreturn left - right;rh  ©r8   rÃ   rÅ   )r„  r´   Úscalar_zeroÚ
scalar_ones       rY   ÚsignzCppOverrides.signW  s´   € ô ‹~ˆØ! !  DÐ)ˆØ    4Ð(ˆ
Ø�‰�wÔØ�[‰[‹]ñ 	3Ø�N‰N˜\¨!¨¨G°J°<¸sÀ;À-ÈqÐQÔRØ�N‰N˜]¨1¨#¨W°Z°LÀÀKÀ=ÐPQÐRÔSØ�N‰NÐ1Ô2÷	3ð 	�‰�tÔØˆ÷	3ð 	3ús   ¸ABÂB$rÈ   rˆ   rð   Ú
extra_metar¹   c                 ó   — t         ‚r  )ÚNotImplementedError)r  rÈ   rˆ   rð   rŒ  s        rY   Úpartial_accumulatezCppOverrides.partial_accumulatee  s
   € ô "Ð!rX   ©NT)Wr÷   rø   rù   rú   Ústaticmethodrt  rË   r8  rz  r‹  rŽ  r‘  r”  r–  r˜  r›  r�  r   r£  r¥  r§  rª  r­  r¸  rº  r¼  r¾  rÀ  rÂ  rÄ  rÆ  rÊ  rÌ  rÎ  rÒ  rÔ  rÖ  rØ  rÚ  rÜ  rÞ  rà  râ  rä  ræ  rê  rí  rï  rñ  r  r  r  r  r  r  r!  r#  r&  r+  r-  r0  r6  r9  rD  rG  rL  rO  rR  rV  rY  r\  r^  rl  ro  r:  ÚExprrt  rz  r|  r€  r‚  r‹  rq   r:   Údictr   r�  rW   rX   rY   rm  rm  _  s§  „ Ù%àñ7ó ð7ð ñ7ó ð7ð ñ7ó ð7ð ò1ó ð1ðf òó ðð ñ<ó ð<ð ñ ó ð ð ñ ó ð ð ñ ó ð ð ñ&ó ð&ð ñ ó ð ð ñ!ó ð!ð ñ"ó ð"ð ñ ó ð ð ñ!ó ð!ð ñ#ó ð#ð ñ!ó ð!ð ñ%ó ð%ð ñ	ó ð	ð ñ ó ð ð ñ!ó ð!ð ñ	
ó ð	
ð ñ%ó ð%ð ñ ó ð ð ñ&ó ð&ð ñ"ó ð"ð ñ2ó ð2ð ñ!ó ð!ð ñ"ó ð"ð ñó ðð ñ&ó ð&ð ñ"ó ð"ð ñ"ó ð"ð ñ#ó ð#ð ñ!ó ð!ð ñ"ó ð"ð ñ!ó ð!ð ñ!ó ð!ð ñ!ó ð!ð ñ"ó ð"ð ñ'ó ð'ð ñ!ó ð!ð ñ"ó ð"ð ñ*ó ð*ð ñ"ó ð"ð  ñ'ó ð'ð ñ"ó ð"ð ñ!ó ð!ð ñ'ó ð'ð ñ+ó ð+ð ñó ðð ñ.ó ð.ð ñ.ó ð.ð ñ"ó ð"ð ñ ó ð ð ñ6ó ð6ð ñ(ó ð(ð ñ4ó ð4ð
 ñ7ó ð7ð  ñó ðð ñó ðð ñó ðð ñó ðð ñ+ó ð+ð ñ&ó ð&ð ñ+ó ð+ð ñ+ó ð+ð ñó ðð& ñó ðð" ð8�5—:‘:ð 8 u§z¡zò 8ó ð8ð ð
=Ø�j‰jð
=à—Z‘Zð
=ð !Ÿ:™:ð
=ð �Z‰Zð	
=ð
 �Z‰Zò
=ó ð
=ð ð.�E—J‘Jð .¨¯
©
ò .ó ð.ð ðA˜Ÿ
™
ð A¨E¯J©Jò Aó ðAð ñIó ðIð ñ
ó ð
ð"àð"ð ð"ð ð	"ð
 ˜˜c˜‘Nð"ð 
ô"rX   rm  r´  c                   ó6  ‡ — e Zd ZdZˆ fd„Zed„ «       Zed„ «       Zed„ «       Zed„ «       Z	ed„ «       Z
ed„ «       Zed	„ «       Zed
„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Z ed„ «       Z!ed„ «       Z"ed „ «       Z#ed!„ «       Z$ed"„ «       Z%ed#„ «       Z&ed$„ «       Z'ed%„ «       Z(ed&„ «       Z)ed'„ «       Z*ed(„ «       Z+ed)„ «       Z,ed*„ «       Z-ed+„ «       Z.ed,„ «       Z/ed-„ «       Z0ed.„ «       Z1ed/„ «       Z2ed0„ «       Z3ed1„ «       Z4ed2„ «       Z5ed3„ «       Z6ed4„ «       Z7ed5„ «       Z8ed6„ «       Z9ed7„ «       Z:ed8„ «       Z;ed9„ «       Z<ed:„ «       Z=ed;„ «       Z>ed<„ «       Z?ed=„ «       Z@ed>„ «       ZAed?„ «       ZBed@„ «       ZCedA„ «       ZDedB„ «       ZEedC„ «       ZFedD„ «       ZGedE„ «       ZHedF„ «       ZIedG„ «       ZJedH„ «       ZKedI„ «       ZLedJ„ «       ZMedTdK„«       ZNedTdL„«       ZOedM„ «       ZPedN„ «       ZQedO„ «       ZRedP„ «       ZSedQ„ «       ZTeUdR„ «       ZVeUdS„ «       ZWˆ xZXS )UÚCppVecOverridesz.Map element-wise ops to aten vectorization C++c                 óê   •‡— t         ‰| �  | «      Šˆˆfd„}t        t        «      j	                  «       D ];  \  }}t        |dd «      t        u sŒ|dvsŒt        ‰| ||j                  «      «       Œ= ‰S )Nc                 ó   •‡ — ˆˆ ˆfd„}|S )Nc                  óî  •— | D �cg c]@  }t        |t        t        j                  f«      st        |t        «      r|j
                  s|‘ŒB }}| D �cg c]   }t        |t        «      r|j
                  r|‘Œ" }}t        | «      }|rÉ|rÇg }| D ]À  }t        |t        t        j                  f«      r�t        |t        j                  «      r1|j                  s%t        j                  |t        j                  «      }n$t        j                  |t        j                  «      }t        |t        «      r|j                  n|}|j                  |«       ŒÂ |r=t!        |«      dk(  rt#        |«      }n#‰
t$        j&                  u rt#        |dd  «      |dd  |r�|r›t        t(        j*                  t,        «      sJ ‚|D �cg c]p  }t        |t        «      r\|j
                  sP‰
t$        j.                  t$        j0                  t$        j2                  fvrt(        j*                  j5                  |«      n|‘Œr }}|r ‰
|i |¤ŽS t7        t$        ‰«      }t9        |‰
j:                  «      }|€J ‚ || i |¤ŽS c c}w c c}w c c}w )Nr   r6   )r£   ro   r:  r’  rH   r§   r  Ú	is_numberr3   r6  r…   Úint64r0  r4   rð   ÚappendrÔ   rN   r•  r+  r5   rG  ÚCppVecKernelrt  r|  r€  Ú	broadcastr  r*  r÷   )ri  Úkwargsrg  ÚscalarsÚvectorsÚnew_argsÚnew_argÚ
scalar_opsÚscalar_funcr  Úfuncr  s            €€€rY   Úwrapperz6CppVecOverrides.__new__.<locals>.wrap.<locals>.wrapper…  s8  ø€ ð  $öàÜ! #¬¬U¯Z©ZÐ'8Ô9Ü" 3¬Ô7ÀÇ
Â
ò ð�ð ð  $öàÜ! #¤~Ô6¸3¿:º:ò ð�ð ô
   ›:�Ù™wØ!�HØ#ò -˜Ü% c¬C´·±Ð+<Ô=Ü)¨#¬u¯z©zÔ:À3Ç=Â=Ü&)§n¡n°S¼%¿+¹+Ó&F¡ä&)§l¡l°3¼¿¹Ó&D Ü/9¸#¼xÔ/H #§)¢)Èc˜CØ Ÿ™¨Õ,ð-ñ ô ˜8“}¨Ò)Ü#/°Ó#9™Ø¤×!6Ñ!6Ñ6Ü'3°H¸Q¸R°LÓ'A˜  ˜ñ ™wÜ%¤a§h¡h´Ô=Ð=Ð=ð  (0ö ð $ô !+¨7´NÔ CØ(/¯ªØ$(ä$3×$8Ñ$8Ü$3×$9Ñ$9Ü$3×$=Ñ$=ð("ñ%"ô	 ŸH™H×.Ñ.¨wÔ7ð ")ñ)ð �Hð  ñ$ Ù Ð4¨VÑ4Ð4ô "'¤¸Ó!=�JÜ")¨*°d·m±mÓ"D�KØ&Ð2Ð2Ð2Ù&¨Ð7°Ñ7Ð7ùòùòùò@ s   †AI(Á%I-Æ5A5I2rW   )r¥  r¦  r  r  s   ` €€rY   Úwrapz%CppVecOverrides.__new__.<locals>.wrapx  s   ù€ ö@8ðD ˆNrX   r  )rD  r6  r9  )	r  Ú__new__Úvarsr•  Úitemsr*  r‘  ÚsetattrÚ__func__)r  ri  Úkargsr§  rÈ   Úmethodr  r  s         @€rY   r¨  zCppVecOverrides.__new__u  sx   ù€ Ü‰w‰˜sÓ#ˆõO	ôb !¤Ó1×7Ñ7Ó9ò 	;‰LˆD�&Ü�v˜{¨DÓ1´\ÒAÀdð Sò Gô
 ˜˜d¡D¨¯©Ó$9Õ:ð	;ð ˆrX   c                 ó   — | › d|› �S )Nrê   rW   rq  s     rY   rt  zCppVecOverrides.addÓ  ó   € à��C˜�sˆ|ÐrX   c                 ó   — | › d|› �S )Nrw  rW   rq  s     rY   rË   zCppVecOverrides.sub×  r°  rX   c                 ó   — | › d|› �S ©Nrš   rW   rq  s     rY   r8  zCppVecOverrides.mulÛ  r°  rX   c                 ó   — | › d|› �S rÐ  rW   rq  s     rY   ÚtruedivzCppVecOverrides.truedivß  r°  rX   c                 ó   — | › d�S )Nz.abs()rW   r�  s    rY   r‘  zCppVecOverrides.absã  ó   € à��Fˆ|ÐrX   c                 ó   — | › d�S )Nz.sin()rW   r�  s    rY   r”  zCppVecOverrides.sinç  r·  rX   c                 ó   — | › d�S )Nz.cos()rW   r�  s    rY   r–  zCppVecOverrides.cosë  r·  rX   c                 ó   — | › d�S )Nz.exp()rW   r�  s    rY   r›  zCppVecOverrides.expï  r·  rX   c                 ó   — | › d�S )Nz.exp2()rW   r�  s    rY   r�  zCppVecOverrides.exp2ó  ó   € à��Gˆ}ÐrX   c                 ó   — d| › d�}| › d|› �S )Nrh  r±  z	.exp() - rW   )r„  Úvec_ones     rY   r   zCppVecOverrides.expm1÷  s#   € ð ˜a˜S Ð%ˆØ��I˜g˜YÐ'Ð'rX   c                 ó   — | › d�S )Nz.erf()rW   r�  s    rY   r£  zCppVecOverrides.erfý  r·  rX   c                 ó   — | › d�S )Nz.erfc()rW   r�  s    rY   r¥  zCppVecOverrides.erfc  r¼  rX   c                 ó   — | › d�S )Nz	.erfinv()rW   r�  s    rY   r§  zCppVecOverrides.erfinv  ó   € à��IˆÐrX   c                 ó   — | › d�S )Nz.sqrt()rW   r�  s    rY   rª  zCppVecOverrides.sqrt	  r¼  rX   c                 óæ   — t        t        j                  t        «      sJ ‚t        | t        «      sJ ‚| j
                  €J ‚t        j                  j                  | j
                  «      › d| › d|› d�S )Nro  ú == r—   ©r£   r5   rG  rœ  rH   r‰   Ú_get_mask_typerè  s     rY   ÚeqzCppVecOverrides.eq  óc   € äœ!Ÿ(™(¤LÔ1Ð1Ð1Ü˜!œ^Ô,Ð,Ð,Ø�w‰wÐ"Ð"Ð"Ü—(‘(×)Ñ)¨!¯'©'Ó2Ð3°1°Q°C°t¸A¸3¸aÐ@Ð@rX   c                 ó¾  — t        t        j                  t        «      sJ ‚t        | t        «      sJ ‚| j
                  t        j                  k(  rO|j
                  t        j                  k(  sJ ‚t        t        j                  j                  | |f«      \  }}|› d|› �S | j
                  €J ‚t        j                  j                  | j
                  «      › d| › d|› d�S )NrQ  ro  r—   )r£   r5   rG  rœ  rH   r‰   r…   rm   rP   r  rÇ  )r„  ré  Úx_castÚy_casts       rY   ÚnezCppVecOverrides.ne  s¸   € äœ!Ÿ(™(¤LÔ1Ð1Ð1Ü˜!œ^Ô,Ð,Ð,Ø�7‰7”e—j‘jÒ Ø—7‘7œeŸj™jÒ(Ð(Ð(Ü1´!·(±(×2BÑ2BÀQÈÀFÓK‰NˆF�FØ�X˜T & Ð*Ð*à—7‘7Ð&Ð&Ð&Ü—h‘h×-Ñ-¨a¯g©gÓ6Ð7°q¸¸¸4À¸sÀ!ÐDÐDrX   c                 óæ   — t        t        j                  t        «      sJ ‚t        | t        «      sJ ‚| j
                  €J ‚t        j                  j                  | j
                  «      › d| › d|› d�S )Nro  r½   r—   rÆ  rè  s     rY   ÚltzCppVecOverrides.lt   óc   € äœ!Ÿ(™(¤LÔ1Ð1Ð1Ü˜!œ^Ô,Ð,Ð,Ø�w‰wÐ"Ð"Ð"Ü—(‘(×)Ñ)¨!¯'©'Ó2Ð3°1°Q°C°s¸1¸#¸QÐ?Ð?rX   c                 óæ   — t        t        j                  t        «      sJ ‚t        | t        «      sJ ‚| j
                  €J ‚t        j                  j                  | j
                  «      › d| › d|› d�S )Nro  z > r—   rÆ  rè  s     rY   ÚgtzCppVecOverrides.gt'  rÐ  rX   c                 óæ   — t        t        j                  t        «      sJ ‚t        | t        «      sJ ‚| j
                  €J ‚t        j                  j                  | j
                  «      › d| › d|› d�S )Nro  ú <= r—   rÆ  rè  s     rY   ÚlezCppVecOverrides.le.  rÉ  rX   c                 óæ   — t        t        j                  t        «      sJ ‚t        | t        «      sJ ‚| j
                  €J ‚t        j                  j                  | j
                  «      › d| › d|› d�S )Nro  ú >= r—   rÆ  rè  s     rY   ÚgezCppVecOverrides.ge5  rÉ  rX   c                 ó   — | › d|› �S ©NrU  rW   rè  s     rY   Úand_zCppVecOverrides.and_<  r°  rX   c                 ó   — | › d�S )Nz.rsqrt()rW   r�  s    rY   r­  zCppVecOverrides.rsqrt@  ó   € à��Hˆ~ÐrX   c                 ó   — | › d|› d�S )Nz.pow(r—   rW   rq  s     rY   rÀ  zCppVecOverrides.powD  s   € à��E˜!˜˜AˆÐrX   c                 ó   — | › d�S )Nz.log()rW   r�  s    rY   rÂ  zCppVecOverrides.logH  r·  rX   c                 ó   — | › d�S )Nz.round()rW   r�  s    rY   rÄ  zCppVecOverrides.roundL  rÝ  rX   c                 ó   — | › d�S )Nz.floor()rW   r�  s    rY   rÆ  zCppVecOverrides.floorP  rÝ  rX   c                 ó   — | › d�S )Nz.ceil()rW   r�  s    rY   rÌ  zCppVecOverrides.ceilT  r¼  rX   c                 ó   — | › d�S )Nz.trunc()rW   r�  s    rY   rÎ  zCppVecOverrides.truncX  rÝ  rX   c                 ó   — | › d|› d�S )Nz.fmod(r—   rW   rq  s     rY   rÔ  zCppVecOverrides.fmod\  s   € à��F˜1˜#˜QÐÐrX   c                 ó   — | › d�S )Nz	.lgamma()rW   r�  s    rY   rÚ  zCppVecOverrides.lgamma`  rÂ  rX   c                 ó.   — t        | |«      \  } }| › d|› �S rÚ  ©rM   rq  s     rY   rG  zCppVecOverrides.logical_andd  ó"   € ä,¨Q°Ó2‰ˆˆ1Ø��C˜�sˆ|ÐrX   c                 ó   — d| › �S ©Nú~rW   rK  s    rY   rL  zCppVecOverrides.logical_noti  rM  rX   c                 ó.   — t        | |«      \  } }| › d|› �S ©Nr[  rç  rq  s     rY   rO  zCppVecOverrides.logical_orm  rè  rX   c                 ó.   — t        | |«      \  } }| › d|› �S ©Nr›   rç  rq  s     rY   rR  zCppVecOverrides.logical_xorr  rè  rX   c                 ó.   — t        | |«      \  } }| › d|› �S rÚ  rç  rq  s     rY   rV  zCppVecOverrides.bitwise_andw  rè  rX   c                 ó   — d| › �S rê  rW   rK  s    rY   rY  zCppVecOverrides.bitwise_not|  rM  rX   c                 ó.   — t        | |«      \  } }| › d|› �S rí  rç  rq  s     rY   r\  zCppVecOverrides.bitwise_or€  rè  rX   c                 ó.   — t        | |«      \  } }| › d|› �S rï  rç  rq  s     rY   r^  zCppVecOverrides.bitwise_xor…  rè  rX   c                 ó   — | › d|› �S )Nz << rW   rq  s     rY   rl  z"CppVecOverrides.bitwise_left_shiftŠ  ó   € à��D˜˜ˆ}ÐrX   c                 ó   — | › d|› �S )Nrn  rW   rq  s     rY   ro  z#CppVecOverrides.bitwise_right_shiftŽ  rõ  rX   c                 ó„   — t        t        j                  t        «      sJ ‚t        j                  j	                  | |«      › S r  )r£   r5   rG  rœ  Úload)rÈ   r'  s     rY   Ú	load_seedzCppVecOverrides.load_seed’  s.   € äœ!Ÿ(™(¤LÔ1Ð1Ð1Ü—(‘(—-‘-  fÓ-Ð.Ð/rX   c                 ó|   — t        t        j                  t        «      sJ ‚t	        «       }d| › d�}t        |||«      S )Nz)result[offset_idx] = normalized_rand_cpu(ú, offset[offset_idx]);©r£   r5   rG  rœ  r8   rG   ©rp  r'  r´   Úrand_functions       rY   rt  zCppVecOverrides.rand—  s@   € äœ!Ÿ(™(¤LÔ1Ð1Ð1Ü‹~ˆà7¸°vÐ=SÐTð 	ô ˜F D¨-Ó8Ð8rX   c                 ó|   — t        t        j                  t        «      sJ ‚t	        «       }d| › d�}t        |||«      S )Nzresult[offset_idx] = randn_cpu(rû  rü  rý  s       rY   r|  zCppVecOverrides.randn   s;   € äœ!Ÿ(™(¤LÔ1Ð1Ð1Ü‹~ˆØ9¸$¸Ð?UÐVˆÜ˜F D¨-Ó8Ð8rX   c                 ó¦   — t        t        j                  t        «      sJ ‚t	        «       }d| › d|› d|› d�}t        |||t        j                  «      S )Nz#result[offset_idx] = randint64_cpu(z, offset[offset_idx], r�   r×   )r£   r5   rG  rœ  r8   rG   r…   rš  )rp  r'  r~  r  r´   rþ  s         rY   r€  zCppVecOverrides.randint64§  sS   € äœ!Ÿ(™(¤LÔ1Ð1Ð1Ü‹~ˆØ=¸d¸VÐCYÐZ]ÐY^Ð^`ÐaeÐ`fÐfhÐiˆÜ˜F D¨-¼¿¹ÓEÐErX   c                 óv  — | j                   |j                   k(  sJ d«       ‚t        | j                   «      rgd| › d�}t        j                  j	                  |j                   «      dk  r,|› ddt        j                  j
                  z  dz
  › d|› d|› d�}d| › d	|› d�S | › d
t        j                  | |«      › d|› �S )Nz;remainder vec implementation expect the same inputs' dtype.rh  r—   r6   z::blend<r�  ú(1), zremainder_integral(r�   z - (z) * )r‰   r   r5   rG  Ú_get_raw_num_vectorsÚtiling_factorr•  rÊ  ©rr  rs  Ú_ts      rY   Ú	remainderzCppVecOverrides.remainder®  sÉ   € à�w‰w˜!Ÿ'™'Ò!ð 	
ØIó	
Ð!ô ˜AŸG™GÔ$à˜Q˜C˜qÐ!ˆBÜ�x‰x×,Ñ,¨Q¯W©WÓ5¸Ò9Ø�d˜( A¬¯©×)?Ñ)?Ñ$?À1Ñ#DÐ"EÀRÈÀtÈ5ÐQRÐPSÐSTÐU�Ø(¨¨¨2¨a¨S°Ð2Ð2Ø��Dœ×1Ñ1°!°QÓ7Ð8¸¸Q¸CÐ@Ð@rX   c                 ó   — | › d�S )Nz.tan()rW   rK  s    rY   rº  zCppVecOverrides.tan»  r·  rX   c           	      ó„   — t         j                  j                  r"d| › d�}d| › d�}d| › d�}|› d|› d|› d| › d|› �	S | › d	�S )
Nrh  r±  z)(2)z)(-2)z / (z + (rš   z).exp()) - z.tanh())r   r´  Úuse_decompose_tanh)rr  r¾  Úvec_twoÚvec_minus_twos       rY   r¼  zCppVecOverrides.tanh¿  sl   € ä�:‰:×(Ò(Ø! !  DÐ)ˆGØ! !  DÐ)ˆGØ'¨ s¨%Ð0ˆMà�)˜4 ˜y¨¨]¨O¸3¸q¸cÀÈWÈIÐVðð �S˜�=Ð rX   c                 ó   — | › d�S )Nz.reciprocal()rW   rK  s    rY   Ú
reciprocalzCppVecOverrides.reciprocalË  s   € à��MÐ"Ð"rX   c                 ó   — | › d�S )Nz.atan()rW   r�  s    rY   rí  zCppVecOverrides.atanÏ  r¼  rX   c                 ó   — | › d�S )Nz.acos()rW   r�  s    rY   rÜ  zCppVecOverrides.acosÓ  r¼  rX   c                 ó   — | › d�S )Nz.asin()rW   r�  s    rY   rä  zCppVecOverrides.asin×  r¼  rX   c                 ó   — | › d�S )Nz.cosh()rW   r�  s    rY   rà  zCppVecOverrides.coshÛ  r¼  rX   c                 ó   — | › d�S )Nz.sinh()rW   r�  s    rY   râ  zCppVecOverrides.sinhß  r¼  rX   c                 ó   — | › d�S )Nz.log10()rW   r�  s    rY   r  zCppVecOverrides.log10ã  rÝ  rX   c                 ó   — | › d�S )Nz.log2()rW   r�  s    rY   r  zCppVecOverrides.log2ç  r¼  rX   c                 ó   — | › d|› d�S )Nz.nextafter(r—   rW   rè  s     rY   r  zCppVecOverrides.nextafterë  s   € à��K ˜s !Ð$Ð$rX   c                 ó   — | › d|› d�S )Nz
.copysign(r—   rW   rq  s     rY   rñ  zCppVecOverrides.copysignï  s   € à��J˜q˜c Ð#Ð#rX   c                 ó   — | › d|› d�S )Nz.atan2(r—   rW   rq  s     rY   rê  zCppVecOverrides.atan2ó  ó   € à��G˜A˜3˜aÐ Ð rX   c                 ó   — | › d|› d�S )Nz.hypot(r—   rW   rq  s     rY   r  zCppVecOverrides.hypot÷  r  rX   c           
      ó<   — d| › d�}d| › d�}|› d|› d| › d|› d| › d�
S )	Nrh  r±  z)(0.5)z * ((rê   z)/(rw  z)).log()rW   )r„  r¾  Úvec_one_halfs      rY   rï  zCppVecOverrides.atanhû  sE   € ð ˜a˜S Ð%ˆØ" 1 # VÐ,ˆØ�˜u W I¨S°°°3°w°i¸sÀ1À#ÀXÐNÐNrX   c                 ó˜  — d| › d�}t        «       }|j                  d«       |j                  «       5  |j                  d| › d�«       |j                  d|› d�«       |j                  d«       |j                  d	«       |j                  d
«       |j                  d| › d�«       d d d «       |j                  d«       |S # 1 sw Y   ŒxY w)Nrh  r—   r`  zauto abs_x = z.abs();zauto one = z(1);zauto inv_abs_x = one / abs_x;zaauto correction = (one / (one + inv_abs_x)) / ((one + inv_abs_x * inv_abs_x).sqrt() + inv_abs_x);z1auto result = abs_x.log1p() + correction.log1p();zreturn result.copysign(r×   rh  rˆ  )r„  Úvec_tr´   s      rY   ræ  zCppVecOverrides.asinh  sÄ   € à˜A˜3˜aÐ ˆÜ‹~ˆØ�‰�wÔØ�[‰[‹]ñ 	<ð
 �N‰N˜]¨1¨#¨WÐ5Ô6Ø�N‰N˜[¨¨¨tÐ4Ô5Ø�N‰NÐ:Ô;Ø�N‰NðFôð
 �N‰NÐNÔOØ�N‰NÐ4°Q°C°rÐ:Ô;÷	<ð 	�‰�tÔØˆ÷!	<ð 	<ús   ²A3C Ã C	c                 ó   — | › d�S )Nz.acosh()rW   r�  s    rY   rÞ  zCppVecOverrides.acosh  rÝ  rX   c                 ó¢   — t         j                  j                  }|dk(  ry|dk(  r| › d�S |dk(  r| › d| › d�S |€	d| › d	| › d
�S t        d|›�«      ‚)Nr  r  r  r  r¯  r°  r±  zat::vec::clamp_min(r  r  r  r  r¶  s     rY   r!  zCppVecOverrides.relu  s|   € ä�j‰j×5Ñ5ˆØ�/Ò!Ø#Ø�OÒ#Ø�S˜	�?Ð"Ø�JÒØ�S˜ Q C tÐ,Ð,Øˆ[Ø(¨¨¨;°q°c¸Ð?Ð?ä ØIÈ#ÈÐQóð rX   c                 ó   — d| › d| › d| › d�S )Nrh  z)(1)/(decltype(z)(1) + z.neg().exp())rW   r�  s    rY   r‚  zCppVecOverrides.sigmoid.  s   € à˜1˜#˜_¨Q¨C¨w°q°c¸ÐGÐGrX   c                 ó   — | › d�S )Nz.neg()rW   r�  s    rY   r˜  zCppVecOverrides.neg2  r·  rX   c           	      ó&  — t        | j                  «      r)| j                  |j                  k(  sJ d«       ‚d| › d|› d�S t        d„ | |fD «       «      sJ ‚d|› d�}|› d|› d|› dt        t        j
                  j                  «      › d�}d	| › d|› d�S )
NzDdiv_floor_floating_vec implementation expect the same inputs' dtype.zdiv_floor_floating_vec(r�   r—   c              3   óF   K  — | ]  }t        |j                  «      –— Œ y ­wr  )r   r‰   )r  Úitems     rY   r  z+CppVecOverrides.floordiv.<locals>.<genexpr>>  s   è ø€ ÒG¸Ô'¨¯
©
×3ÑGùó   ‚!rh  ú::set(r  rÉ  )r   r‰   r  rF   r5   rG  Ú	num_elemsr  s      rY   rÊ  zCppVecOverrides.floordiv6  s®   € ä˜!Ÿ'™'Ô"Ø—7‘7˜aŸg™gÒ%ð ØVóÐ%ð -¨Q¨C¨r°!°°AÐ6Ð6äÑGÀÀAÀÔGÔGÐGÐGà˜Q˜C˜qÐ!ˆBØ�$�f˜R˜D  a S¨¬;´q·x±x×7IÑ7IÓ+JÐ*KÈ1ÐMˆAØ+¨A¨3¨b°°°1Ð5Ð5rX   c           	      óz   — d|› d�}|› d|› d|› dt        t        j                  j                  «      › d�}| › d|› �S )Nrh  r—   r'  r  r�   rÑ  )rF   r5   rG  r(  r  s      rY   rÒ  zCppVecOverrides.truncdivD  sP   € ð ˜˜˜1ÐˆØˆd�&˜˜˜E !  B¤{´1·8±8×3EÑ3EÓ'FÐ&GÀqÐIˆØ��C˜�sˆ|ÐrX   c                 óì   — | j                   t        j                  k(  rO|j                   t        j                  k(  sJ ‚t        t        j
                  j                  | |f«      \  }}|› d|› �S d| › d|› d�S )NrU  úat::vec::minimum(r�   r—   ©r‰   r…   rm   rP   r5   rG  r  ©rr  rs  Úa_castÚb_casts       rY   r#  zCppVecOverrides.minimumK  ól   € à�7‰7”e—j‘jÒ Ø—7‘7œeŸj™jÒ(Ð(Ð(Ü1´!·(±(×2BÑ2BÀQÈÀFÓK‰NˆF�FØ�X˜S  Ð)Ð)à& q c¨¨A¨3¨aÐ0Ð0rX   c                 óì   — | j                   t        j                  k(  rO|j                   t        j                  k(  sJ ‚t        t        j
                  j                  | |f«      \  }}|› d|› �S d| › d|› d�S )Nr[  úat::vec::maximum(r�   r—   r,  r-  s       rY   r&  zCppVecOverrides.maximumT  r0  rX   c                 ó   — | › d| › �S r³  rW   rK  s    rY   ÚsquarezCppVecOverrides.square]  r°  rX   c                 óœ  — t        t        j                  t        «      sJ ‚|j                  t
        j                  k(  rY|j                  t
        j                  k(  sJ ‚t        t        j                  j                  | ||f«      \  }}}d|› d|› d|› d|› d�	S d|› d|› d|› dt        j                  j                  | |j                  «      › d�	S )Nrh  ú
)::blendv(r�   r—   )
r£   r5   rG  rœ  r‰   r…   rm   rP   r  Ú_get_mask_cast)rr  rs  r*  Úblendv_aÚblendv_bÚblendv_cs         rY   r+  zCppVecOverrides.wherea  sÁ   € äœ!Ÿ(™(¤LÔ1Ð1Ð1Ø�7‰7”e—j‘jÒ Ø—7‘7œeŸj™jÒ(Ð(Ð(Ü+?Ü—‘× Ñ  1 a¨ )ó,Ñ(ˆH�h ð ˜x˜j¨
°8°*¸B¸x¸jÈÈ8È*ÐTUÐVÐVà˜q˜c ¨A¨3¨b°°°2´a·h±h×6MÑ6MÈaÐQR×QXÑQXÓ6YÐ5ZÐZ[Ð\Ð\rX   c                 ó~  — t        «       }d| › d�}d| › d�}d| › d|› d|› d|› d| › d�}d| › d|› d|› d| › d|› d�}|j                  d«       |j                  «       5  |j                  d	|› d
�«       |j                  d|› d
�«       |j                  d«       d d d «       |j                  d«       |S # 1 sw Y   ŒxY w)Nrh  r„  r±  r6  r�   r½   r—   r`  r…  r¼   r†  r‡  rh  rˆ  )r„  r´   Úvec_zeror¾  Úblendv_lÚblendv_rs         rY   r‹  zCppVecOverrides.signm  sè   € ä‹~ˆØ˜q˜c Ð&ˆØ˜a˜S Ð%ˆØ˜q˜c ¨H¨:°R¸°yÀÀ8À*ÈCÐPQÈsÐRSÐTˆØ˜q˜c ¨H¨:°R¸°yÀÀ1À#ÀSÈÈ
ÐRSÐTˆØ�‰�wÔØ�[‰[‹]ñ 	3Ø�N‰N˜\¨(¨°1Ð5Ô6Ø�N‰N˜]¨8¨*°AÐ6Ô7Ø�N‰NÐ1Ô2÷	3ð 	�‰�tÔØˆ÷	3ð 	3ús   Á<B3Â3B<c                 óF  — |t         j                  t         j                  t         j                  t         j                  t         j
                  t         j                  t         j                  t         j                  t         j                  t         j                  t         j                  fv sJ t        › d|› �«       ‚t        | t        «      sJ ‚| j                  }t         j"                  j%                  | ||«      }t         j"                  j&                  j)                  t         j"                  j*                  |«      }|j-                  d| |fd|i«       |rM|t.        v rE|t         j                  k(  r2t0        j2                  s"t         j"                  j5                  | |||«       |S )Nú does not support rz  r«   )r…   rm   Úfloat64rp   Úbfloat16Úfloat16Úuint8Úint8r  rš  Úfloat8_e4m3fnÚfloat8_e5m2r÷   r£   rH   r‰   r5   rG  r|  r}  r~  r  r€  r„   r   r�  r‚  rƒ  s         rY   rz  zCppVecOverrides.to_dtype|  s/  € àÜ�J‰JÜ�M‰MÜ�K‰KÜ�N‰NÜ�M‰MÜ�K‰KÜ�J‰JÜ�K‰KÜ�K‰KÜ×ÑÜ×Ñð
ñ 
ð 	2ô ˆZÐ)¨%¨Ð1ó	2ð 
ô ˜!œ^Ô,Ð,Ð,Ø—G‘Gˆ	Ü�x‰x×)Ñ)¨!¨U°IÓ>ˆÜ—‘—‘×&Ñ&¤q§x¡x×'7Ñ'7¸Ó>ˆØ×Ñ˜j¨1¨e¨*°{ÀIÐ6NÔOáØœÑ&ØœUŸ[™[Ò(Ü×2Ò2ä�H‰H×(Ñ(¨¨I°v¸uÔEØˆrX   c                 óV  — |t         j                  t         j                  t         j                  fv sJ t        › d|› �«       ‚t        | t        «      sJ ‚| j                  }t        j                  j                  | ||d¬«      }t        j                  j                  j                  t        j                  j                  |«      }|j                  d| |fd|i«       |t        v rE|t         j                   k(  r2t"        j$                  s"t        j                  j'                  | |||«       |S )Nr@  Tr‰  r‹  r«   )r…   rD  rE  r  r÷   r£   rH   r‰   r5   rG  r|  r}  r~  r  r€  r„   rp   r   r�  r‚  rƒ  s         rY   r‹  zCppVecOverrides.round_to_int™  sõ   € àÜ�K‰KÜ�J‰JÜ�K‰Kð
ñ 
ð 	2ô ˆZÐ)¨%¨Ð1ó		2ð 
ô
 ˜!œ^Ô,Ð,Ð,Ø—G‘Gˆ	Ü�x‰x×)Ñ)¨!¨U°IÈÐ)ÓMˆÜ—‘—‘×&Ñ&¤q§x¡x×'7Ñ'7¸Ó>ˆØ×Ñ˜n¨q°%¨j¸;È	Ð:RÔSà”]Ñ"ØœUŸ[™[Ò(Ü×2Ò2ä�H‰H×(Ñ(¨¨I°v¸uÔEØˆrX   c                 óz   — t         j                  j                  }|dk(  r| › d| › d�S |€| › d�S t        d|›�«      ‚)Nr¯  r°  r±  z.log1p()r²  r³  r¶  s     rY   r¸  zCppVecOverrides.log1p­  sT   € ä�j‰j×6Ñ6ˆØ�*ÒØ�S˜ Q C tÐ,Ð,Øˆ[Ø�S˜�>Ð!ä ØJÈ3È'ÐRóð rX   c                 ó¾	  ‡— t        t        j                  t        «      sJ ‚t	        «       }t        j                  j
                  j                  «       }t        j                  j                  | «      5 }|j                  d|› d�«       t        j                  j                  |«      5  |j                  «       5   |«       }|j                  d|› d�«       d d d «       d d d «       d d d «       |j                  d«       t        j                  j                  j                  |«       j                  Š|› d�}ˆfd„}|j                  r|}	n ||«      }	t        |t         ‰   «      }
 ||
«      }t        t"        «      sJ |«       ‚|j                  �r6t	        «       }|j                  d«       t        j                  j                  |«      5  |j                  «       5  |j                  d|› d	�«       |j                  «       5  |j                  d|› d�«       d d d «       |j                  d
«       |j                  «       5  t        j                  j
                  j%                  t        j                  j                  |	«      }t        j                  j
                  j%                  t        j                  j                  |«      }t        |t"        «      sJ |«       ‚t        |t"        «      sJ |«       ‚‰|_        ‰|_        t        j                  j&                  }|j                  d|j)                  |||«      › d�«       d d d «       d d d «       d d d «       |j                  d«       t        j                  j
                  j%                  t        j                  j                  |«      }t        |t"        «      sJ ‚d|_        n¡|j                  rKt        j                  j
                  j%                  t        j                  j                  | › d|	› d|› �«      }nJt        j                  j
                  j%                  t        j                  j                  | › d|› d|
› �«      }|j+                  d| |||fi «       |S # 1 sw Y   �Œ°xY w# 1 sw Y   �ŒµxY w# 1 sw Y   �ŒºxY w# 1 sw Y   �Œ’xY w# 1 sw Y   �ŒuxY w# 1 sw Y   �ŒzxY w# 1 sw Y   �ŒxY w)NrÖ   r;  r<  r¼   rh  c                 ó¼   •— ‰t         j                  k(  r$t        j                  j	                  «       › d| › d�S t        j                  j                  ‰«      › d| › d�S )Nú::from(r—   ro  )r…   rm   r5   rG  rÇ  Ú_get_vec_type)r´   r‰   s    €rY   Úmaskify_or_vecifyz1CppVecOverrides.masked.<locals>.maskify_or_vecifyÉ  s]   ø€ ð œEŸJ™JÒ&ô —8‘8×*Ñ*Ó,Ð-¨W°T°F¸!Ð<ðô Ÿ™×.Ñ.¨uÓ5Ð6°a¸°v¸QÐ?ðrX   ú[&]úif (z.all_zero())ÚelseTr(  r)  rD  )r£   r5   rG  rœ  r8   r}  r  rD  rÃ   r=  rÅ   r  r  r‰   r§   rQ   rI   rH   r~  Ú	overridesr+  r€  )r>  r?  r@  r´   r¨   Únew_maskrB  Ú	body_coderN  Úbody_code_vecrC  Úother_code_vecÚbody_vec_varÚother_vec_varrR  r‡  r‰   s                   @rY   rD  zCppVecOverrides.masked¹  sã  ø€ äœ!Ÿ(™(¤LÔ1Ð1Ð1Ü‹~ˆÜ�h‰h�l‰l×!Ñ!Ó#ˆÜ�X‰X�_‰_˜TÓ"ð 	4 hØ�N‰N˜U 3 % vÐ.Ô/Ü—‘×&Ñ& tÓ,ñ 4¨d¯k©k«mñ 4Ù›�Ø—‘ ¨¨°Ð2Ô3÷4÷ 4÷	4ð
 	�‰�sÔÜ	�‰×Ñ×Ñ Ô%à—‘ˆØ�e˜2�Jˆ	ô	ð �=Š=Ø%‰Má-¨iÓ8ˆMÜ! %¬°eÑ)<Ó=ˆ
á*¨:Ó6ˆÜ˜(¤NÔ3Ð=°XÓ=Ð3Ø�?‹?Ü“>ˆDØ�N‰N˜5Ô!Ü—‘×&Ñ& tÓ,ñ ¨d¯k©k«mñ Ø—‘  h Z¨|Ð<Ô=Ø—[‘[“]ñ @Ø—N‘N W¨^Ð,<¸AÐ#>Ô?÷@à—‘˜vÔ&Ø—[‘[“]ñ ä#$§8¡8§<¡<×#8Ñ#8ÜŸ™×(Ñ(Ø%ó$�Lô %&§H¡H§L¡L×$9Ñ$9ÜŸ™×(Ñ(Ø&ó%�Mô & l´NÔCÐQÀ\ÓQÐCÜ% m´^ÔDÐSÀmÓSÐDØ).�LÔ&Ø*/�MÔ'ô Ÿ™×*Ñ*ð ð —N‘NØ! )§/¡/°(¸LÈ-Ó"XÐ!YÐYZÐ[ô÷%÷÷ ð4 �N‰N˜4Ô Ü—X‘X—\‘\×*Ñ*Ü—‘× Ñ ØóˆFô ˜f¤nÔ5Ð5Ð5Ø ˆF�MØ�]Š]Ü—X‘X—\‘\×*Ñ*Ü—‘× Ñ  T F¨#¨m¨_¸CÀÐ?OÐ"Pó‰Fô —X‘X—\‘\×*Ñ*Ü—‘× Ñ  T F¨#¨i¨[¸¸J¸<Ð"HóˆFð
 	×Ñ˜h¨¨t°U¸FÐ(CÀRÔHØˆ÷O4ñ 4ú÷ 4ñ 4ú÷	4ñ 	4ú÷>@ñ @ú÷ñ ú÷ñ ú÷ ñ ús‘   Á35RÂ(RÂ9RÃRÃRÇSÇ&SÈR+È)SÉDR8ÍSÍSÒRÒ	RÒR	ÒRÒR(Ò+R5Ò0SÒ8SÒ=SÓS	Ó
SÓSc                 óX  — t        t        j                  t        «      sJ ‚t        j                  j	                  | «      }t        j                  j
                  t        j                  j                     }t        j                  j                  ||«      }|dk(  rt        j                  | |«      S |�©t        j                  j                  j                  t        j                  j                  t        |«      t        | «      ¬«      }t        j                   ||«      }t        |t"        «      r|j$                  }t        j                  j'                  ||«      }n:t        j                  j)                  d ||t        j                  j                  «      }|j+                  d| |fi «       |S )Nr   r2  r6  )r£   r5   rG  rœ  r4  ÚitervarsÚ
tiling_idxÚ_try_get_const_striderm  r6  r}  r~  r  rE   r'   r3   rz  r4   rð   ÚarangeÚ_load_or_store_non_contiguousr€  )r†  r‰   r“   Ú
tiling_varÚstrider=  rð   r‡  s           rY   r6  zCppVecOverrides.index_expr	  s3  € äœ!Ÿ(™(¤LÔ1Ð1Ð1Ü—‘×(Ñ(¨Ó.ˆÜ—X‘X×&Ñ&¤q§x¡x×':Ñ':Ñ;ˆ
Ü—‘×/Ñ/°°zÓBˆØ�QŠ;Ü×*Ñ*¨4°Ó7Ð7ØÐÜ—(‘(—,‘,×'Ñ'Ü—‘× Ñ ¤%¨£,Ô7LÈTÓ7Rð (ó ˆCô —L‘L  eÓ,ˆEÜ˜%¤Ô*ØŸ™�Ü—X‘X—_‘_ U¨FÓ3‰Fä—X‘X×;Ñ;Ø�e˜U¤A§H¡H×$4Ñ$4óˆFð 	×Ñ˜l¨T°5¨M¸2Ô>ØˆrX   c                 ó.   — t         j                  | |«      S r  )r•  r6  r8  s     rY   r9  zCppVecOverrides.value_expr!  s   € ô ×)Ñ)¨$°Ó6Ð6rX   c           
      óx  — d| › d�d| › d�f}t        d„ |D «       «      rt        d„ |D «       «      S t        | j                     }t        j
                  j                  rt        j
                  j                  nt        j
                  j                  }t        «       }t        j
                  j                  j                  t        j                  ¬«      }t        j
                  j                  j                  | j                  ¬«      }|j                  d| fi ¬«       |j                  d| fi ¬«       t        j
                  j                  | j                  «      }|d	k(  rd
|› d�nd|› d|› d�}|j                  |d	k(  rd|› d�nd|› d|› d�«       |j                  |› d|› d�«       |j                  d«       |j!                  «       5  |j                  d|› dt        j
                  j                  › d�«       |j                  | › dt#        |«      › d�«       |j                  dt        j
                  j                  › d�«       |j                  d|› dt        j
                  j                  › d�«       |j                  dt#        |«      › d�«       |j!                  «       5  |j                  d«       d d d «       |j                  |d	k(  r|› dt#        |«      › d�n|› d|› d t#        |«      › d�«       |j                  |› d!|› d"t#        |«      › d�«       d d d «       |j                  d#«       t        j
                  j$                  j'                  |«       ||f}	t)        ||	«      D ]/  \  }
}t        j
                  j                  j+                  |
|«       Œ1 ||fS # 1 sw Y   ŒíxY w# 1 sw Y   Œ•xY w)$Nró  rô  rõ  c              3   ór   K  — | ]/  }t         j                  j                  j                  |«      d u–— Œ1 y ­wr  r÷  rù  s     rY   r  z(CppVecOverrides.frexp.<locals>.<genexpr>*  rû  rü  c              3   ón   K  — | ]-  }t         j                  j                  j                  |«      –— Œ/ y ­wr  r÷  rù  s     rY   r  z(CppVecOverrides.frexp.<locals>.<genexpr>+  rþ  rÿ  ©r‰   r  )rž  r6   úat::vec::Vectorized<r�   úat::vec::VectorizedN<r�   zat::vec::Vectorized<int32_t> r¼   zat::vec::VectorizedN<int32_t, ú> r™   r`  ú__at_align__ std::array<ú	> tmpbuf;ú.store(tmpbuf.data(), r×   z!__at_align__ std::array<int32_t, z> tmpbuf_exponent;z> tmpbuf_mantissa;rÚ   rÛ   z@tmpbuf_mantissa[i] = std::frexp(tmpbuf[i], &tmpbuf_exponent[i]);z? = at::vec::Vectorized<int32_t>::loadu(tmpbuf_exponent.data(), z! = at::vec::VectorizedN<int32_t, z!>::loadu(tmpbuf_exponent.data(), r»   z ::loadu(tmpbuf_mantissa.data(), z();)r  r¤   rI   r‰   r5   rG  Ú	tail_sizer  r8   r}  r  r…   r  r€  Ú_get_num_vectorsrÃ   rÅ   rF   r  r  r  r  )r„  r  rŠ   r&  r´   r  r  Ún_vecÚ
mantissa_tr	  rú  r
  s               rY   r  zCppVecOverrides.frexp'  s{  € à˜a˜S Ð%¨°¨s°$Ð'7Ð7ˆ
ÜÑWÈJÔWÔWÜÑUÈ*ÔUÓUÐUä˜aŸg™gÑ&ˆÜ%&§X¡X×%7Ò%7Œq�x‰x×!Ò!¼Q¿X¹X×=SÑ=SˆÜ‹~ˆÜ—8‘8—<‘<×&Ñ&¬U¯[©[Ð&Ó9ˆÜ—8‘8—<‘<×&Ñ&¨Q¯W©WÐ&Ó5ˆØ×Ñ ¨!¨°bÐÔ9Ø×Ñ ¨!¨°bÐÔ9Ü—‘×)Ñ)¨!¯'©'Ó2ˆð ˜Šzð # 6 (¨!Ñ,à(¨¨°°5°'¸Ð;ð 	ð
 	�‰à˜Šzð ,¨H¨:°QÑ7à1°%°¸¸8¸*ÀAÐFô	
ð
 	�‰˜*˜ Q x j°Ð2Ô3Ø�‰�wÔØ�[‰[‹]ñ 	Ø�N‰NØ*¨6¨(°"´Q·X±X×5KÑ5KÐ4LÈIÐVôð �N‰N˜a˜SÐ 6´{À4Ó7HÐ6IÈÐLÔMØ�N‰NØ3´A·H±H×4JÑ4JÐ3KÐK]Ð^ôð �N‰NØ*¨6¨(°"´Q·X±X×5KÑ5KÐ4LÐL^Ð_ôð �N‰NÐ1´+¸dÓ2CÐ1DÀFÐKÔLØ—‘“ñ Ø—‘ØVô÷ð �N‰Nà˜A’:ð �*Ð[Ô\gÐhlÓ\mÐ[nÐnpÑqà �zÐ!BÀ5À'ÐIjÔkvÐw{Ók|Ðj}Ð}ð  Aôð
 �N‰NØ�*˜C 
˜|Ð+KÌKÐX\ÓL]ÐK^Ð^`Ðaô÷+	ð0 	�‰�uÔÜ	�‰×Ñ×Ñ Ô%Ø˜hÐ'ˆÜ"% j°(Ó";ò 	1ÑˆI�wÜ�H‰H�L‰L×Ñ˜Y¨Õ0ð	1à˜Ð!Ð!÷#ð ú÷	ð 	ús&   Ç
CN0Ê&N$Ê8A$N0Î$N-	Î)N0Î0N9c                 ó   ‡— ˆfd„}|S )Nc                  óä  •— |rJ ‚t         j                  }t        |t        «      sJ ‚t	        «       }|j                  d«       | d   j                  }|j                  |«      }|j                  r|j                  n|j                  }g }t        |   }‰j                  dv }	|	rdn|}
‰j                  dk(  rt        | d      n|
}
|j                  «       5  t        | «      D ]£  \  }}t        |t        «      r}|j                  sJ ‚|j                  |k(  sJ ‚|j                  d|› d|j                  › d	|› d
�«       |j                  |› d|› dt!        |«      › d�«       |j#                  d|› d�«       Œ“|j#                  |«       Œ¥ |j                  d|
› d|j                  › d�«        ‰|Ž }|j                  dt!        |«      › d�«       |j                  «       5  |j                  d|› d
�«       d d d «       dt!        |«      › �}|	r
d|› d|› d�}n|dk(  rd|
› d�}n	d|
› d|› d�}|j                  d|› d|› d�«       d d d «       |j                  d«       |S # 1 sw Y   ŒkxY w# 1 sw Y   Œ(xY w)Nr`  r   )rÖ  rØ  r¾  rm   rŽ  éþÿÿÿri  r�   z> tmpbufr¼   z.store(tmpbufz	.data(), r×   Útmpbufz[i]z> tmpbuf_out;rÚ   rÛ   ztmpbuf_out[i] = ztmpbuf_out.data(), úat::vec::VecMask<ú,z>::fromr6   rf  z>::loaduz at::vec::VectorizedN<r<  ro  rh  )r5   rG  r£   rœ  r8   rÃ   r‰   rm  rl  r  rI   r÷   rÅ   rÆ   rH   r§   rF   r›  )ri  rž  rG  r´   Ú	vec_dtypern  r&  Úscalar_argsrŠ   Úoutput_maskÚoctypeÚargidxrg  ÚresÚ	load_argsÚload_fnr¤  s                   €rY   rF  z)CppVecOverrides._scalarize.<locals>.innerb  s¬  ø€ ÙÐ�:Ü—X‘XˆFÜ˜f¤lÔ3Ð3Ð3Ü“>ˆDØ�N‰N˜7Ô#Ø˜Q™Ÿ™ˆIØ×+Ñ+¨IÓ6ˆEØ'-×'7Ò'7�6×#Ò#¸V×=QÑ=QˆDØˆKÜ! )Ñ,ˆFØ%×.Ñ.ð 3ð ˆKñ
  +‘V°ˆFð  ×(Ñ(Ð,>Ò>ô ˜T "™XÒ&àð ð
 —‘“ñ BÜ#,¨T£?ò 0‘K�F˜CÜ! #¤~Ô6Ø"ŸzšzÐ)˜zØ"Ÿy™y¨IÒ5Ð5Ð5ØŸ™Ø6°v°h¸bÀ×AUÑAUÐ@VÐV^Ð_eÐ^fÐfgÐhôð Ÿ™Ø"˜e =°°¸	Ä+ÈdÓBSÐATÐTVÐWôð $×*Ñ*¨V°F°8¸3Ð+?Õ@à#×*Ñ*¨3Õ/ð0ð —‘Ø.¨v¨h°b¸×9MÑ9MÐ8NÈmÐ\ôñ " ;Ð/�Ø—‘Ð!5´kÀ$Ó6GÐ5HÈÐOÔPØ—[‘[“]ñ >Ø—N‘NÐ%5°c°U¸!Ð#<Ô=÷>à1´+¸dÓ2CÐ1DÐE�	ÙØ 1°&°¸¸5¸'ÀÐI‘GØ˜a’ZØ 4°V°H¸HÐE‘Gà 6°v°h¸bÀÀÀxÐP�GØ—‘ ¨¨	°°9°+¸RÐ@ÔA÷9Bð: �N‰N˜4Ô ØˆK÷>ð >ú÷'Bð Bús&   ÃDI&ÇIÇ0AI&ÉI#	ÉI&É&I/rW   )r  r¤  rF  s    ` rY   Ú
_scalarizezCppVecOverrides._scalarize`  s   ø€ ô4	ðl ˆrX   c                 ó  — t        t        «      }t        t        «      j                  «       D ]S  \  }}t	        |t
        «      sŒ||vsŒ| j                  |j                  «      }||_        t        | |t        |«      «       ŒU y r  )
r©  r•  rm  rª  r£   r‘  r~  r¬  r÷   r«  )r  Úvec_varsrÈ   r®  r¥  s        rY   Ú_initialize_scalarizez%CppVecOverrides._initialize_scalarizeš  sh   € äœÓ(ˆÜ ¤Ó.×4Ñ4Ó6ò 	7‰LˆD�&Ü˜&¤,Õ/°DÀÒ4HØ—~‘~ f§o¡oÓ6�Ø $�”Ü˜˜T¤<°Ó#5Õ6ñ		7rX   r�  )Yr÷   rø   rù   rú   r¨  r‘  rt  rË   r8  rµ  r‘  r”  r–  r›  r�  r   r£  r¥  r§  rª  rÈ  rÍ  rÏ  rÒ  rÕ  rØ  rÛ  r­  rÀ  rÂ  rÄ  rÆ  rÌ  rÎ  rÔ  rÚ  rG  rL  rO  rR  rV  rY  r\  r^  rl  ro  rù  rt  r|  r€  r  rº  r¼  r  rí  rÜ  rä  rà  râ  r  r  r  rñ  rê  r  rï  ræ  rÞ  r!  r‚  r˜  rÊ  rÒ  r#  r&  r4  r+  r‹  rz  r‹  r¸  rD  r6  r9  r  rL  r~  r�  rM  rN  s   @rY   r•  r•  r  s|  ø„ Ù8ô\ð| ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ(ó ð(ð
 ñó ðð ñó ðð ñó ðð ñó ðð ñAó ðAð ñ	Eó ð	Eð ñ@ó ð@ð ñ@ó ð@ð ñAó ðAð ñAó ðAð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ ó ð ð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ0ó ð0ð ñ9ó ð9ð ñ9ó ð9ð ñFó ðFð ñ
Aó ð
Að ñó ðð ñ	!ó ð	!ð ñ#ó ð#ð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ%ó ð%ð ñ$ó ð$ð ñ!ó ð!ð ñ!ó ð!ð ñOó ðOð ñó ðð, ñó ðð ñó ðð  ñHó ðHð ñó ðð ñ6ó ð6ð ñó ðð ñ1ó ð1ð ñ1ó ð1ð ñó ðð ñ	]ó ð	]ð ñó ðð òó ðð8 òó ðð& ñ	ó ð	ð ñMó ðMð^ ñó ðð. ñ7ó ð7ð
 ñ6"ó ð6"ðp ñ7ó ð7ðr ñ7ó ô7rX   r•  Úcppvecc                   ó,   — e Zd Zed„ «       Zed„ «       Zy)ÚCppTile2DOverridesc                 ó¬   — t        t        j                  t        «      sJ ‚t        j                  j	                  | «      } t
        j                  | |«      S r  )r£   r5   rG  ÚCppTile2DKernelÚtransform_indexingr•  r6  r8  s     rY   r6  zCppTile2DOverrides.index_expr©  s=   € äœ!Ÿ(™(¤OÔ4Ð4Ð4Ü�x‰x×*Ñ*¨4Ó0ˆÜ×)Ñ)¨$°Ó6Ð6rX   c                 ó.   — t         j                  | |«      S r  )r„  r6  r8  s     rY   r9  zCppTile2DOverrides.value_expr¯  s   € ô "×,Ñ,¨T°5Ó9Ð9rX   N)r÷   rø   rù   r‘  r6  r9  rW   rX   rY   r„  r„  ¨  s(   „ Øñ7ó ð7ð
 ñ:ó ñ:rX   r„  c                   óÀ  ‡ — e Zd ZdZeZeZdZdZ	ˆ fd„Z
eefd„Zd„ Zd4dedz  fd	„Zej$                  d
„ «       Z	 d5dej*                  fd„Zdej*                  defd„Zdej*                  dej2                  fd„Zdej*                  dej2                  fd„Zd„ Zdej*                  dej*                  dedefd„Zdedej*                  fd„Zd4d„Z d„ Z!de"ez  dedede#jH                  fd „Z%d4de&dz  fd!„Z'd"„ Z(	 	 d6d#„Z)	 d7d$„Z*d%„ Z+d&„ Z,d'„ Z-d(„ Z.d)„ Z/d*„ Z0e1defd+„«       Z2d,„ Z3ej$                  d-„ «       Z4d.„ Z5d7d/„Z6d0„ Z7	 	 d8ded1edz  d2ej2                  dz  fd3„Z8ˆ xZ9S )9Ú	CppKernela%  
    Base class for C++ kernel code generation in PyTorch Inductor.
    This class is responsible for generating C++ code from the intermediate representation.

    Args:
        args: Kernel arguments used for code generation
        num_threads: Number of threads for parallel execution
    rÖ   r¼   c                 óÔ  •— t         ‰| �  |«       i | _        g | _        d | _        g | _        g | _        d | _        t        «       | _	        g | _
        t        «       | _        t        «       | _        t        «       | _        t        «       | _        t        «       | _        d| _        t        «       | _        t        «       | _        t'        | j(                  | j*                  d¬«      | _        t'        | j(                  | j*                  d¬«      | _        t'        | j(                  | j*                  d¬«      | _        t        «       | _        t        «       | _        || _        i | _        g | _        y )NFÚtmp_acc)Úname_prefixÚwelford_helperÚcascade_helper)r  r  Úactive_rangesÚinner_itervarsÚcall_rangesr9  rZ  Úreduction_depthr>   Úreduction_prefixÚreduction_prefix_generatorsÚreduction_suffixÚparallel_reduction_prefixÚparallel_reduction_suffixÚlocal_reduction_initÚlocal_reduction_storesÚis_reductionÚnon_parallel_reduction_prefixÚnon_parallel_reduction_suffixr9   Únewvar_prefixÚsuffixÚreduction_cseÚwelford_helper_cseÚcascade_helper_cseÚpreloadsÚ
poststoresÚnum_threadsÚreduction_omp_decÚreduction_var_names)r  ri  r¥  r  s      €rY   r  zCppKernel.__init__Å  s7  ø€ Ü‰Ñ˜Ôð HJˆÔð 35ˆÔØ:>ˆÔØ(*ˆŒØ,.ˆŒØ#ˆÔÜ .Ó 0ˆÔð <>ˆÔ(Ü .Ó 0ˆÔÜ)7Ó)9ˆÔ&Ü)7Ó)9ˆÔ&Ü$2Ó$4ˆÔ!Ü&4Ó&6ˆÔ#Ø!ˆÔÜ-;Ó-=ˆÔ*Ü-;Ó-=ˆÔ*Ü  ×!3Ñ!3°T·[±[ÈiÔXˆÔÜ"%Ø×Ñ §¡Ð9Iô#
ˆÔô #&Ø×Ñ §¡Ð9Iô#
ˆÔô 'Ó(ˆŒÜ(Ó*ˆŒØ&ˆÔØ=?ˆÔØ.0ˆÕ rX   c                 ó8  — t         j                  j                  r'| j                  s| j                  j	                  d«       |› d�}t         j                  j                  rdn	t        «       }|› d�}	| j                  j	                  |› d|› d |||«      › d�«       | j                  j                  t        ||||||«      «       | j                  j	                  |	› d|› d�«       | j                  j                  d|› d	�d
d|› d ||||	|¬«      › d�dg«       y )Nz(int max_threads = omp_get_max_threads();Ú_localÚmax_threadsz	_arr[tid]r™   r»   r¼   zfor (int tid = 0; tid < z; tid++)rÜ   rÝ   ©r«   r�   )r   r´  Údynamic_threadsr—  rÃ   r,   r™  r  rã   rš  r˜  rß   )
r  r³   rÓ   rˆ   r‰   Úreduction_combine_fnÚreduction_init_fnÚ	acc_localr¥  Úacc_local_in_arrays
             rY   Ú_gen_parallel_reduction_buffersz)CppKernel._gen_parallel_reduction_buffersî  s=  € ô �:‰:×%Ò%¨d×.LÒ.LØ×*Ñ*×4Ñ4Ø:ôð �e˜6�Nˆ	ä#ŸZ™Z×7Ò7‰MÔ=QÓ=Sð 	ð !$˜u IÐ.ÐØ×!Ñ!×+Ñ+Øˆj˜˜)˜ CÑ(9¸.È%Ó(PÐ'QÐQRÐSô	
ð 	×&Ñ&×-Ñ-Ü"ØØØØØØ!óô		
ð 	×#Ñ#×-Ñ-Ð1CÐ0DÀCÈ	À{ÐRSÐ.TÔUØ×&Ñ&×1Ñ1à*¨;¨-°xÐ@ØØ�s�e˜3Ñ3°NÀCÐI[ÐglÔmÐnÐnoÐpØð	õ	
rX   c                 óZ   — | j                   D ]  }t        | j                  ||› d�«       Œ y )Nr©  )r§  rè   Ústores)r  Úvar_names     rY   Ú%update_stores_with_parallel_reductionz/CppKernel.update_stores_with_parallel_reduction  s0   € Ø×0Ñ0ò 	IˆHÜ˜TŸ[™[¨(°x°jÀÐ4GÕHñ	IrX   Nr´   c                 ó”  — |�J ‚t        «       }t        j                  «       5 }t        | d«      rK|j	                  | j
                  «       | j                  |«       |j                  |j                  «       «       |j	                  | j                  «       |j	                  | j                  «       |j	                  | j                  «       d d d «       t        | d«      r|j	                  | j                  «       | j                  r5| j                  D ]&  }| j                  |   \  }}t        |||› d�||«      }Œ( |S # 1 sw Y   ŒsxY w)NÚcodegen_inner_loopsÚ_tail)r8   rÁ   rÂ   r¦   r  r£  r·  rÄ   rÅ   Úloadsr  r³  r¤  r‘  r�  rÑ   )r  r´   rÍ   r=  ÚstartÚends         rY   Úgen_bodyzCppKernel.gen_body  s  € Øˆ|Ðˆ|Ü‹~ˆÜ×!Ñ!Ó#ð 	% uÜ�tÐ2Ô3Ø—‘˜DŸM™MÔ*Ø×(Ñ(¨Ô.Ø×#Ñ# D§K¡K£MÔ2Ø�K‰K˜Ÿ
™
Ô#Ø�K‰K˜Ÿ™Ô%Ø�K‰K˜Ÿ™Ô$÷	%ô �4Ð.Ô/Ø�K‰K˜Ÿ™Ô(à×ÒØ×*Ñ*ò X�Ø!×/Ñ/°Ñ4‘
��sÜ1°$¸À¸uÀE¸]ÈEÐSVÓW‘ðXð ˆ÷	%ð 	%ús   £B)D>Ä>Ec              #   ó  K  — | j                   }|rYt        j                  ||«      }t        |t        «      r3|j
                  }t        |t        «      sJ ‚t        j                  |_	        || _         	 |–— || _         y# || _         w xY w­w)z>Context manager to add an additional mask to loads and stores.N)
Ú
_load_maskr3   rÛ  r£   r4   rð   rH   r…   rm   r‰   )r  r>  Úpriors      rY   rD  zCppKernel.masked/  ss   è ø€ ð —‘ˆÙÜ—8‘8˜D %Ó(ˆDÜ˜$¤Ô)Ø—z‘z�Ü! $¬Ô7Ð7Ð7ô #ŸZ™Z�”
àˆŒð	$ØŠJà#ˆD�Oø˜eˆD�Oüs   ‚A/B
Á2A> Á6B
Á>	BÂB
r“   c                 óP   — | j                   |   }|||z  |z   i}t        ||«      }|S r  )rZ  r1   )r  r“   ÚscaleÚitervar_idxr'  r¨   ÚreplacementÚ	new_indexs           rY   Úscale_index_with_offsetz!CppKernel.scale_index_with_offsetB  s7   € ð �m‰m˜KÑ(ˆØ˜C %™K¨&Ñ0Ð1ˆÜ˜u kÓ2ˆ	ØÐrX   r¹   c                 ó6   — t        | j                  |«      «      S )z¦
        Convert an index expr to a string that can be used in cpp code.
        e.g. a sympy expression "s2" may actually appear as "ks1" in the cpp kernel.
        )rE   r4  ©r  r“   s     rY   Úindex_to_strzCppKernel.index_to_strJ  s   € ô
 �T×)Ñ)¨%Ó0Ó1Ð1rX   Úitervarc                 óD   ‡ ‡— t        ˆˆ fd„|j                  D «       «      S )z]
        Check if an index has free symbol CppCSEVariable that depends on `itervar`.
        c              3   ó(  •K  — | ]‰  }|j                   ‰j                  j                  v ret        ‰j                  j                  |j                      t        «      r4‰j                  j                  |j                      j                  ‰«      –— Œ‹ y ­wr  )rÈ   r}  Úvarname_mapr£   rH   Ú
depends_on)r  ÚsrÉ  r  s     €€rY   r  z6CppKernel.index_indirect_depends_on.<locals>.<genexpr>U  sj   øè ø€ ò 
àØ�v‰v˜Ÿ™×-Ñ-Ñ-Ü˜4Ÿ8™8×/Ñ/°·±Ñ7¼ÔHð �H‰H× Ñ  §¡Ñ(×3Ñ3°G×<ñ
ùs   ƒBB)rh   Úfree_symbols©r  r“   rÉ  s   ` `rY   Úindex_indirect_depends_onz#CppKernel.index_indirect_depends_onQ  s%   ù€ ô ô 
à×'Ñ'ô
ó 
ð 	
rX   c                 óF   — ||j                   v xs | j                  ||«      S r  )rÏ  rÑ  rÐ  s      rY   Úindex_depends_onzCppKernel.index_depends_on\  s,   € Ø˜%×,Ñ,Ð,ò 
°×0NÑ0NØ�7ó1
ð 	
rX   c                 óT   — t        t        | j                  | j                  «      «      S r  )r“  r  rZ  r9  r  s    rY   Ú
var_rangeszCppKernel.var_rangesa  s   € Ü”C˜Ÿ™ t§{¡{Ó3Ó4Ð4rX   r†  r&  ÚlowerÚupperc                 ó   — |s|sy t        |t        j                  «      }|rJt        j                  |t
        j                  «      j                  }t        j                  j                  }�nt        | j                  |«      «      }| j                  j                  |«      �It        j                  |t
        j                  «      j                  }t        j                  j                  }n‰t        j                  j                  }		 | j                  t        j                  _
        t        j                  |t
        j                  «      j                  }|	t        j                  _
        | j                  }|r.t        j                  j!                  | j                  |«      «      nd }
| j#                  ||rdnd |
| j$                  «      }| j                  j'                  ||d¬«       y # |	t        j                  _
        w xY w)NÚ0F)Ú
assignment)r   r   ÚTMPr3   r6  r…   rš  rð   r5   rG  r  rE   r4  r}  rø  r¹  ÚsexprÚindirect_assertr¾  r~  )r  r†  r&  rÖ  r×  Úindirectr‡  rä   r5  Úprior_computeÚsize_strrÉ   s               rY   Úcheck_boundszCppKernel.check_boundsd  s]  € ñ ™Øä& t¬T¯X©XÓ6ˆÙä—^‘^ D¬%¯+©+Ó6×<Ñ<ˆFÜ—X‘X×%Ñ%ŠFô ˜D×0Ñ0°Ó6Ó7ˆGØ�x‰x×Ñ Ó(Ð4ÜŸ™¨¬e¯k©kÓ:×@Ñ@�ÜŸ™×)Ñ)‘ä !§¡× 0Ñ 0�ð5Ø'+§z¡z”A—H‘HÔ$Ü Ÿ^™^¨D´%·+±+Ó>×DÑD�Fà'4”A—H‘HÔ$ØŸ™�áAF”1—8‘8—>‘> $×"6Ñ"6°tÓ"<Ô=ÈDˆà×#Ñ#Ø™5‘C d¨H°d·o±oó
ˆð 	�‰×Ñ˜& $°5ÐÕ9øð (5”A—H‘HÕ$ús   ÄAG& Ç&G=rÈ   c                 ó6  — | j                   j                  |«      }| j                  |«      }|› dt        |«      › d�}| j                  j                  | j                  |t        j                  j                  |«      ¬«      }|j                  d| ||fi «       |S )Nú[ú]re  rø  )ri  Úinputr4  rF   r}  r~  r¹  r5   ÚgraphÚ	get_dtyper€  )r  rÈ   r“   r¨   rÉ   r‡  s         rY   rø  zCppKernel.loadŽ  sˆ   € Ø�i‰i�o‰o˜dÓ#ˆØ×$Ñ$ UÓ+ˆØ��aœ EÓ*Ð+¨1Ð-ˆØ—‘×"Ñ" 4§:¡:¨t¼1¿7¹7×;LÑ;LÈTÓ;RÐ"ÓSˆØ×Ñ˜f t¨T°5Ð&9¸2Ô>ØˆrX   c                 ó  — d|v sJ ‚| j                   j                  |«      }| j                  |«      }|€|› dt        |«      › d|› d�}n–|dk(  rƒt        j
                  j                  s$| j                  dk(  r|› dt        |«      › d|› d�}nSt        j                  j                  |«      }dt        |   › d	|› d
�}d|› dt        |«      › d|› d�}nt        d|› �«      ‚| j                  j                  t        ||«      «       y )NÚbufrã  ú] = r¼   Ú
atomic_addr6   z] += zstatic_cast<r�  r—   zatomic_add(&z], r×   ústore mode=)ri  Úoutputr4  rF   r   r´  r¬  r¥  r5   ræ  rç  rI   rŽ  r³  rÃ   r<   )r  rÈ   r“   rð   Úmoder¨   rÉ   r‰   s           rY   ÚstorezCppKernel.store–  s  € Ø˜‰}Ðˆ}Ø�i‰i×Ñ˜tÓ$ˆØ×$Ñ$ UÓ+ˆØˆ<Ø�U˜!œK¨Ó.Ð/¨t°E°7¸!Ð<‰DØ�\Ò!Ü—:‘:×-Ò-°$×2BÑ2BÀaÒ2GØ˜˜a¤¨EÓ 2Ð3°5¸¸¸qÐA‘äŸ™×)Ñ)¨$Ó/�à&¤|°EÑ':Ð&;¸2¸e¸WÀAÐF�Ø% c U¨!¬K¸Ó,>Ð+?¸sÀ5À'ÈÐL‘ä%¨°D°6Ð&:Ó;Ð;Ø�‰×Ñœl¨4°Ó6Õ7rX   c                 óH   — | j                   j                  d|› d|› d�«       y )Nro  z" ? 0 : (throw std::runtime_error("z"), 0));)r  rÃ   )r  ÚcondÚmsgs      rY   Údevice_assert_asynczCppKernel.device_assert_async¨  s'   € Ø�‰×ÑØ�ˆvÐ7¸°u¸HÐEõ	
rX   r³   rÓ   Úrtyper‰   c                 ó8   ‡‡‡‡‡— ddt         d z  fˆˆˆˆˆfd„}|S )Nr&  c                 óL   •— | €‰› d‰› d ‰‰‰«      › d�S t        ‰‰‰‰| ‰«      S )Nr™   r»   r¼   )rã   )r&  r³   rÓ   r‰   rà   rô  s    €€€€€rY   rF  z.CppKernel._gen_reduction_prefix.<locals>.inner»  sH   ø€ Øˆ|Ø"˜ 1 S E¨©W°U¸EÓ-BÐ,CÀ1ÐEÐEä-ØØØØØØóð rX   r  )ro   )r  r³   rÓ   rô  r‰   rà   rF  s    ````` rY   Ú_gen_reduction_prefixzCppKernel._gen_reduction_prefix­  s   ü€ ñ	œ˜d™
÷ 	ñ 	ð ˆrX   c                 óh   — | j                   D ]#  }| j                  j                   ||«      «       Œ% y r  )r•  r”  r  )r  r&  Úgen_fns      rY   Úfinalize_reduction_prefixz#CppKernel.finalize_reduction_prefixÊ  s/   € Ø×6Ñ6ò 	7ˆFØ×!Ñ!×(Ñ(©°«Õ6ñ	7rX   c                 óP  — |dk(  ry|dk(  r›|t         j                  k(  rˆ| j                  €J ‚t        j                  t
        j                  | j                  | j                  d  «      }d}t        j                  j                  j                  t        j                  ||«      «      S y)Nri   Tre   é   F)r…   rp   r’  r5  r6  r7  r8  r“  r5   ræ  ÚsizevarsÚguard_or_falser:  ÚGt)r  rˆ   r‰   Ú
use_scalarÚreduction_sizeÚ
chunk_sizes         rY   Úneed_use_acc_helperzCppKernel.need_use_acc_helperÎ  s”   € ð Ð-Ò-Øð ˜UÒ" u´·±Ò';Ø×#Ñ#Ð/Ð/Ð/Ü&×-Ñ-Ü—‘˜d×.Ñ.¨t×/CÑ/CÐ/EÐFóˆNð
 ˆJä—7‘7×#Ñ#×2Ñ2´5·8±8¸NÈJÓ3WÓXÐXØrX   c           
      óR  — |rt        ||«      n|}t        |«      }|dv sJ ‚d}	t        ||	«      }
|dk(  rdnd}|r
t        |   }n&t        | d«      r| j	                  |«      nt        |   }|› d|› d|	› d	|› d
|› d�
}|dk(  r|S t        |
t        j                  «      r
|
dk  rd|› �S |S )N)ri   re   rü  ri   ÚWelfordHelperÚCascadeSumHelperrM  ú<r�   rh  ro  r×   re   r6   zstatic )r   rF   rI   r¦   rM  r£   r:  r;  )r  rˆ   rª   Úhelper_ranger‰   r¥  r   Únum_range_threadÚnum_range_thread_exprr  Ú
num_chunksÚhelper_typeÚh_typeÚhelper_init_lines                 rY   Ú_acc_helper_initzCppKernel._acc_helper_initä  s   € ñ 3>ŒG�L +Ô.À<ð 	ô !,Ð,<Ó =ÐØÐ!:Ñ:Ð:Ð:Øˆ
ÜÐ-¨zÓ:ˆ
ð Ð!1Ò1ñ à#ð 	ñ
 Ü! %Ñ(‰Fô ˜4 Ô1ð ×"Ñ" 5Ô)ä! %Ñ(ð ð ˆm˜1˜V˜H B z l°"°Z°LØØ$Ð%Øðð 	ð ˜UÒ"Ø#Ð#Ü�j¤%§-¡-Ô0°ZÀ1²_ð Ð-Ð.Ð/Ð/à#Ð#rX   c           
      ó>  — t         j                  j                  rdn	t        «       }| j                  j                  | j                  ||||d |«      «       | j                  j                  | j                  ||||||«      «       |r|n|› d�}|dk(  rI| j                  j                  |› d|› d|› d�«       | j                  j                  |› d|› d|› d�«       y | j                  j                  |› d	|› d�«       | j                  j                  |› d
|› d�«       y )Nrª  Ú_vecri   z = welford_combine(r–   r×   z_local = welford_combine(z	_local, &z = cascade_sum_final(&z_local = cascade_sum_final(&)
r   r´  r¬  r,   rœ  rÃ   r  r™  r�  rš  )	r  rˆ   r³   rª   r  r‰   r   r¥  rB  s	            rY   Ú_use_acc_helperzCppKernel._use_acc_helper	  sA  € ô $ŸZ™Z×7Ò7‰MÔ=QÓ=Sð 	ð 	×*Ñ*×4Ñ4Ø×!Ñ!Ø 
¨L¸%ÀÀzóô	
ð
 	×!Ñ!×+Ñ+Ø×!Ñ!Ø 
¨L¸%ÀÈjóô	
ñ
 #‘¨3¨%¨t¨ˆØÐ-Ò-Ø×.Ñ.×8Ñ8Ø�(Ð-¨f¨X°S¸¸ÀBÐGôð ×'Ñ'×1Ñ1Ø�(Ð3°F°8¸9ÀZÀLÐPRÐSõð ×.Ñ.×8Ñ8Ø�(Ð0°°¸BÐ?ôð ×'Ñ'×1Ñ1Ø�(Ð6°z°lÀ"ÐEõrX   c           
      ó¸  — |dv }d }|rt        |t        «      r|\  }}|�t        t        |«      }||||ff}n|||f}|| j                  j
                  v r| j                  j
                  |   S | j                  j                  | j                  d|› �d¬«      }	| j                  j                  |	› «       d| _
        |r|n|}
t        ||
«      }| j                  j                  | j                  |	|||
t        «      «       | j                  ||d«      rát!        j"                  t$        j&                  | j(                  | j*                  d  «      }|dk(  r,| j,                  j                  | j.                  d|› �d¬«      }n+| j0                  j                  | j.                  d|› �d¬«      }d|› �}| j3                  ||	|||d¬«       | j4                  j7                  |	› d	t9        ||	||«      › d
�«       n­|�|}nz| j*                  €J ‚| j:                  | j*                     }t=        | j*                  dz   t?        | j:                  «      «      D ]$  }|| j(                  |   z  | j:                  |   z   }Œ& | j4                  j7                  |	› d	t9        ||	||¬«      › d
�«       | jA                  |	|||
«       tC        ||	«      }|| j                  j
                  |<   |S )N©rc   rb   ú
reduction F©ÚwriteTri   Úscalar_©r   r»   r¼   r6   )r“   )"r£   r¤   r   r:   r   Úreduction_cacher~  r¹  r§  r›  r›  r’   r•  r÷  rŽ   r  r5  r6  r7  r8  r9  r“  r¡  r  r¢  r  r³  rÃ   r±   rZ  r3  rÔ   r±  r¥   )r  r‰   r«   rˆ   rð   Úargmax_or_argminÚlogical_indexÚlogical_index_keyÚreduction_keyr³   Ú
init_dtyperÓ   r  rª   Úscalar_helper_valr“   rç   rB  s                     rY   Ú	reductionzCppKernel.reduction0	  s  € Ø)Ð-AÐAÐØˆÙ¤
¨5´%Ô 8Ø#(Ñ ˆE�=àÐ$Ü $¤[°-Ó @ÐØ% ~¸Ð?PÐ7QÐQ‰Mà% ~°uÐ<ˆMØ˜D×.Ñ.×>Ñ>Ñ>Ø×%Ñ%×5Ñ5°mÑDÐDà× Ñ ×)Ñ)Ø�J‰J˜* ] OÐ4¸Eð *ó 
ˆð 	× Ñ ×'Ñ'¨3¨%Ô1Ø ˆÔÙ"2‘Y¸ˆ
Ü% n°jÓAˆØ×(Ñ(×/Ñ/Ø×&Ñ&Ø�X˜~¨z¼>óô	
ð ×#Ñ# N°E¸4Ô@ä&×-Ñ-Ü—‘˜dŸk™k¨$×*>Ñ*>Ð*@ÐAóˆNð Ð!1Ò1Ø!×4Ñ4×=Ñ=Ø—L‘L J¨}¨oÐ">Àeð >ó ‘
ð "×4Ñ4×=Ñ=Ø—L‘L J¨}¨oÐ">Àeð >ó �
ð #*¨*¨Ð 6ÐØ× Ñ ØØØ!ØØØð !ô ð �K‰K×!Ñ!Ø�%�sÔ,¨^¸SÀ%ÐIZÓ[Ð\Ð\]Ð^õð Ð(Ø%‘à×+Ñ+Ð7Ð7Ð7ØŸ™ d×&:Ñ&:Ñ;�Ü˜t×3Ñ3°aÑ7¼¸T¿]¹]Ó9KÓLò F�AØ! D§K¡K°¡NÑ2°T·]±]À1Ñ5EÑE‘EðFà�K‰K×!Ñ!Ø�%�sÔ,¨^¸SÀ%ÈuÔUÐVÐVWÐXôð 	×,Ñ,¨S°(¸NÈJÔWÜ" >°3Ó7ˆØ<Bˆ×Ñ×*Ñ*¨=Ñ9ØˆrX   c                 óÊ   — | j                  |«      }| j                  j                  |«      }| j                  j	                  t        ||› dt        |«      › d|› d�«      «       y )Nrã  rê  r¼   )r4  ri  rí  r–  rÃ   r<   rF   )r  rÈ   r“   rð   r¨   s        rY   Ústore_reductionzCppKernel.store_reductionw	  s]   € Ø×$Ñ$ UÓ+ˆØ�i‰i×Ñ˜tÓ$ˆØ×Ñ×'Ñ'Ü˜ #  a¬°EÓ(:Ð';¸4À¸wÀaÐHÓIõ	
rX   c                 óª  — | j                   ri| j                   t        |«      t        |«      z   k(  s+J | j                   › dt        |«      › dt        |«      › �«       ‚| j                  t        |«      k(  s¥J ‚t        |«      t        |«      z   | _         | j                   D �cg c]  }| j	                  |«      ‘Œ c}| _        t        t        | j
                  «      «      D �cg c]  }t        t        j                  |«      ‘Œ c}| _
        t        |«      | _        | j                  d | j                   | j                  | j                  d  fS c c}w c c}w )NrÅ  rê   )r’  r¤   r“  rÔ   r4  r9  r3  r/   r   ÚXBLOCKrZ  )r  ÚlengthsÚreduction_lengthsr„  r  s        rY   Ú
set_rangeszCppKernel.set_ranges~	  s2  € Ø×ÒØ×#Ñ#¤u¨W£~¼Ð>OÓ8PÑ'PÒPð Ø×#Ñ#Ð$ D¬¨w«Ð(8¸¼EÐBSÓ<TÐ;UÐVóÐPð ×'Ñ'¬3¨w«<Ò7Ð7Ð7ä$ W›~´Ð6GÓ0HÑHˆDÔØ<@×<LÑ<LÖM°q˜4×/Ñ/°Õ2ÒMˆDŒKô œs 4§;¡;Ó/Ó0öàô /¬t¯{©{¸AÕ>òˆDŒMô
 $' w£<ˆDÔ à�M‰MÐ0˜D×0Ñ0Ð1Ø�M‰M˜$×.Ñ.Ð0Ð1ð
ð 	
ùò Nùòs   Â EÃ !Ec                 óš   — | j                   €J ‚t        | j                   «      }t        j                  j                  j                  |«      S r  )r’  r0   r5   ræ  rý  Úoptimization_hint)r  r†  s     rY   Ú	size_hintzCppKernel.size_hint’	  s?   € Ø×ÑÐ+Ð+Ð+Ü˜T×-Ñ-Ó.ˆÜ�w‰w×Ñ×1Ñ1°$Ó7Ð7rX   c                 ó^  ‡‡‡‡‡‡‡‡‡‡— t        | t        «      sJ ‚t        «       Š| j                  €J ‚t        |j                  t
        «      r+|j                  j                  |j                  «       ‰«      Šn | j                  |j                  «       ‰«      Š|j                  d uxr# |j                  ‰j                     j                  Št        j                  «       5 }‰j                  r6‰r‰j                  «        n‰j                  ‰«       |j!                  ‰«       n4‰dkD  r/‰j#                  «       r|j%                  ‰j'                  «       «       dt(        fˆˆˆfd„Šdd„Š	 ddt(        dt*        fˆˆˆˆˆˆfd„Šddt(        dt*        fˆˆfd„Š	 	 ddt(        dt*        dt,        fˆˆfd	„Š|j%                  ‰j'                  «       «       t        |j                  t
        «      �r t        t.        j0                  t2        «      �rt.        j0                  j4                  rçt.        j0                  j4                  }|j7                  «       D ]º  }t9        |j;                  «       j<                  D �cg c]  }| j?                  |«      ‘Œ c}«      }t@        |j;                  «       jB                     }	d
|	› dtE        |«      › d�}
|jG                  «       }‰jI                  d|	› d|› d|
› d�«       ‰jI                  |	› d|› d|› d�«       Œ¼  ‰|«       d d d «       y c c}w # 1 sw Y   y xY w)Nr6   Ú
_loop_nestc                 ó¤  •‡ — ˆ ˆfd„}‰ j                  «       }t        |t        «      r|j                  D ]
  Š  ‰‰ «       Œ y t        |t        «      sJ ‚‰ j
                  � |«       r|j                  «        t        j                  «       5 }|j                  ‰j                  «       «       |j                  ‰«       d d d «       y # 1 sw Y   y xY w)Nc                  ó†   •— ‰j                   sJ ‚‰j                   ‰j                     } | j                  xr | j                  S r  )r.  rö   r›  Úparallel)Úrootr-  Ú	par_depths    €€rY   Úis_parallel_reductionzOCppKernel.codegen_loops_impl.<locals>.gen_kernel.<locals>.is_parallel_reductionµ	  s=   ø€ Ø%×+Ò+Ð+Ð+Ø%×+Ñ+¨I×,AÑ,AÑB�DØ×,Ñ,Ò>°·±Ð>rX   )Ú
get_kernelr£   rD  rF  ÚCppKernelProxyr.  rµ  rÁ   rÂ   rÄ   rÅ   r¼  )r-  r3  rG  rÍ   r´   Úgen_loop_nestr2  s   `   €€€rY   Ú
gen_kernelz0CppKernel.codegen_loops_impl.<locals>.gen_kernel´	  s±   ù€ õ?ð
 $×.Ñ.Ó0�Ü˜fÔ&:Ô;Ø&,§l¡lò 2˜
Ù% jÕ1ñ2ô & f¬nÔ=Ð=Ð=Ø!×'Ñ'Ð3Ñ8MÔ8OØ×DÑDÔFÜ#×-Ñ-Ó/ð .°5Ø×+Ñ+¨D¯K©K«MÔ:ØŸ™¨Ô-÷.÷ .ñ .ús   Â1CÃCc                 óÆ   — |r0| j                   }|r| j                  |z   }|S | j                  |z   }|S | j                  }|r|| j                  z   }|S || j
                  z   }|S r  )r–  r˜  r�  r”  r—  rœ  )rG  r0  Ú	is_suffixrŸ  Úprefixs        rY   Úget_reduction_prefix_suffixzACppKernel.codegen_loops_impl.<locals>.get_reduction_prefix_suffixÆ	  s}   € ÙØ#×4Ñ4�FÙØ!'×!AÑ!AÀFÑ!J˜ð "�Mð "(×!EÑ!EÈÑ!N˜Ø!�Mà#×4Ñ4�FÙØ!'¨&×*JÑ*JÑ!J˜ð "�Mð "(¨&×*NÑ*NÑ!N˜Ø!�MrX   Údepthc                 ó  •— | j                  «       }| j                  sJ ‚| j                  |   }t        j                  «       5 }|j                  rI|sG ‰	||j
                  d¬«      }|r|j                  ‰j                  «       «       ‰j                  |«       ‰
rR|j
                  rF‰j                  ‰«       |j                  r)|j                  sJ ‚‰j                  |j                  «        ‰| |«       ‰
rC|j
                  r7|j                  r‰j                  |j                  «       ‰j                  «        |j                  r&|s$‰j                   ‰	||j
                  d¬«      «       d d d «       y # 1 sw Y   y xY w)NF)r9  T)r4  r.  rÁ   rÂ   r›  r0  rÄ   rÅ   r  r™  rš  Úclose)r-  r<  Úin_reductionrG  ÚloopÚstack_outerr”  r´   Úgen_loop_atr;  Úis_reduction_loopÚthreadsÚworksharings          €€€€€€rY   Úgen_loop_with_reductionz=CppKernel.codegen_loops_impl.<locals>.gen_loop_with_reductionÖ	  sA  ø€ ð $×.Ñ.Ó0�Ø!×'Ò'Ð'Ð'Ø!×'Ñ'¨Ñ.�Ü×)Ñ)Ó+ð ¨{Ø×(Ò(±Ù+FØ" D§M¡M¸Uô,Ð(ñ ,Ø'×5Ñ5°d·k±k³mÔDØŸ™Ð$4Ô5Ù(¨T¯]ª]Ø#×,Ñ,¨WÔ5Ø!×6Ò6Ø#)×#@Ò#@Ð@Ð#@Ø ŸK™K¨×(CÑ(CÔDá 
¨EÔ2á(¨T¯]ª]Ø!×8Ò8Ø ŸK™K¨×(EÑ(EÔFØ#×)Ñ)Ô+Ø×(Ò(±ØŸ™Ù7Ø &¨¯©Àôô÷+÷ ñ ús   ÁD*E6Å6E?c                 óX  •— t        j                  «       5 }| j                  sJ ‚| j                  |   }|j                  «       }|€
	 d d d «       y ‰j	                  |«       |j                  ‰j                  «       «        ‰| |dz   |j                  «       d d d «       y # 1 sw Y   y xY w©Nr6   )rÁ   rÂ   r.  Úlinesrß   rÄ   rÅ   r›  )r-  r<  rÍ   r@  Ú
loop_linesr´   r6  s        €€rY   rB  z1CppKernel.codegen_loops_impl.<locals>.gen_loop_at÷	  s¡   ø€ Ü×)Ñ)Ó+ð L¨uØ%×+Ò+Ð+Ð+Ø%×+Ñ+¨EÑ2�DØ!%§¡£�JØ!Ð)Ø÷Lð Lð —O‘O JÔ/Ø×'Ñ'¨¯©«Ô6Ù! *¨e°a©i¸×9JÑ9JÔK÷L÷ Lñ Lús   –1B ÁAB Â B)r?  c                 ót   •— | j                   �|t        | j                   «      k(  r	 ‰| «       y  ‰| ||«       y r  )r.  rÔ   )r-  r<  r?  r7  rF  s      €€rY   r6  z3CppKernel.codegen_loops_impl.<locals>.gen_loop_nest
  s7   ø€ ð
 ×#Ñ#Ð+¨u¼¸J×<LÑ<LÓ8MÒ/MÙ˜zÕ*á+¨J¸¸|ÕLrX   zstd::make_unique<z []>(r—   zstd::unique_ptr<z	 []> buf_r»   r¼   z* z = buf_z.get();)FF)r   F)r   )%r£   r5  r,   r’  rG  rD  Údecide_parallel_depthÚmax_parallel_depthr.  rö   r›  rÁ   rÂ   rõ   r>  r0  Úmark_parallelÚsinglerÄ   rÅ   r2  ro   rm   r5   Úlocal_buffer_contextrL   Úlocal_buffersÚvaluesr0   Ú
get_layoutr&  r4  rI   r‰   rE   Úget_namer  )r  r4  r´   rE  rÍ   rQ  Úlocal_bufferÚsize_valÚlocal_buf_sizeÚlocal_buf_dtypeÚallocateÚlocal_buffer_namer7  rB  r6  rF  r;  rC  r2  rD  s     ``        @@@@@@@@rY   Úcodegen_loops_implzCppKernel.codegen_loops_impl—	  s  ÿù€ Ü˜$¤Ô/Ð/Ð/Ü&Ó(ˆØ×ÑÐ+Ð+Ð+Ü�i×&Ñ&Ô(<Ô=Ø!×(Ñ(×>Ñ>Ø×,Ñ,Ó.°ó‰Ið ×2Ñ2Ø×,Ñ,Ó.°óˆIð
 �O‰O 4Ð'ò DØ—‘ 	× 5Ñ 5Ñ6×CÑCð 	ô ×!Ñ!Ó#ð ~	% uØ×'Ò'Ù$à×%Ñ%Õ'à×(Ñ(¨Ô1Ø×'Ñ'¨	Õ2Ø˜1’Ø×%Ñ%Ô'Ø×'Ñ'¨¯©«Ô6ð.¤x÷ .ó$"ð" DIñÜ$ðÜ-0÷ò ñB	L¬ð 	L¼ö 	Lð Ø%*ñMÜ$ðMäðMô #öMð ×Ñ §¡£Ô.ô ˜9×+Ñ+Ô-AÕBÜœq×5Ñ5Ô7IÕJÜ×*Ñ*×8Ò8ô !"× 6Ñ 6× DÑ D�Ø$1×$8Ñ$8Ó$:ò �Lä%2ð -9×,CÑ,CÓ,E×,JÑ,Jöà (ð !×0Ñ0°Õ:òó&�Nô '3°<×3JÑ3JÓ3L×3RÑ3RÑ&S�OØ!2°?Ð2CÀ5ÌÈ~ÓI^ÐH_Ð_`Ða�HØ(4×(=Ñ(=Ó(?Ð%Ø—K‘KØ*¨?Ð*;¸9ÐEVÐDWÐWZÐ[cÐZdÐdeÐfôð —K‘KØ*Ð+¨2Ð.?Ð-@ÀÐHYÐGZÐZaÐbõðñ" ˜)Ô$÷}~	%ð ~	%ùò`÷a~	%ð ~	%ús    Ã!FL#É4LÊB	L#ÌL#Ì#L,c                 óT   — t         j                  | «      }| j                  |||«       y r  )r2  Úbuildr[  )r  r´   rE  r4  s       rY   Úcodegen_loopszCppKernel.codegen_loops(
  s"   € Ü—N‘N 4Ó(ˆ	Ø×Ñ 	¨4°Õ=rX   c                 ó:   — t         j                  j                  ryy)NÚAOTI_TORCH_CHECKÚTORCH_CHECK)r5   ræ  Úaot_moder  s    rY   Úassert_functionzCppKernel.assert_function,
  s   € ä�7‰7×ÒØ%à rX   c                 ó  — | j                   €J ‚| j                   |j                  |j                  |j                  z    }| j                  «       }d}d}|D ]m  }t        j
                  j                  j                  |d¬«      }|d|z  k\  s||k(  r n3||z  t        j                  j                  k  r n|dz  }||z  }||z  }Œo t        j                  j                  r|dk(  rt        |«      dkD  rd}t        ||j                  ¬«      S )Nr6   r   i    ©Úfallbackr   ©rõ   rö   )r’  rö   rõ   r+  r5   ræ  rý  r*  r   r´  Úmin_chunk_sizer¬  rÔ   rô   )	r  rM  rD  r9  ÚseqÚparr<  r†  Úhints	            rY   rL  zCppKernel.decide_parallel_depth3
  s  € Ø×ÑÐ+Ð+Ð+Ø×!Ñ!Ø×*Ñ*Ø"×.Ñ.Ð1C×1RÑ1RÑRð
ˆð
 �n‰nÓˆØˆØˆØò 		ˆDÜ—7‘7×#Ñ#×5Ñ5°dÀTÐ5ÓJˆDØ�a˜'‘kÒ! S¨G¢^ÙØ�g‰~¤§
¡
× 9Ñ 9Ò9áØ�Q‰JˆEØ�4‰KˆCØ�4‰K‰Cð		ô �:‰:×%Ò%¨%°1ª*¼¸V»ÀqºØˆEÜØ Ð.@×.LÑ.Lô
ð 	
rX   c              #   ó  K  — | j                   | j                  | j                  | j                  f}t	        «       | _         t	        «       | _        t	        «       | _        | j                  j                  «       | _        d –— | j                  j                  | j                   «       | j                  j                  | j                  «       | j                  j                  | j                  «       |\  | _         | _        | _        | _        y ­wr  )r¹  r  r³  r}  r>   Úcloner–  r  )r  r¿  s     rY   Úwrite_to_suffixzCppKernel.write_to_suffixP
  sµ   è ø€ à—‘˜TŸ\™\¨4¯;©;¸¿¹ÐAˆÜ#Ó%ˆŒ
Ü%Ó'ˆŒÜ$Ó&ˆŒØ—8‘8—>‘>Ó#ˆŒÛØ×Ñ×$Ñ$ T§Z¡ZÔ0Ø×Ñ×$Ñ$ T§\¡\Ô2Ø×Ñ×$Ñ$ T§[¡[Ô1Ø<AÑ9ˆŒ�T”\ 4¤;°µùs   ‚D
Dc                 ó   — t        |i |¤ŽS r  )rH   )r  ri  rž  s      rY   Úcreate_cse_varzCppKernel.create_cse_var]
  s   € Ü˜tÐ. vÑ.Ð.rX   c                 óÎ   — |rT|t         j                  t         j                  fv r2|t         j                  t         j                  fv rdt
        |   › d|› d�S dt
        |   › d|› d�S )Nzc10::convert<z>(std::round(r¢   r�  r—   )r…   rE  rD  rp   rl   rI   )r  Úsrcr‰   r«   rŠ  s        rY   r|  zCppKernel.get_to_dtype_expr`
  se   € áØœ%Ÿ*™*¤e§k¡kÐ2Ñ2ØœeŸk™k¬5¯<©<Ð8Ñ8à"¤<°Ñ#6Ð"7°}ÀSÀEÈÐLÐLØœ|¨EÑ2Ð3°2°c°U¸!Ð<Ð<rX   c                 ób   — | j                  |||«      }| j                  j                  ||«       y r  )r|  r}  r  )r  ÚdstÚ	dst_dtyperr  r«   r†  s         rY   r‚  zCppKernel.cache_dtype_converti
  s(   € Ø×%Ñ% c¨9°iÓ@ˆØ�‰�‰�T˜3ÕrX   r:  r¨   c                 óV  ‡ ‡
— |€d}‰ j                   syg Š
ˆ
ˆ fd„}|�-|‰ j                   v sJ ‚‰ j                   |   \  }} ||||«      s4y‰ j                   j                  «       D ]  \  }}|\  }} ||||«      rŒ y dj                  ‰
«      }	|	r|j                  d|› d|	› d�«       yy)	NrT   Tc                 ó6  •— | |k(  ryd }t        ‰j                  «      D ]  \  }}||k(  sŒ|} n t        ‰«      t        u r|r| dk(  r|‰j                  |   k(  rd}‰j                  |› dt        | «      › �«       ‰j                  |› dt        |«      › �«       y)NFr   r6   r×  r½   T)rÆ   rZ  r  rŠ  r9  r›  rF   )rº  r»  r¨   Úvar_idrç   Ú_varÚ
conditionsr  s         €€rY   Úgenz)CppKernel.codegen_conditions.<locals>.geny
  s¬   ø€ Ø˜Š|ØØˆFÜ$ T§]¡]Ó3ò ‘��4Ø˜$“;Ø�FÙðô
 �T“
œiÑ'ÙØ˜Q’JØ˜4Ÿ;™; vÑ.Ò.à�à×Ñ   T¬+°eÓ*<Ð)=Ð>Ô?à×Ñ   S¬°SÓ)9Ð(:Ð;Ô<ØrX   FrF  zif(ro  r¢   )r�  rª  ÚjoinrÃ   )r  r´   r:  r¨   r{  rº  r»  ry  Ú_rangeÚjoined_conditionsrz  s   `         @rY   Úcodegen_conditionszCppKernel.codegen_conditionsm
  sÔ   ù€ ð ˆ>ØˆFØ×!Ò!ØØˆ
õ	ð* ˆ?Ø˜$×,Ñ,Ñ,Ð,Ð,Ø×+Ñ+¨CÑ0‰JˆE�3Ù�u˜c 3Ô'Øà $× 2Ñ 2× 8Ñ 8Ó :ò !‘��fØ#‘
��sÙ˜5 # tÕ,Ù ð!ð #ŸK™K¨
Ó3ÐÙØ�N‰N˜S  ¨Ð*;Ð)<¸BÐ?Ô@ØàrX   r  )r6   éÿÿÿÿr   ©NF©F©NN):r÷   rø   rù   rú   rm  rR  rE   rÜ  rž  rŸ  r  r±   rŽ   r±  rµ  r8   r¼  rÁ   ÚcontextmanagerrD  r:  r’  rÅ  rq   rÈ  ÚSymbolrÑ  rÓ  rÕ  rm   rá  rø  rï  ró  r:   r…   r‰   r÷  ro   rú  r  r  r  r!  r#  r(  r+  r[  r^  Úpropertyrc  rL  rn  rp  r|  r‚  r  rM  rN  s   @rY   rŠ  rŠ  ¶  s)  ø„ ñð €IØ€EØ€MØ€Fô'1ð^ /Ø(ó'
òRIñ˜\¨DÑ0ó ð( ×Ññ$ó ð$ð& BCñØ—Z‘Zóð2 %§*¡*ð 2°ó 2ð	
¨u¯z©zð 	
ÀEÇLÁLó 	
ð
 e§j¡jð 
¸5¿<¹<ó 
ò
5ð(:à�j‰jð(:ð �j‰jð(:ð ð	(:ð
 ó(:ðT˜ð  U§Z¡Zó ó8ò$
ð
à˜3Ñðð ðð ð	ð
 �{‰{óñ:7¨c°D©jó 7òð8 Øó*$ðZ PUóò@EòN
ò
ò(8ò
O%òb>ð ð! ò !ó ð!ò
ð: ×Ññ
Bó ð
Bò/ó=ò ð "Ø#'ñ	0àð0ð �d‘
ð0ð �\‰\˜DÑ ÷	0rX   rŠ  c                   óz  ‡ — e Zd ZeZ	 d(ˆ fd„	Zdej                  dej                  fd„Z	de
j                  defd„Zde
j                  defd	„Zde
j                  defd
„Ze
j                  fde
j                  defd„Zdede
j                  defd„Z	 d(dedej                  de
j                  dedz  fd„Z	 	 	 d)dedz  dej                  de
j                  dedz  deez  dz  dededz  fd„Zdedej                  fˆ fd„Z	 d*deez  dedej                  de
j                  def
d„Zd(d„Zdej                  dej                  fd„Zd„ Zd„ Zdedefd„Zdedej                  defd „Z d!„ Z!d"„ Z"ddde
jF                  fdej                  ez  dz  d#edz  d$e
j                  dz  fd%„Z$d(ˆ fd&„	Z%d*ˆ fd'„	Z&ˆ xZ'S )+rœ  Nc                 óØ   •— t         ‰| �  ||«       t        j                  «       | _        | j                  sJ ‚|dkD  sJ d«       ‚|| _        || _        || _        |r|| _        y || _        y )Nr   z0Expect pass in Non-Zero tiling_factor explicitly)	r  r  r   Úpick_vec_isaÚvec_isar  r[  rl  r(  )r  ri  r¥  r  r[  rl  r  s         €rY   r  zCppVecKernel.__init__£
  si   ø€ ô 	‰Ñ˜˜{Ô+Ü"×/Ñ/Ó1ˆŒØ�|Š|Ðˆ|Ø˜qÒ ÐTÐ"TÓTÐ Ø*ˆÔØ$ˆŒØ"ˆŒÙ&/˜ˆ�°]ˆ�rX   r“   rÉ  c                 óì   ‡ — ‰ j                  ||«      ry ˆ fd„|j                  D «       D ]"  }t        |t        «      sJ ‚|j                  sŒ" y  t        ||‰ j                  «      }|j                  r|S d S )Nc              3   ó˜   •K  — | ]A  }t        |t        j                  «      r%‰j                  j                  |j
                     –— ŒC y ­wr  ©r   r   rÛ  r}  rÌ  rÈ   ©r  rÎ  r  s     €rY   r  z5CppVecKernel._try_get_const_stride.<locals>.<genexpr>·
  s:   øè ø€ ò 
àÜ˜a¤§¡Ô*ð �H‰H× Ñ  §¡Õ(ñ
ùó   ƒAA
)rÑ  rÏ  r£   rH   r§   r%   r  r™  )r  r“   rÉ  Úindirect_varr`  s   `    rY   r\  z"CppVecKernel._try_get_const_stride´
  s|   ø€ Ø×)Ñ)¨%°Ô9Øó
à×'Ñ'ô
ò 	ˆLô
 ˜l¬NÔ;Ð;Ð;Ø×"Ó"Ùð	ô % U¨G°T×5GÑ5GÓHˆØ×)Ò)ˆvÐ3¨tÐ3rX   r‰   r¹   c                 ó¨   — t        j                  | j                  |j                  z  dz  | j                  j                  «       z  «      }|dk\  sJ ‚|S )Né   r6   )ÚmathrÌ  r  ÚitemsizerŠ  Ú	bit_width©r  r‰   Únum_vectorss      rY   rm  zCppVecKernel._get_num_vectorsÂ
  sO   € Ü—i‘iØ×Ñ §¡Ñ/°!Ñ3°d·l±l×6LÑ6LÓ6NÑNó
ˆð ˜aÒÐÐØÐrX   c                 óp   — | j                   |j                  z  dz  | j                  j                  «       z  S )Nr’  )r  r”  rŠ  r•  )r  r‰   s     rY   r  z!CppVecKernel._get_raw_num_vectorsÉ
  s0   € ð ×!Ñ! E§N¡NÑ2°QÑ6¸¿¹×9OÑ9OÓ9QÑQÐQrX   c                 óh   — | j                  |«      }|dk(  rdt        |   › d�S dt        |   › d|› d�S )Nr6   rf  r�   rg  ru  )rm  rI   r–  s      rY   rM  zCppVecKernel._get_vec_typeÎ
  sJ   € Ø×+Ñ+¨EÓ2ˆØ˜!ÒØ)¬,°uÑ*=Ð)>¸aÐ@Ð@à*¬<¸Ñ+>Ð*?¸qÀÀÈQÐOÐOrX   c                 ól   — |t         j                  k(  ry| j                  |«      }dt        |   › d|› d�S )NrT   rt  ru  r�   )r…   rm   rm  rI   r–  s      rY   rÇ  zCppVecKernel._get_mask_typeÕ
  s<   € Ø”E—J‘JÒØØ×+Ñ+¨EÓ2ˆØ"¤<°Ñ#6Ð"7°q¸¸ÀQÐGÐGrX   r>  c                 ó¢   — |j                   t        j                  k(  sJ t        |«      «       ‚| j	                  |«      }|› dt
        |   › d|› d�S )Nz.template cast<ru  rƒ   )r‰   r…   rm   Úreprrm  rI   )r  r>  r‰   r—  s       rY   r7  zCppVecKernel._get_mask_castÛ
  sP   € Ø�z‰zœUŸZ™ZÒ'Ð3¬¨d«Ó3Ð'Ø×+Ñ+¨EÓ2ˆØ��¤|°EÑ':Ð&;¸1¸[¸MÈÐMÐMrX   r¨   Ú	load_maskc                 ó  — t         |   }| j                  |«      }d}|rS|j                  s&| j                  t        j
                  «      › d|› d�}n!| j                  |t        j
                  «      › }|dk7  r|› dt        |«      › �n|}|t        j                  k(  r.| j                  «       › d|› dt        | j                  «      › d�}	|	S |r|› d|› d|› d	|› d�n,| j                  |«      › d
|› dt        | j                  «      › d�}	|	S )a  
        Get a load line str that loads a vector from `var` at `index` of type `dtype`.
        If `load_mask` is not None, we do a masked load accordingly.
        Notes on the `dtype`:
        1. We always load `self.tiling_factor` number of elements regardless of the `dtype`.
           It means we load half of the vector lanes for 16-bit data types and quarter of the
           vector lanes for 8-bit data types.
        2. `torch.bool` and `torch.uint8` could mean masks and we load them as float mask vectors.
        NrL  r—   r   rê   r�   z.template loadu<ru  r�  ú::loadu()rI   rm  r§   rÇ  r…   rp   r7  rF   rm   r(  rM  )
r  r¨   r“   r‰   r�  Úcpp_typer—  Úload_mask_strÚloadbufrÉ   s
             rY   Ú_get_vec_load_linezCppVecKernel._get_vec_load_lineà
  s*  € ô    Ñ&ˆØ×+Ñ+¨EÓ2ˆØˆÙØ×#Ò#à#'×#6Ñ#6´u·{±{Ó#CÐ"DÀGÈIÈ;ÐVWÐ X‘à#'×#6Ñ#6°yÄ%Ç+Á+Ó#NÐ"O�Ø5:¸a²Z�S�E˜œ[¨Ó/Ð0Ñ1ÀSˆØ”E—J‘JÒà×)Ñ)Ó+Ð,¨G°G°9¸B¼{È4Ï>É>Ó?ZÐ>[Ð[\Ð]ˆDð ˆñ !ð !�/Ð!1°(°¸1¸[¸MÈÈGÈ9ÐTUÑVà×*Ñ*¨5Ó1Ð2°(¸7¸)À2ÄkÐRV×R`ÑR`ÓFaÐEbÐbcÐdð ð
 ˆrX   rä   Ústore_valueÚ
accu_storec                 ó^  ‡ ‡‡‡— |r	|€J d«       ‚|r|sJ ‚‰€‰ j                   Šdt        j                  dt        fˆ fd„Šdt        j                  dt        fˆ fd„Šdt        dt        fˆˆˆˆ fd„}t        «       }|j                  d	«       |j                  «       5   ‰|«      }	 ‰|«      }
d
t        |   › d|
› d�}|j                  |«       |r |j                  |› dt        |	«      › d�«       t        ‰ j                  ‰ j                     › d�«      }i }ˆ fd„|j                  D «       D ]4  }t        |t        «      sJ ‚|j                  sŒ" ||«      }|› d|› d�||<   Œ6 ‰ j!                  |‰ j                  |¬«      }d}‰ j"                  �l|rJ d«       ‚t        ‰ j"                  t        «      sJ ‰ j"                  «       ‚‰ j"                  j                  r‰ j"                  › d|› d�}n‰ j"                  › d�}t%        j&                  «       r|j                  d‰ j(                  › �«       n|j                  d‰ j(                  › �«       |j                  d|› d�|› dt        ‰ j*                  «      › d�z   |› d�z   «       |j                  «       5  t-        j.                  «       5 }t        |«      }|D ]#  }t1        j2                  d|› z   dz   ||   |«      }Œ% |�|› d|› d�n|› }|r4|j                  d |› d�«       |j5                  |j                  «       «       |r!|rd!nd"}|j                  |› d#|› d$|› d%�«       n|j                  d&|› d'|› d(�«       ddd«       ddd«       |s(‰ j7                  d)d*|«      }|j                  d+|› d(�«       ddd«       |j                  d,«       |r#|j                  d(«       ‰j9                  |«       y‰ j:                  j=                  ‰||¬-«      }t        |t        «      sJ ‚d.|_        |S # 1 sw Y   Œ²xY w# 1 sw Y   Œ¶xY w# 1 sw Y   Œ�xY w)/a÷  
        Load or store a vector in a non-contiguous way. The vector is initialized from an array that is
        filled in an inner loop over the tiling factor.
        :param var: buffer to load from or store to, i.e. `var[transformed(index)]`. If None, we load the index
                    as index expression, i.e. `transformed(index)`.
        :param index: index into the `var` or the index expression by its own if `var` is None.
                      The `index` could contain indirect indexing or the tiling itervar. When used in
                      the inner loop, the index is transformed as follows:
                      1. the index is linearized along the tiling dim.
                      2. the indirect indexing vector variables are transformed into arrays over the tiling dim.
        :param dtype: data type of `var` or `index` if `var` is None.
        :param buffer: the code buffer to write the generated code to. If None, we write to `self.loads`.
        :param store_value: the value to store. If None, we load the vector.
        :param accu_store: whether accumulate the store_value to store_ptr. If True, a store_value should be provided
        :return: a CppCSEVariable that represents the loaded vector or None if it is a store.
        Nzstore var must be providedr‰   r¹   c                 ór   •— | j                   dk  r‰j                  d| j                   z  z  S ‰j                  S ©Né   )r”  r(  ©r‰   r  s    €rY   Úget_result_sizezCCppVecKernel._load_or_store_non_contiguous.<locals>.get_result_size$  s1   ø€ Ø�~‰~ Ò!Ø—~‘~¨¨e¯n©nÑ)<Ñ=Ð=à—~‘~Ð%rX   c                 ór   •— | j                   dk  r‰j                  d| j                   z  z  S ‰j                  S r¨  )r”  r  rª  s    €rY   Úget_tiling_sizezCCppVecKernel._load_or_store_non_contiguous.<locals>.get_tiling_size*  s5   ø€ Ø�~‰~ Ò!Ø×)Ñ)¨Q°%·.±.Ñ-@ÑAÐAà×)Ñ)Ð)rX   Úvec_varc                 óL  •— | j                   sJ ‚t        «       }|j                  d«       |j                  «       5  | j                  }|€J ‚|t
        j                  k(  rt
        j                  } ‰|«      } ‰	|«      }|j                  dt        |   › d|› d�«       | › dt        |«      › d�}|j                  |«       |j                  d«       d d d «       |j                  d«       ‰
j                  j                  ‰|«      }t        |t        «      sJ ‚|S # 1 sw Y   ŒJxY w)	NrO  ri  r�   rj  rk  r×   zreturn tmpbuf;rh  )r§   r8   rÃ   rÅ   r‰   r…   rm   rp   rI   rF   r}  r~  r£   rH   )r®  r´   rv  Úresult_sizeÚtiling_sizerÉ   r‡  rä   r«  r­  r  s          €€€€rY   Úvec_to_arrayz@CppVecKernel._load_or_store_non_contiguous.<locals>.vec_to_array0  s  ø€ Ø—>’>Ð!�>Ü“>ˆDØ�N‰N˜5Ô!Ø—‘“ñ 1Ø#ŸM™M�	Ø Ð,Ð,Ð,Ø¤§
¡
Ò*Ü %§¡�IÙ-¨iÓ8�Ù-¨iÓ8�Ø—‘Ø.¬|¸IÑ/FÐ.GÀrÈ+ÈÐV_Ð`ôð "˜Ð"8¼À[Ó9QÐ8RÐRTÐU�Ø—‘˜tÔ$Ø—‘Ð/Ô0÷1ð �N‰N˜4Ô Ø—X‘X×&Ñ& v¨tÓ4ˆFÜ˜f¤nÔ5Ð5Ð5ØˆM÷!1ð 1ús   »BDÄD#rO  ri  r�   rj  rk  r×   r0  c              3   ó˜   •K  — | ]A  }t        |t        j                  «      r%‰j                  j                  |j
                     –— ŒC y ­wr  r�  rŽ  s     €rY   r  z=CppVecKernel._load_or_store_non_contiguous.<locals>.<genexpr>W  s:   øè ø€ ò !àÜ! !¤T§X¡XÔ.ð —‘×$Ñ$ Q§V¡VÕ,ñ!ùr�  rã  rä  ©rÂ  r'  zunexpected store with load maskz.is_masked(r—   z != 0z#pragma GCC unroll z#pragma unroll ú
for (long ú = 0; r½   r¾   r¿   rÀ   rP  ú+=ú=r™   z tmpbuf[rÙ   ztmpbuf[rê  r¼   ztmpbuf.data()r   r<  rh  re  T)r¹  r…   r‰   ro   rH   r8   rÃ   rÅ   rI   rF   r.   rZ  r[  rÏ  r£   r§   rÅ  r¾  r   Úis_gccr  r(  rÁ   rÂ   rÊ   rË   rÄ   r£  r  r}  r~  )r  r¨   r“   r‰   rä   r¤  r¥  r²  r´   r°  r±  Úresult_declareÚitervar_innerÚreplacementsr�  Ú	array_varr�  rÍ   Úindex_cÚrhsr­   Ú	load_liner‡  r«  r­  s   `   `                  @@rY   r^  z*CppVecKernel._load_or_store_non_contiguous  sd  û€ ñ2  # /ÐOÐ3OÓOÐ1ÙÙÐ�;Øˆ>Ø—Z‘ZˆFð	&¤5§;¡;ð 	&´3õ 	&ð	*¤5§;¡;ð 	*´3õ 	*ð	¤.ð 	´^÷ 	ð 	ô, ‹~ˆØ�‰�uÔØ�[‰[‹]ñ @	7Ù)¨%Ó0ˆKÙ)¨%Ó0ˆKà*¬<¸Ñ+>Ð*?¸rÀ+ÀÈiÐXð ð �N‰N˜>Ô*ÙØ—‘Ø"�mÐ#9¼+ÀkÓ:RÐ9SÐSUÐVôô /Ø—=‘= §¡Ñ1Ð2°&Ð9óˆMð ˆLó!à×+Ñ+ô!ò Q�ô
 " ,´Ô?Ð?Ð?Ø×&Ó&Ù ,¨\Ó :�IØ4=°;¸aÀ¸ÈaÐ1P�L Ò.ðQð ×0Ñ0Ø 4§?¡?¸=ð 1ó ˆEð ˆIØ�‰Ð*Ù&ÐIÐ(IÓI�Ü! $§/¡/´>ÔBÐSÀDÇOÁOÓSÐBØ—?‘?×)Ò)Ø#'§?¡?Ð"3°;¸}¸oÈQÐ O‘Ià#'§?¡?Ð"3°5Ð 9�IÜ×!Ñ!Ô#Ø—‘Ð!4°T×5GÑ5GÐ4HÐIÕJà—‘ °×1CÑ1CÐ0DÐEÔFØ�N‰NØ˜]˜O¨6Ð2Ø"�O 3¤{°4·>±>Ó'BÐ&CÀ2ÐFñGà"�O 3Ð'ñ(ôð
 —‘“ñ H¤
× 4Ñ 4Ó 6ð H¸%Ü% eÓ,�à$0ò �LÜ Ÿf™fØ < .Ñ1°EÑ9Ø$ \Ñ2Øó‘Gðð .1¨_˜˜˜Q˜w˜i qÑ)ÀWÀI�ÙØ—N‘N T¨)¨°AÐ#6Ô7Ø×'Ñ'¨¯©«Ô6ÙÙ*4¡$¸#�KØ—N‘N c U¨!¨K¨=¸ÀÀÈrÐ#RÕSà—N‘N W¨]¨O¸4À¸uÀAÐ#FÔG÷#H÷ Hñ$ Ø ×3Ñ3°OÀQÈÓN�	Ø—‘ ¨¨°1Ð5Ô6÷A@	7ðB 	�‰�tÔÙØ�N‰N˜3ÔØ�M‰M˜$ÔØà—X‘X×&Ñ& v¨t¸5Ð&ÓAˆFÜ˜f¤nÔ5Ð5Ð5Ø ˆFŒMØˆM÷=Hð Hú÷ Hð Hú÷Y@	7ð @	7úsE   ÂB1P#ÅEP#ÊPÊ(B2PÍPÍ"2P#ÐPÐPÐP 	ÐP#Ð#P,rÈ   c                 ó:  •— | j                   j                  |«      }| j                  |«      }t        j                  j                  |«      }| j                  | j                     }| j                  ||«      }|dk(  rt        ‰	| �)  ||«      S |dk(  rG| j                  |||| j                  «      }| j                  j                  | j                  ||¬«      }n| j!                  |||«      }t#        |t$        «      sJ ‚|j'                  d| ||fi «       d|_        |S )Nr   r6   re  rø  T)ri  rå  r4  r5   ræ  rç  rZ  r[  r\  r  rø  r£  r¾  r}  r~  r¹  r^  r£   rH   r€  r§   )
r  rÈ   r“   r¨   r‰   r_  r`  rÉ   r‡  r  s
            €rY   rø  zCppVecKernel.load”  sþ   ø€ Ø�i‰i�o‰o˜dÓ#ˆØ×$Ñ$ UÓ+ˆÜ—‘×!Ñ! $Ó'ˆØ—]‘] 4§?¡?Ñ3ˆ
Ø×+Ñ+¨E°:Ó>ˆØ�QŠ;ä‘7‘<  eÓ,Ð,Ø�qŠ[à×*Ñ*¨3°°u¸d¿o¹oÓNˆDØ—X‘X×&Ñ& t§z¡z°4¸uÐ&ÓE‰Fà×7Ñ7¸¸UÀEÓJˆFÜ˜&¤.Ô1Ð1Ð1Ø×Ñ˜f t¨T°5Ð&9¸2Ô>ØˆŒØˆrX   rð   c           	      óØ  — t        |t        «      s#t        |t        «      r|j                  sJ |«       ‚| j                  | j
                     }|› dt        |«      › �}| j                  ||«      }t        «       }	|dk(  rÕ|rl|t        j                  k(  r#| j                  €| j                  |«      › d|› d�n,| j                  |«      › d|› dt        | j                  «      › d�}
d|› d|
› d�}|t        j                  k(  r%| j                  €|	j                  |› d|› d�«       |	S |	j                  |› d|› dt        | j                  «      › d�«       |	S | j                  ||||	||¬	«       |	S )
a2  
        Get a store line buffer that stores `value` into `var` at `index` of `dtype`. It handles
        both contiguous and non-contiguous store cases.
        :param value: Vectorized type templaterized on `dtype`.
        :param var: buffer to store into.
        :index: index into the `var`.
        rê   r6   rŸ  r—   r�   ro  ú.store(r×   )rä   r¤  r¥  )r£   rq   rH   r§   rZ  r[  rF   r\  r>   r…   rp   rl  rM  r(  rÃ   r^  )r  rð   r¨   r“   r‰   r¥  r_  Úvar_exprr`  r´   rø  s              rY   Ú_get_store_linezCppVecKernel._get_store_line¨  s‘  € ô" ˜%¤Ô%Ü�uœnÔ-°%·,²,ð	àó	ð 
ð —]‘] 4§?¡?Ñ3ˆ
Ø�U˜#œk¨%Ó0Ð1Ð2ˆØ×+Ñ+¨E°:Ó>ˆÜÓˆØ�QŠ;Ùð ¤§¡Ò+°·±Ð0Fð ×)Ñ)¨%Ó0Ð1°¸(¸À1ÑEà ×.Ñ.¨uÓ5Ð6°h¸x¸jÈÌ;ÐW[×WeÑWeÓKfÐJgÐghÐið ð
 ˜E˜7 # d V¨1Ð-�ØœŸ™Ò#¨¯©Ð(>Ø—‘ % ¨°¨z¸Ð<Ô=ð ˆð —‘Ø�g˜W X J¨b´¸T¿^¹^Ó1LÐ0MÈRÐPôð ˆð ×.Ñ.Ø�U˜E¨$¸EÈjð /ô ð ˆrX   c                 óŠ  ‡— d‰v sJ ‚t        |t        «      sJ |«       ‚|j                  s| j                  |«      }| j                  j                  ‰«      }| j                  |«      }t        j                  j                  ‰«      }|€B| j                  ||||«      }| j                  j                  |j                  ˆfd„«      «       y |dk(  �rft        j                  j                   sT| j"                  dk(  rE| j                  |› |||d¬«      }| j                  j                  |j                  ˆfd„«      «       y | j%                  |«      }| j%                  t&        j(                  «      }	t*        |   }
t-        j.                  |t&        j(                  «      j0                  }t        |t        «      r|j                  sJ ‚| j2                  r,d|
› d	|	› d	|› d
|› d	|› d	|› d	t5        | j2                  «      › d�}nd|
› d	|	› d	|› d
|› d	|› d	|› d�}| j                  j7                  t9        ‰|«      «       y t;        d|› �«      ‚)Nré  c                 ó   •— t        ‰| «      S r  ©r<   ©r„  rÈ   s    €rY   ú<lambda>z$CppVecKernel.store.<locals>.<lambda>ß  s   ø€ ´,¸tÀQÓ2G€ rX   rë  r6   T)r¥  c                 ó   •— t        ‰| «      S r  rÈ  rÉ  s    €rY   rÊ  z$CppVecKernel.store.<locals>.<lambda>é  s   ø€ ´lÀ4ÈÓ6K€ rX   zatomic_add_vec<r�   r�  r×   rì  )r£   rH   r§   r�  ri  rí  r4  r5   ræ  rç  rÅ  r³  r  Úmapr   r´  r¬  r¥  rm  r…   rš  rI   r3   r6  rð   rl  rF   rÃ   r<   rŽ  )r  rÈ   r“   rð   rî  r¨   r‰   r´   Ún_srcÚn_idxrŠ   rÉ   s    `          rY   rï  zCppVecKernel.storeÔ  s  ø€ Ø˜‰}Ðˆ}Ü˜%¤Ô0Ð7°%Ó7Ð0Ø�|Š|à—N‘N 5Ó)ˆEØ�i‰i×Ñ˜tÓ$ˆØ×$Ñ$ UÓ+ˆÜ—‘×!Ñ! $Ó'ˆØˆ<Ø×'Ñ'¨¨s°E¸5ÓAˆDØ�K‰K×Ñ˜tŸx™xÓ(GÓHÕIØ�\Ó!Ü—:‘:×-Ò-°$×2BÑ2BÀaÒ2GØ×+Ñ+Ø�gØØØØ#ð ,ó �ð —‘×"Ñ" 4§8¡8Ó,KÓ#LÕMà×-Ñ-¨eÓ4�Ø×-Ñ-¬e¯k©kÓ:�Ü% eÑ,�ÜŸ™ u¬e¯k©kÓ:×@Ñ@�Ü! %¬Ô8¸U¿\º\ÐIÐIØ—>’>Ø,¨V¨H°B°u°g¸RÀ¸wÀbÈÈÈRÐPUÈwÐVXÐY^ÐX_Ð_aÔbmÐnr×n|Ñn|Ób}Ða~ð  Að  B‘Dà,¨V¨H°B°u°g¸RÀ¸wÀbÈÈÈRÐPUÈwÐVXÐY^ÐX_Ð_aÐb�DØ—‘×%Ñ%¤l°4¸Ó&>Õ?ä%¨°D°6Ð&:Ó;Ð;rX   c                 ó   — |S r  rW   rÇ  s     rY   Ú_adjust_argreduce_indexz$CppVecKernel._adjust_argreduce_indexø  s   € ØˆrX   c           
      ó&  — |t         v sJ ‚|dv }d}|rt        |t        «      r|\  }}| j                  | j                  k\  }|r|n|}t        |t
        «      sJ |«       ‚t        |t
        t        d«      f«      sJ |«       ‚|j                  s| j                  |«      }|�t        t
        |«      }	||||	ff}
n|||f}
|
| j                  j                  v r| j                  j                  |
   S d}|› dt        |   › d�}t        ||«      }| j                  ||«      }| j                  j                  | j                   d|
› �d¬«      }t        |t
        «      sJ ‚|› d	�}d
|› �}d
|› �}| xj"                  |› ||gz  c_        d| _        | j&                  j)                  | j+                  ||||t,        «      «       | j&                  j)                  | j+                  ||||| j.                  «      «       | j1                  ||d«      }|�rå|�J |«       ‚| j&                  j)                  | j+                  ||||| j.                  «      «       | j                  €J ‚t3        j4                  t6        j8                  | j:                  | j                  d «      }|dk(  r,| j<                  j                  | j>                  d|
› �d¬«      }n+| j@                  j                  | j>                  d|
› �d¬«      }d
|› �}tC        | j:                  | j                     | jD                  «      rl| j                  | j                  k\  rQtC        || j:                  | j                     «      tC        | j:                  | j                     | jD                  «      z  n|ntG        jH                  d«      }| j:                  | j                     | jD                  z  r>| j                  | j                  k\  r#tC        || j:                  | j                     «      n|ntG        jH                  d«      }d|› �}| jK                  |||||d¬«       | jK                  |||||«       | jK                  |||||«       | jL                  r|n|}| jL                  r|n|}|dk(  r4| jN                  jQ                  |› d| jS                  ||||«      › d�«       nñ| jN                  jQ                  |› d| jS                  ||||«      › d�«       n½|�|}d}n|| j                  €J ‚| jT                  | j                     }tW        | j                  dz   tY        | jT                  «      «      D ]$  }|| j:                  |   z  | jT                  |   z   }Œ& |}||||dœ}| jN                  jQ                  |› d | jR                  ||fi |¤Ž› d�«       | j[                  ||||| jR                  | j.                  ¬«       | j[                  ||||t\        t,        ¬«       |r+| j[                  ||||| jR                  | j.                  ¬«       |t^        j`                  k(  } |�rDtc        |«      rS| je                  |«      dv sJ d«       ‚d|› d�}!d|› d�}"| jf                  jQ                  |› dt]        |||"«      › d�«       nµ|r	|› d|› d�}!nª| r|dv rd|› d�}!n�|dk(  sJ ‚|› d�}!n�d | jS                  |d!d"«      z   d#z   }#|t^        j`                  k(  } | rt^        jh                  n|}$d$t        |$   › d�}d%t        |$   › d&| je                  |$«      › d�}%|› }&|r|dk(  sJ ‚|› d'|› �}&|%› d(|› d)|› d*|#› d&|&› d�
}!| jf                  jQ                  |› dt]        |||!|¬+«      › d�«       |}'nq|}'tc        |«      r2d
|'› �}(| jf                  jQ                  |'› dt]        ||'|(«      › d�«       n2|r0|dk(  sJ ‚d
|'› �}(| jf                  jQ                  |'› d|'› d'|(› d�«       tk        ||'«      })|)| j                  j                  |
<   |)S ),aw  
        Perform vectorized reduction operation.

        This method handles vectorized reduction for different reduction types.
        It manages special cases for low-precision floating point types and
        employs precision improvement techniques for certain reduction operations.

        Args:
            dtype: The output data type for the reduction result
            src_dtype: The source data type of the input value
            reduction_type: Type of reduction operation (sum, min, max, etc.)
            value: The input value to reduce

        Returns:
            The result of the reduction operation
        r  Nzat::vecz::Vectorized<r�   r  Fr  r  Úmasked_Tri   r   r  r  re   r»   r¼   r6   )r©   r“   Úhorizontal_reductionr«   )r­  r®  )r6   r   z4Welford reduction does not support VectorizedN (N>2)zwelford_vec_reduce_all(r—   z_vec_reduce_all()rh   re   ra   rJ  z.all_zero()r`   z.all_masked()z	{ return r„  ré  z; }rf  zat::vec::vec_reduce_all<r�   rê   z([](z& x, z& y) r«  )6ÚVECTORIZABLE_RTYPESr£   r¤   r[  r“  rH   r  r§   r�  r   r   r  rI   r’   Úreduction_acc_type_vecr~  r¹  r§  r›  r•  r›  r÷  rŽ   Úreduction_init_vecr  r5  r6  r7  r8  r9  r¡  r  r¢  r   r  r:  r;  r  rl  r³  rÃ   Úreduction_combine_vecrZ  r3  rÔ   r±  r±   r…   rm   r+   rm  r–  rp   r¥   )*r  r‰   r«   rˆ   rð   r  r  rÓ  r  r  r  Úvec_nsrx  rÓ   Úacc_type_vecr³   Úacc_vecÚ
masked_accÚmasked_acc_vecÚuse_acc_helperr  rª   Úmasked_helper_valÚhelper_vec_rangeÚmasked_helper_vec_ranger   Úacc_vec_Úhelper_val_r“   Úindex_horizontal_reductionrç   rž  r¬   r©   Úmasked_next_valueÚreduce_all_bodyrv  Úvec_reduce_all_funcÚ
result_vecÚtmpvarÚmasked_tmpvarrB  s*                                             rY   r!  zCppVecKernel.reductionû  sG	  € ð& Ô!4Ñ4Ð4Ð4Ø)Ð-AÐAÐØˆÙ¤
¨5´%Ô 8Ø#(Ñ ˆE�=à#Ÿ™°$×2FÑ2FÑFÐÙ"2‘Y¸ˆ
Ü˜%¤Ô0Ð7°%Ó7Ð0Ü˜-¬.¼$¸t»*Ð)EÔFÐUÈÓUÐFà�|Š|Ø—N‘N 5Ó)ˆEàÐ$Ü $¤^°]Ó CÐØ% ~¸Ð?PÐ7QÐQ‰Mà% ~°uÐ<ˆMØ˜D×.Ñ.×>Ñ>Ñ>Ø×%Ñ%×5Ñ5°mÑDÐDàˆØ�˜¤l°5Ñ&9Ð%:¸!Ð<ˆÜ% n°jÓAˆØ×2Ñ2°>À:ÓNˆà× Ñ ×)Ñ)Ø�J‰J˜* ] OÐ4¸Eð *ó 
ˆô ˜#œ~Ô.Ð.Ð.Ø�E˜�,ˆØ˜s˜e�_ˆ
Ø" 7 )Ð,ˆØ× Ò ¨ u¨w¸Ð$GÑGÕ Ø ˆÔØ×(Ñ(×/Ñ/Ø×&Ñ&Ø�X˜~¨z¼>óô	
ð
 	×(Ñ(×/Ñ/Ø×&Ñ&ØØØØØ×'Ñ'óô	
ð ×1Ñ1°.À%ÈÓOˆÚØ Ð(Ð7¨-Ó7Ð(à×,Ñ,×3Ñ3Ø×*Ñ*Ø"Ø Ø"ØØ×+Ñ+óôð ×'Ñ'Ð3Ð3Ð3Ü&×-Ñ-Ü—‘˜dŸk™k¨$×*>Ñ*>Ð*@ÐAóˆNð Ð!1Ò1Ø!×4Ñ4×=Ñ=Ø—L‘L J¨}¨oÐ">Àeð >ó ‘
ð "×4Ñ4×=Ñ=Ø—L‘L J¨}¨oÐ">Àeð >ó �
ð #*¨*¨Ð 6Ðô ˜DŸK™K¨¯©Ñ8¸$×:LÑ:LÔMð —‘¨$×*>Ñ*>Ò>ô ˜^¨T¯[©[¸¿¹Ñ-IÓJÜ˜tŸ{™{¨4¯?©?Ñ;¸T×=OÑ=OÓPòQñ (ô —]‘] 1Ó%ð ð  —;‘;˜tŸ™Ñ/°$×2DÑ2DÒDð —‘¨$×*>Ñ*>Ò>ô ˜^¨T¯[©[¸¿¹Ñ-IÔJá'ô —]‘] 1Ó%ð $ð #*¨*¨Ð 6ÐØ× Ñ ØØØ!ØØØð !ô ð × Ñ Ø  ZÐ1AÀ5ôð × Ñ ØØØ!Ø'Øôð *.¯ª‘~¸WˆHØ/3¯~ª~Ñ+À:ˆKØ Ò&Ø—‘×%Ñ%Ø�j  D×$>Ñ$>¸~ÈxÐY^Ð`kÓ$lÐ#mÐmnÐoõð —‘×%Ñ%Ø�j  D×$>Ñ$>¸~ÈxÐY^Ð`kÓ$lÐ#mÐmnÐoõð Ð(Ø%�Ø-2Ñ*à×+Ñ+Ð7Ð7Ð7ØŸ™ d×&:Ñ&:Ñ;�Ü˜t×3Ñ3°aÑ7¼¸T¿]¹]Ó9KÓLò F�AØ! D§K¡K°¡NÑ2°T·]±]À1Ñ5EÑE‘EðFà-AÐ*à#ØØ(BØ&ñ	ˆFð �K‰K×!Ñ!Ø�)˜3Ð9˜t×9Ñ9¸.È'Ñ\ÐU[Ñ\Ð]Ð]^Ð_ôð 	×,Ñ,ØØØØØ!%×!;Ñ!;Ø"×5Ñ5ð 	-ô 	
ð 	×,Ñ,ØØØØÜ!2Ü,ð 	-ô 	
ñ à×0Ñ0ØØØØØ%)×%?Ñ%?Ø"&×"9Ñ"9ð 1ô ð œ5Ÿ:™:Ñ%ˆÚä# NÔ3Ø×,Ñ,¨UÓ3ð 8ñ ð Jð JóJð ð  7°w°i¸qÐA�
Ø&=¸nÐ=MÈQÐ$OÐ!Ø×%Ñ%×/Ñ/Ø�e˜3Ô0°ÀÐFWÓXÐYÐYZÐ[õñ "Ø .Ð/Ð/?À¸yÈÐJ‘
ÙØ!ð &ñ ð
 $% W I¨[Ð!9‘Jà)¨UÒ2Ð2Ð2Ø$+ 9¨MÐ!:‘Jð  Ø×0Ñ0°ÀÀcÓJñKàñð  ð
  ¤5§:¡:Ñ-�á+2œEŸKšK¸�	Ø,¬\¸)Ñ-DÐ,EÀQÐG�Ø(@ÄÈiÑAXÐ@YÐY[Ð\`×\qÑ\qÐr{Ó\|Ð[}Ð}~Ð&Ð#Ø '˜y�
Ù!Ø)¨UÒ2Ð2Ð2Ø$+ 9¨C°Ð/?Ð!@�JØ 3Ð4°D¸¸¸UÀ3À%ÀuÈ_ÐL]Ð]_Ð`jÐ_kÐklÐm�
à×!Ñ!×+Ñ+Ø�%�sÔ,¨^¸SÀ*ÐXaÔbÐcÐcdÐeôð ‰FàˆFÜ# NÔ3Ø")¨&¨Ð 2�Ø×%Ñ%×/Ñ/Ø�h˜cÔ"3°NÀFÈMÓ"ZÐ![Ð[\Ð]õñ  Ø%¨Ò.Ð.Ð.Ø")¨&¨Ð 2�Ø×%Ñ%×/Ñ/Ø�h˜c & ¨¨]¨O¸1Ð=ôô # >°6Ó:ˆØ<Bˆ×Ñ×*Ñ*¨=Ñ9ØˆrX   c                 ó(  ‡— | j                  |«      }| j                  j                  ‰«      }t        j                  j                  ‰«      }|j                  r$|t        j                  k7  rt        j                  }n|}t        j                  j                  |«      }t        j                  j                  |«      }t        «       }	| j                  | j                  k\  r.|	j                  |› dt!        |«      › dt"        |   › d|› d�«       nØ||k7  r°t"        |   j%                  dd«      › d|› �}
|t        j&                  k(  r&|› d| j                  t        j&                  «      › d�}n?||cxk(  rd	k(  rn nd
t"        |   › d|› d�}n d
t"        |   › d|› dt"        |   › d|› d|› d�}|	j                  d|
› d|› d�«       |
}|	j)                  | j+                  ||||«      «       | j,                  j)                  |	j/                  ˆfd„«      «       y )Nrã  z] = static_cast<r�  r×   z::rÎ   z.template cast<bool,rƒ   r6   zat::vec::convert<r—   ru  rÖ   r»   r¼   c                 ó   •— t        ‰| «      S r  rÈ  rÉ  s    €rY   rÊ  z.CppVecKernel.store_reduction.<locals>.<lambda>#  s   ø€ ¼ÀTÈ1Ó8M€ rX   )r4  ri  rí  r5   ræ  rç  Úis_floating_pointr…   rl   rp   rG  rm  r>   r[  r“  rÃ   rF   rI   Úreplacerm   r  rÅ  r–  rÌ  )r  rÈ   r“   rð   r¨   Ú	out_dtyper‰   Úout_num_vectorsÚsrc_num_vectorsr´   Úconverted_valueÚconverts    `          rY   r#  zCppVecKernel.store_reductionþ  s  ø€ Ø×$Ñ$ UÓ+ˆØ�i‰i×Ñ˜tÓ$ˆÜ—G‘G×%Ñ% dÓ+ˆ	Ø×&Ò&¨9¼¿¹Ò+DÜ—K‘K‰EàˆEÜŸ(™(×3Ñ3°IÓ>ˆÜŸ(™(×3Ñ3°EÓ:ˆÜÓˆØ�?‰?˜d×2Ñ2Ò2à�N‰NØ�%�qœ UÓ+Ð,Ð,<¼\È)Ñ=TÐ<UÐUWÐX]ÐW^Ð^`Ðaõð
 ˜EÒ!ä# IÑ.×6Ñ6°t¸SÓAÐBÀ!ÀEÀ7ÐKð  ð ¤§
¡
Ò*Ø!& Ð';¸D×<QÑ<QÔRW×R\ÑR\Ó<]Ð;^Ð^aÐb‘Gà&¨/Ô>¸QÕ>à/´¸YÑ0GÐ/HÈÈ5È'ÐQRÐSñ  ð
 0´¸YÑ0GÐ/HÈØ.Ð/¨q´¸eÑ1DÐ0EÀQÀÐFWÐWYÐZ_ÐY`Ð`aðcð  ð —‘  Ð&7°s¸7¸)À1ÐEÔFØ'�Ø�K‰K˜×,Ñ,¨U°C¸À	ÓJÔKØ×Ñ×$Ñ$ T§X¡XÓ.MÓ%NÕOrX   Ú
scalar_varc                 ó  — |j                   rJ ‚|j                  t        j                  k(  rE| j                  j                  | j                  | j                  «       › d|j                  › d�«      }n]|j                  €J ‚| j                  j                  | j                  | j                  |j                  «      › d|j                  › d�«      }t        |t        «      sJ ‚|j                  |_        |j                  |_        d|_         |S )NrL  r—   ro  T)r§   r‰   r…   rm   r}  r~  r  rÇ  rÈ   rM  r£   rH   Údependent_itervars)r  ró  r®  s      rY   r�  zCppVecKernel.broadcast%  së   € Ø×$Ò$Ð$Ð$Ø×ÑœuŸz™zÒ)Ø—h‘h×'Ñ'Ø—‘ ×!4Ñ!4Ó!6Ð 7°w¸z¿¹Ð>OÈqÐQó‰Gð ×#Ñ#Ð/Ð/Ð/Ø—h‘h×'Ñ'Ø—‘Ø×%Ñ% j×&6Ñ&6Ó7Ð8¸¸*¿/¹/Ð9JÈ!ÐLóˆGô ˜'¤>Ô2Ð2Ð2Ø"×(Ñ(ˆŒØ%/×%BÑ%BˆÔ"ØˆŒØˆrX   r`  c           	      ó"  — |j                   rJ ‚|j                  €J ‚| j                  j                  | j                  | j                  |j                  «      › d|› d|› d�«      }t        |t        «      sJ ‚|j                  |_        d|_         |S )Nz	::arange(r�   r—   T)r§   r‰   r}  r~  r  rM  r£   rH   )r  r“   r`  r‡  s       rY   r]  zCppVecKernel.arange7  s‰   € Ø—<’<ÐÐØ�{‰{Ð&Ð&Ð&Ø—‘×"Ñ"Ø�L‰LØ×!Ñ! %§+¡+Ó.Ð/¨y¸¸¸rÀ&ÀÈÐKó
ˆô ˜&¤.Ô1Ð1Ð1Ø—{‘{ˆŒØˆŒØˆrX   c                 ó^  — t         |   }| j                  |«      }t        |«      rd|› d�S |dv r¢|t        j                  k(  rt        j
                  n|}t        |   }| j                  ||«      }|dk(  r+t        |«      s|t        j                  k(  rd|› d�nd|› d�}n*t        |«      s|t        j                  k(  rd|› d�nd|› d	�}|› d
|› d�S |dk(  r| j                  «       › d�S t        ||«      }	|› d
|	› d�}
|t        j                  k(  r|dv sJ ‚| j                  «       › d|	› d�S |
S )Nr‚   rƒ   rz   rb   r}   r|   r   r{   r~   ro  r—   rh   z	::from(0))r`   ra   re   rL  )r=   rM  r+   r…   rm   rp   rI   rÕ  r   rÇ  rŽ   )r  rˆ   r‰   r‘   Úvec_typeÚcompute_dtyperŠ   rÓ   r/  Úscalar_initÚvec_inits              rY   rÖ  zCppVecKernel.reduction_init_vecC  sm  € Ü0°Ñ7ˆØ×%Ñ% kÓ2ˆä Ô/Ø˜h˜Z sÐ+Ð+àÐ1Ñ1à+0´E·J±JÒ+>œEŸKšKÀKˆMÜ! -Ñ0ˆFØ×2Ñ2°>À5ÓIˆHØ Ò)ô & eÔ,°¼¿¹Ò0Cð +¨6¨(°-Ñ@à/°¨x°xÐ@ñ ô & eÔ,°¼¿¹Ò0Cð ,¨F¨8°=ÑAà/°¨x°xÐ@ð ð
 �Z˜q   QÐ'Ð'à˜UÒ"Ø×)Ñ)Ó+Ð,¨IÐ6Ð6ä$ ^°UÓ;ˆØ�Z˜q  ¨QÐ/ˆØ”E—J‘JÒØ!Ð%:Ñ:Ð:Ð:Ø×)Ñ)Ó+Ð,¨G°K°=ÀÐBÐBØˆrX   c                 óä  — t         |   }| j                  |«      }t        |«      rd|› d�S |dv r–| j                  t        j
                  «      }|t        j                  k(  r@| j                  t        j                  «      }dt        t        j                     › d|› d|› d�S | j                  |«      }dt        |   › d|› d|› d�S |t        j                  k(  r|dv sJ ‚| j                  «       › S |S )Nr‚   r�   rz   zIndexValueVec<r�   )r`   ra   rh   re   )
r=   rM  r+   rm  r…   rš  rm   rp   rI   rÇ  )r  rˆ   r‰   r‘   rø  rÎ  rÍ  s          rY   rÕ  z#CppVecKernel.reduction_acc_type_vecg  sü   € Ü0°Ñ7ˆØ×%Ñ% kÓ2ˆÜ Ô/Ø˜h˜Z qÐ)Ð)ØÐ1Ñ1Ø×)Ñ)¬%¯+©+Ó6ˆEØœŸ
™
Ò"ð ×-Ñ-¬e¯k©kÓ:�Ø'¬´U·[±[Ñ(AÐ'BÀ"ÀUÀGÈ2ÈeÈWÐTUÐVÐVØ×)Ñ)¨+Ó6ˆEØ#¤L°Ñ$=Ð#>¸bÀÀÀrÈ%ÈÐPQÐRÐRØ”E—J‘JÒØ!Ð%AÑAÐAÐAØ×)Ñ)Ó+Ð,Ð-ØˆrX   rÓ  r«   c                 ó²  — |t         j                  k(  }|dk(  r=| j                  rd|› d|› dt        | j                  «      › d�S |r|› d|› �S d|› d|› d�S |dk(  r=| j                  rd|› d|› dt        | j                  «      › d�S |r|› d	|› �S d
|› d|› d�S |dk(  rq|r4| j                  rd|› dt        | j                  «      › d|› d�S d|› d|› d�S | j                  rd|› d|› dt        | j                  «      › d�S |rdnd}	|› d|	› d|› �S |dk(  r2| j                  rd|› d|› dt        | j                  «      › d�S |› d|› �S |dk(  r2| j                  rd|› d|› dt        | j                  «      › d�S |› d|› �S |dk(  rp|r:| j                  r"d|› d|› dt        | j                  «      › d|› d�	S d|› d|› d|› d�S | j                  rd|› d|› dt        | j                  «      › d�S d|› d|› d�S |dk(  rgt	        |t
        «      r|\  }
}}nt        ||«      \  }
}}| j                  r%d|› d|
› d|› d|› dt        | j                  «      › d�S d|› d|
› d|› d|› d�	S |dv �rd|€J ‚t        |   }|}|t         j                  k(  r\t        t         j                     }t         j                  }t	        |t        «      r%|j                  rt        | j                  |f«      \  }| j                  |«      }| j                  t         j                  «      }d}d}|�^t	        |t        «      r|j                  rd|› �}n<d }d|› �}n4|€J ‚dt        |«      j!                  «       › �}d| j#                  |«      › �}| j                  r.|› d!|› d|› d|› |› d"|› d|› |› dt        | j                  «      › d�S |› d!|› d|› d|› |› d"|› d|› |› d�S |d#k(  rˆt	        |t        «      rF|j$                  t         j                  k(  sJ ‚t        t&        j(                  j                  |f«      \  }| j                  rd$|› d|› dt        | j                  «      › d�S |› d|› �S t*        ‚)%z>Emit the C++ expression for combining vector reduction values.ra   zmax_masked_reduce(r�   r—   r[  r2  r`   zmin_masked_reduce(rU  r+  re   r•   r–   zsum_masked_reduce(r˜   r\   r™   rf   zprod_masked_reduce(rš   rg   zxor_sum_masked_reduce(r›   ri   rž   rj   rŸ   z}, r    rz   rT   z, falsez_combine_vec<r�  rh   zany_masked_reduce()r…   rm   rl  rF   r£   r¤   r¥   rI   rp   rH   r§   rP   r  rm  rš  rq   rÖ  rÐ  r‰   r5   rG  rŽ  )r  rˆ   r¨   r©   rª   r“   rÓ  r«   r¬   r­   r®   r¯   r°   rŠ   rù  rÍ  rÎ  Út_extraÚ	arg_extras                      rY   r×  z"CppVecKernel.reduction_combine_vecz  sq  € ð œuŸz™zÑ)ˆØ˜UÒ"Ø�~Š~Ø+¨C¨5°°:°,¸bÄÈTÏ^É^ÓA\Ð@]Ð]^Ð_Ð_ñ ð �e˜3˜z˜lÐ+ðð -¨S¨E°°J°<¸qÐAðð
 ˜uÒ$Ø�~Š~Ø+¨C¨5°°:°,¸bÄÈTÏ^É^ÓA\Ð@]Ð]^Ð_Ð_ñ ð �e˜3˜z˜lÐ+ðð -¨S¨E°°J°<¸qÐAðð
 ˜uÒ$ÙØ—>’>Ø1°*°¸RÄÈDÏNÉNÓ@[Ð?\Ð\_Ð`jÐ_kÐklÐmÐmà1°*°¸SÀÀÈAÐNÐNà—>’>Ø/°¨u°B°z°lÀ"Ä[ÐQU×Q_ÑQ_ÓE`ÐDaÐabÐcÐcá)0¡#°c�KØ!˜U ! K =°°*°Ð>Ð>Ø˜vÒ%Ø�~Š~Ø,¨S¨E°°J°<¸rÄ+ÈdÏnÉnÓB]ÐA^Ð^_Ð`Ð`à˜˜c * Ð.Ð.Ø˜yÒ(Ø�~Š~Ø/°¨u°B°z°lÀ"Ä[ÐQU×Q_ÑQ_ÓE`ÐDaÐabÐcÐcà˜˜c * Ð.Ð.ØÐ/Ò/ÙØ—>’>Ø-¨c¨U°"°Z°LÀÄ;ÈtÏ~É~ÓC^ÐB_Ð_bÐcmÐbnÐnoÐpÐpà-¨c¨U°"°Z°LÀÀJÀ<ÈqÐQÐQà—>’>Ø-¨c¨U°"°Z°LÀÄ;ÈtÏ~É~ÓC^ÐB_Ð_`ÐaÐaà-¨c¨U°"°Z°LÀÐBÐBØÐ0Ò0Ü˜*¤eÔ,à#-Ñ ��b™&ô $5°^ÀZÓ#PÑ ��b˜&Ø�~Š~Ø)¨#¨¨d°4°&¸¸2¸$¸bÀÀÈÌ[ÐY]×YgÑYgÓMhÐLiÐijÐkÐkà)¨#¨¨d°4°&¸¸2¸$¸bÀÀÈÐLÐLØÐ3Ò3ØÐ(Ð(Ð(Ü! )Ñ,ˆFØ%ˆMØœEŸJ™JÒ&ä%¤e§k¡kÑ2�Ü %§¡�ä˜j¬.Ô9¸j×>OÒ>OÜ$8¸¿¹È
À}Ó$U‘M�ZØ×)Ñ)¨-Ó8ˆEØ×)Ñ)¬%¯+©+Ó6ˆEØˆGØˆIØÐ Ü˜e¤^Ô4Ø—|’|Ø&(¨¨ L™	à"+˜Ø&(¨¨ L™	à/Ð;Ð;Ð;Ø "¤3Ð';Ó#<×#BÑ#BÓ#DÐ"EÐF�GØ"$ T×%AÑ%AÀ%Ó%HÐ$IÐ J�IØ�~Š~à%Ð& m°F°8¸2¸e¸WÀBÀuÀgÈgÈYð WØ�u˜B˜z˜l¨9¨+°R¼ÀDÇNÁNÓ8SÐ7TÐTUðWðð
 )Ð)¨°v°h¸bÀÀÀrÈ%ÈÐQXÐPYÐY[Ð\_Ð[`Ð`bÐcmÐbnÐoxÐnyÐyzÐ{Ð{Ø˜uÒ$Ü˜*¤nÔ5Ø!×'Ñ'¬5¯:©:Ò5Ð5Ð5Ü 4´Q·X±X×5EÑ5EÈ
À}Ó U‘�Ø�~Š~Ø+¨C¨5°°:°,¸bÄÈTÏ^É^ÓA\Ð@]Ð]^Ð_Ð_à˜˜c * Ð.Ð.ä%Ð%rX   c           	      ó€  •— t        |t        «      sJ ‚|j                  €J ‚|j                  s4t        |t        «      r|j                  rd|› d�}t        ‰	| �  ||||«      S |}|}|r!| j                  |j                  «      › d|› d�}|r!| j                  |j                  «      › d|› d�}|r|rd|› d|› d|› d|› d�	}|› d|› d|› �}n#|r|› d|› �}|› d|› �}n|sJ ‚|› d|› �}|› d|› �}| j                  |j                  «      › d|› d�}|r6|j                  s!| j                  |j                  «      › d|› d�}d|› d|› d�}| j                  rS| j                  |j                  «      › d	| j                  |j                  «      › d
|› dt        | j                  «      › d�}d|› d�}| j                  › d|› d|› d�S )Nro  z).all_masked()r—   rÔ  z) & (r½   rL  z) | ~(r'  z::from(1), (r¡   z, "index out of bounds: z"))r£   rH   r‰   r§   r  rÝ  rM  rÇ  rl  rF   rc  )
r  r¨   rÖ  r×  r>  Úlower_scalarÚupper_scalarrñ  Ú
cond_printr  s
            €rY   rÝ  zCppVecKernel.indirect_assertð  s$  ø€ Ü˜#œ~Ô.Ð.Ð.Ø�y‰yÐ$Ð$Ð$Ø�zŠzÜ˜$¤Ô/°D·K²KØ˜4˜& Ð/�Ü‘7Ñ*¨3°°u¸dÓCÐCØˆØˆÙØ×)Ñ)¨#¯)©)Ó4Ð5°Q°u°g¸QÐ?ˆEÙØ×)Ñ)¨#¯)©)Ó4Ð5°Q°u°g¸QÐ?ˆEÙ‘UØ�u�g˜T #  e¨C¨5°°E°7¸!Ð<ˆDØ(˜>¨¨c¨U°#°l°^ÐD‰JÙØ�W˜D  Ð&ˆDØ(˜>¨¨c¨UÐ3‰JáˆL�5Ø�U˜#˜e˜WÐ%ˆDØ˜5  L >Ð2ˆJØ×%Ñ% c§i¡iÓ0Ð1°°4°&¸Ð:ˆÙØ—;’;Ø×-Ñ-¨c¯i©iÓ8Ð9¸ÀÀÀaÐH�à�t�f˜F 4 &¨Ð*ˆDØ�>Š>à×&Ñ& s§y¡yÓ1Ð2°&¸×9LÑ9LÈSÏYÉYÓ9WÐ8Xð YØ�V˜3œ{¨4¯>©>Ó:Ð;¸1ð>ð ð �4�&˜Ð'ˆØ×&Ñ&Ð' q¨¨Ð.FÀzÀlÐRTÐUÐUrX   c                 óÒ  •— t        |t        «      sJ ‚|j                  st        ‰
| �  ||||«      S t
        |   }| j                  |«      }t
        |   }| j                  |«      }d|› d�}	|t        j                  k7  r2|t        j                  k(  r| j                  |«      › d|› d|› d|› d�}	|	S |t        j                  k(  r |t        j                  k7  r|› d|› d|› d�}	|	S ||k7  r~d}	|rG|t        j                  t        j                  fv r%|t        j                  t        j                  fv rd	}	nd
}	||cxk(  rdk(  rn n|	d|› d|› d�z   }	|	S |	d|› d|› d|› d|› d|› d�z   }	|	S )Nro  r—   z::from<ru  r�  z.to<rƒ   rT   zat::vec::round_convertzat::vec::convertr6   r  )r£   rH   r§   r  r|  rI   rm  r…   rm   rÇ  rp   rl   rE  rD  )r  rr  r‰   r«   rŠ  Úsrc_cpp_typerð  Údst_cpp_typeÚdst_num_vectorsr†  r  s             €rY   r|  zCppVecKernel.get_to_dtype_expr  s«  ø€ Ü˜#œ~Ô.Ð.Ð.Ø�zŠzÜ‘7Ñ,¨S°%¸ÀHÓMÐMÜ# IÑ.ˆØ×/Ñ/°	Ó:ˆÜ# EÑ*ˆØ×/Ñ/°Ó6ˆØ�3�%�qˆzˆØœŸ
™
Ò" u´·
±
Ò':Ø×)Ñ)¨)Ó4Ð5°W¸\¸NÈ!ÈOÐK\Ð\^Ð_bÐ^cÐcdÐeˆDð( ˆð' œ%Ÿ*™*Ò$¨´%·*±*Ò)<Ø�U˜$˜|˜n¨A¨oÐ->¸cÐBˆDð$ ˆð# ˜%ÒØˆDáØ¤%§+¡+¬u¯|©|Ð!<Ñ<ØœeŸj™j¬%¯+©+Ð6Ñ6à/‘à)�Ø /Ô6°QÕ6Ø  , ¨r°#°°aÐ8Ñ8�ð ˆð Ø˜,˜ q¨Ð(9¸¸<¸.ÈÈ/ÐIZÐZ\Ð]`Ð\aÐabÐcñdð ð ˆrX   r  )NNFr‚  )(r÷   rø   rù   r•  rR  r  r:  r’  r…  r\  r…   r‰   ro   rm  rp   r  rq   rM  rÇ  rH   r7  r£  r>   rm   r^  rø  rÅ  rï  rÐ  r!  r#  r�  r]  rÖ  rÕ  r†   r×  rÝ  r|  rM  rN  s   @rY   rœ  rœ   
  s©  ø„ Ø€Ið õCð"4¨5¯:©:ð 4ÀÇÁó 4ð e§k¡kð °có ðR¨%¯+©+ð R¸%ó Rð
P 5§;¡;ð P°3ó Pð 38·+±+ñ H E§K¡Kð HÀ#ó HðN >ð N¸%¿+¹+ð NÈ#ó Nð ,0ñ#àð#ð �z‰zð#ð �{‰{ð	#ð
 " DÑ(ó#ðT )-Ø37Ø ñMà�4‰ZðMð �z‰zðMð �{‰{ð	Mð
  Ñ%ðMð ˜>Ñ)¨DÑ0ðMð ðMð 
˜$Ñ	óMð^˜ð  U§Z¡Zõ ð4 !ñ*à�^Ñ#ð*ð ð*ð �z‰zð	*ð
 �{‰{ð*ð ó*óX"<ðH¨U¯Z©Zð ¸E¿J¹Jó òAòF%PðN Nð °~ó ð$
˜Nð 
°E·L±Lð 
À^ó 
ò"òHð0 Ø48Ø,0Ø(-¯©ñt&ð �z‰z˜NÑ*¨TÑ1ðt&ð # T™kðt&ð —;‘; Ñ%ót&õl#V÷Jñ rX   rœ  c                   óö   ‡ — e Zd ZdZeZ	 	 dˆ fd„	Zd„ Zd„ Z	 dd„Z	de
dej                  fˆ fd„Zdˆ fd	„	Zd
„ Zˆ fd„Zdej                  dej                  fd„Zdej                  dej                  fd„Zˆ xZS )r†  an  
    A vector kernel that handles the 2d tiles with the tile size defined in `tiling_factor` on
    the inner-most loop level and one of the outer loop level (`outer_tiling_idx`). When the data
    tile is accessed in a contiguous way from the outer loop axis, a transposition is applied on the
    tile to make the access contiguous from the inner-most loop axis. Then, the same vectorization
    logic from its parent `CppVecKernel` is leveraged for load/store/compute. The transposed tile load
    and store are generated into kernel.preloads and kernel.poststores buffers.

    The loop structure looks like below:
    for ...
      for i_outer ...
        for ...
          for inner_most ...
            // generated by CppTile2DKernel
            float tmp0[16*16]; at::vec::transpose_mxn<...>(tmp0, in_ptr0 + ..., ...); // into kernel.preloads
            float tmp1[16*16]; // into kernel.preloads
            for i_inner ... { // the kernel inner loop
              vectorized loads/compute/stores (e.g., load tmp0, store tmp1) // into kernel.loads/compute/stores
            }
            at::vec::transpose_mxn(out_ptr0 + ..., tmp1, ...) // into kernel.poststores
          for inner_most ... (tail)
            // generated by CppVecKernel
            ...
      for i_outer ... (tail)
        for ...
          for ...
            // generated by CppKernel
            ...
    c                 ó–   •— t         ‰| �  ||||d   |«       || _        || _        || _        |r|n|| _        |r|n|| _        d| _        y )Nr6   T)r  r  Útiling_indicesÚinner_tail_sizeÚouter_tail_sizeÚinner_num_elemsÚouter_num_elemsÚinner_is_tiling_idx)r  ri  r¥  r  r
  r  r  r  s          €rY   r  zCppTile2DKernel.__init__W  s`   ø€ ô 	‰ÑØØØØ˜1ÑØô	
ð -ˆÔØ.ˆÔØ.ˆÔÙ2A™À}ˆÔÙ2A™À}ˆÔØ#'ˆÕ rX   c                 óL   — t        | j                  | j                     › d�«      S )Nr0  )r.   rZ  Ú	outer_idxr  s    rY   Úinner_itervarzCppTile2DKernel.inner_itervarn  s"   € Ü! T§]¡]°4·>±>Ñ%BÐ$CÀ6Ð"JÓKÐKrX   c                 ó˜  — | j                   | j                     }| j                   | j                     }t        |t        j
                  «      ryt        ||| j                  «      }t        ||| j                  «      }| j                  d u xr@ |dk(  xr9 |j                  |«      xr& |j                  |«       xr |j                  |«       S )NFr6   )
rZ  r  r[  r   r   rÛ  r%   r  r¾  Úhas)r  r“   Ú	outer_varÚ	inner_varÚouter_strideÚinner_strides         rY   Úneed_vec_transposez"CppTile2DKernel.need_vec_transposeq  sÀ   € Ø—M‘M $§.¡.Ñ1ˆ	Ø—M‘M $§/¡/Ñ2ˆ	ô ˜u¤d§h¡hÔ/ØÜ*¨5°)¸T×=OÑ=OÓPˆÜ*¨5°)¸T×=OÑ=OÓPˆà�O‰O˜tÐ#ò 0Ø Ñ!ò0à—	‘	˜)Ó$ò0ð !×$Ñ$ YÓ/Ð/ò0ð !×$Ñ$ YÓ/Ð/ð	
rX   c                 ón  — t         j                  j                  |«      }| j                  }|› dt	        |«      › �}d}	t	        t        || j                  | j                     | j                  «      «      › }
t	        | j                  «      › }|r|	|}	}||
}}
d}| j                  |z  r| j                  | j                  }}n| j                  | j                  }}|r|dk(  rdnd}t        |t        j                  «      r|j                  r&t        |t        j                  «      rA|j                  s5dt         |   › d|› d	|› d
|
› d
|	› d
|› d
t	        |«      › d
t	        |«      › d�}n4dt         |   › dt	        |«      › dt	        |«      › d|› d	|› d
|
› d
|	› d
|› d�}|r| j"                  j%                  «       }na| j"                  j'                  |«      s)| j"                  j)                  | j*                  |d¬«      }nd}| j"                  j-                  |«      }|r>t         |   }d|› d|› d�}|› d|› d|› d|› d|› d�
}| j*                  j/                  |«       |j1                  dt3        |«      «      }|r'| j4                  j/                  t7        ||«      «       |S | j*                  j/                  |«       |S )Nrê   Ú__place_holder__Trë  ÚtrueÚfalseztranspose_mxn<ru  r�  r�   r×   Fr  zalignas(std::max(std::size_t(z), alignof(z)))r™   rã  r]   rÙ   )r5   ræ  rç  r  rF   r%   rZ  r[  r(  r  r  r  r£   r:  r’  r™  rI   r}  r  Úcontainsr~  r£  ÚgetrÃ   rí  rq   r¤  r<   )r  rÈ   r¨   r“   Úis_storeÚ
store_moder‰   Úfactorrr  rt  Úld_srcÚld_dstÚneed_defineÚMÚNrë  Úload_or_storeÚtile_varÚ	cpp_dtypeÚalignasÚdefine_lines                        rY   Úgen_transposed_tile_load_storez.CppTile2DKernel.gen_transposed_tile_load_store‚  sÆ  € ô —‘×!Ñ! $Ó'ˆØ×#Ñ#ˆØ��Sœ UÓ+Ð,Ð-ˆØ ˆÜÔ 3°E¸4¿=¹=ÈÏÉÑ;YÐ[_×[mÑ[mÓ nÓoÐpˆÜ §¡Ó/Ð0ˆÙØ˜C�ˆCØ# V�FˆFàˆØ×#Ñ# hÒ.Ø×'Ñ'¨×)=Ñ)=ˆq‰Að ×$Ñ$Ø×$Ñ$ð ˆAñ !)¨j¸LÒ.H‘VÐPWˆ
Ü�qœ%Ÿ*™*Ô%¨a¯kªkÜ�qœ%Ÿ*™*Ô%¨a¯kªkð !¤¨eÑ!4Ð 5°Q°z°lð CØ�5˜˜6˜( " S E¨¨F¨8°2´kÀ!³nÐ5EÀRÌÐTUËÐGWÐWYð[ñ ð !¤¨eÑ!4Ð 5°Q´{À1³~Ð6FÀaÌÐTUËÐGWÐWXÐYcÐXdð eØ�5˜˜6˜( " S E¨¨F¨8°2ð7ð ñ Ø—x‘x—‘Ó(‰HØ—‘×"Ñ" =Ô1Ø—x‘x×(Ñ(¨¯©¸ÈUÐ(ÓS‰HàˆKØ—x‘x—|‘| MÓ2ˆHáÜ$ UÑ+ˆIð 6°f°X¸[ÈÈÐSVÐWˆGØ$˜I Q y k°°8°*¸A¸f¸XÀQÀvÀhÈbÐQˆKØ�M‰M×#Ñ# KÔ0à%×-Ñ-Ð.@Ä#ÀhÃ-ÓPˆÙØ�O‰O×%Ñ%¤l°4¸Ó&GÔHð ˆð �M‰M×#Ñ# MÔ2àˆrX   rÈ   r“   c                 óZ  •— | j                   j                  |«      }| j                  |«      }| j                  «       }| j	                  |«      r½| j                  |||d¬«      }|› dt        || j                  z  «      › �}t        j                  j                  |«      }| j                  |d|«      }| j                  j                  | j                  ||¬«      }	|	j                  d| ||fi «       t!        |	t"        «      sJ ‚d|	_        |	S | j'                  |«      }
t(        ‰| �U  ||
«      S )NF)r   rê   r   re  rø  T)ri  rå  r4  r  r  r-  rF   r(  r5   ræ  rç  r£  r}  r~  r¹  r€  r£   rH   r§   r‡  r  rø  )r  rÈ   r“   r¨   rF  r)  r¢  r‰   rÉ   r‡  rÄ  r  s              €rY   rø  zCppTile2DKernel.load¾  s  ø€ Ø�i‰i�o‰o˜dÓ#ˆØ×$Ñ$ UÓ+ˆà×"Ñ"Ó$ˆØ×"Ñ" 5Ô)Ø×:Ñ:Ø�c˜5¨5ð ;ó ˆHð "˜
 #¤k°%¸$¿.¹.Ñ2HÓ&IÐ%JÐKˆGÜ—G‘G×%Ñ% dÓ+ˆEØ×*Ñ*¨7°A°uÓ=ˆDØ—X‘X×&Ñ& t§z¡z°4¸uÐ&ÓEˆFØ×!Ñ! &¨4°°uÐ*=¸rÔBÜ˜f¤nÔ5Ð5Ð5Ø ˆFŒMØˆMà×/Ñ/°Ó6ˆIÜ‘7‘<  iÓ0Ð0rX   c                 ó:  •— d|v sJ ‚t        |t        «      sJ |«       ‚|j                  s| j                  |«      }| j                  j                  |«      }| j                  «       }| j                  |«      }| j                  |«      rð| j                  |||d|¬«      }|› dt        || j                  z  «      › �}| j                  sdt        j                  j                  |«      t         t"        j$                  t"        j&                  t"        j(                  t"        j*                  gz   v r|› d|› dt        | j                  «      › d�}	n|› d|› d�}	| j,                  j/                  t1        ||	«      «       y | j3                  |«      }
t4        ‰| �m  ||
||«       y )Nré  T)r   r!  rê   rÃ  r�   r×   )r£   rH   r§   r�  ri  rí  r  r4  r  r-  rF   r(  rl  r5   ræ  rç  r„   r…   rD  rE  rF  rG  r³  rÃ   r<   r‡  r  rï  )r  rÈ   r“   rð   rî  r¨   rF  r)  ÚstorebufrÉ   rÄ  r  s              €rY   rï  zCppTile2DKernel.storeÔ  sv  ø€ Ø˜‰}Ðˆ}Ü˜%¤Ô0Ð7°%Ó7Ð0Ø�|Š|à—N‘N 5Ó)ˆEà�i‰i×Ñ˜tÓ$ˆà×"Ñ"Ó$ˆØ×$Ñ$ UÓ+ˆØ×"Ñ" 5Ô)Ø×:Ñ:Ø�c˜5¨4¸Dð ;ó ˆHð #˜ 3¤{°5¸4¿>¹>Ñ3IÓ'JÐ&KÐLˆHØ�~Š~¤§¡×!2Ñ!2°4Ó!8¼MÜ—‘Ü—
‘
Ü×#Ñ#Ü×!Ñ!ð	Mñ =ñ "ð  ˜ ¨ z°´KÀÇÁÓ4OÐ3PÐPRÐS‘à˜ ¨ z°Ð4�Ø�K‰K×!Ñ!¤,¨t°TÓ":Õ;à×/Ñ/°Ó6ˆIÜ‰G‰M˜$ 	¨5°$Õ7rX   c                 ó  — | j                  «       }| j                  r2|j                  d|› d|› dt        | j                  «      › d|› d�	«       y |j                  d|› d|› dt        | j
                  «      › d|› d�	«       y )Nrµ  r¶  r½   r¾   r¿   )r  r  rÃ   rF   r  r  )r  r´   rF  s      rY   r·  z#CppTile2DKernel.codegen_inner_loopsó  s�   € Ø×"Ñ"Ó$ˆØ×#Ò#Ø�N‰NØ˜U˜G 6¨%¨°´KÀ×@TÑ@TÓ4UÐ3VÐVXÐY^ÐX_Ð_bÐcõð �N‰NØ˜U˜G 6¨%¨°´KÀ×@TÑ@TÓ4UÐ3VÐVXÐY^ÐX_Ð_bÐcõrX   c                 óœ  •— t         ‰| �  ||«      }| j                  d   | j                  k  r| j                  nt	        | j                  «      \  | _        | _        | j                  | j                  d   k(  r+| j                  | _        | j                  | _
        d| _        |S | j                  | _        | j                  | _
        d| _        |S )Nr6   r   FT)r  r(  r
  r“  Úreversedr  r[  r  rl  r  r(  r  r  r  )r  ÚgroupÚreduction_groupr©  r  s       €rY   r(  zCppTile2DKernel.set_rangesþ  sÀ   ø€ Ü‰wÑ! %¨Ó9ˆð ×"Ñ" 1Ñ%¨×(<Ñ(<Ò<ð ×Òä˜$×-Ñ-Ó.ñ 	(ˆŒ˜œð
 �?‰?˜d×1Ñ1°!Ñ4Ò4Ø!×1Ñ1ˆDŒNØ!×1Ñ1ˆDŒNØ',ˆDÔ$ð
 ˆð "×1Ñ1ˆDŒNØ!×1Ñ1ˆDŒNØ'+ˆDÔ$ØˆrX   r¹   c                 óZ   — | j                  || j                  | j                  «       ¬«      S )Nr´  )rÅ  r  r  rÇ  s     rY   r‡  z"CppTile2DKernel.transform_indexing  s0   € Ø×+Ñ+ØØŸ™Ø×%Ñ%Ó'ð ,ó 
ð 	
rX   c                 ó$   — | j                  |«      S r  )r‡  rÇ  s     rY   rÐ  z'CppTile2DKernel._adjust_argreduce_index  s   € Ø×&Ñ& uÓ-Ð-rX   rƒ  r  )r÷   rø   rù   rú   r„  rR  r  r  r  r-  rq   r:  r’  rø  rï  r·  r(  r‡  rÐ  rM  rN  s   @rY   r†  r†  6  sŽ   ø„ ñð< #€Ið Øõ(ò.Lò
ð$ 6:ó:ðx1˜ð 1 U§Z¡Zõ 1õ,8ò>	ôð$
¨¯
©
ð 
°u·z±zó 
ð.¨U¯Z©Zð .¸E¿J¹J÷ .rX   r†  Ú_bodyc                 ój  — | j                   gt        | j                  j                  «       «      z   }d}d}|D ]ö  }|j                  j
                  D ]Û  }|j                  dk(  s|j                  dv rŒ!|j                  dvrd}t        |d«      r�|j                  r‘t        j                  |j                  v sJ ‚|j                  t        j                     }|j                  r|j                  t        vrd}Œ¥|�&||j                  k7  sŒ·t        j                  d«       ŒÍ|j                  }ŒÚd}ŒÝ Œø ||fS )	zÞ
    Returns the low precision data type (torch.float16/torch.bfloat16) contained in the nodes
    and if all the nodes can codegen with this data type without converting to float.
    Otherwise returns None and True.
    NFÚplaceholder)Ú	get_indexr6  r9  )rø  rï  r‘  r˜  rí  TrX  z.bf16 and fp16 are mixed in the scheduler node.)Ú
root_blockr  Ú	subblocksrR  ræ  ÚnodesÚopÚtargetr¦   rX  rB   rW  r‰   r„   ÚwarningsÚwarn)r8  Ú
sub_blocksÚ_lowp_fp_typeÚ	_use_fp32Ú	sub_blockr  rT  s          rY   Úget_loop_body_lowp_fprG    s%  € ð ×"Ñ"Ð#¤d¨5¯?©?×+AÑ+AÓ+CÓ&DÑD€Jà(,€MØ€IØò !ˆ	Ø—_‘_×*Ñ*ò 	!ˆEØ�x‰x˜=Ò(¨E¯L©Lð =ñ -ð
 ð �|‰|ð $ñ ð !�	ä�u˜fÔ%¨%¯*ª*Ü*×.Ñ.°%·*±*Ñ<Ð<Ð<Ø/4¯z©zÔ:M×:QÑ:QÑ/R�Ø—}’}¨¯©¼]Ñ(JØ $‘IØ"Ð.Ø$¨¯©Ó5Ü Ÿ™Ð&VÕWà$+§M¡M‘Mà ‘	ñ;	!ð!ð@ ˜)Ð#Ð#rX   c                   ó¨   — e Zd ZU dZdZee   ed<    ej                  e
¬«      Ze
eef   ed<    ej                  e
¬«      Ze
eef   ed<   dZeed<   y	)
ÚKernelOpStatsz0Op statistics collected during tiling selection.g¸…ëQ¸¾?ÚOVERHEAD_RATIO_THRESHOLD)Údefault_factoryÚ
op_counterÚnon_contig_indexing_op_counterr   Úmask_op_countN)r÷   rø   rù   rú   rJ  r   rp   rû   ÚdataclassesÚfieldr“  rL  rq   ro   rM  rN  rW   rX   rY   rI  rI  H  sf   … á:ð
 15Ð˜h u™oÓ4à!2 ×!2Ñ!2À4Ô!H€J��S˜#�X‘ÓHØ5F°[×5FÑ5FØô6Ð" D¨¨c¨¡Nó ð €M�3ÔrX   rI  c                   ó:   — e Zd ZdZdeee   ee   ef   fd„Zd„ Z	y)ÚTilingSelectz˜
    Implement the heuristic to select the tiling factors and tiling indices.
    In the future, we can implement advanced heuristic in a subclass.
    r¹   c           	      ó˜	  ‡!— t        |«      }t        |«      }|sJ ‚t        d„ |D «       «      rg g t        «       fS t        j
                  }t        |d   «      d   Š!‰!rt        ˆ!fd„|dd  D «       «      r‰!}t        j                  «       j                  |¬«      }| j                  |||«      }|�r	t        |d„ ¬«      \  }}	t        |«      t        |	«      z   }
t        «       }t        j                  j                   �rºd„ }d	„ }d
„ }t#        t%        |
«      «      D �cg c]  }t'        t(        j*                  |«      ‘Œ }}t%        |«      }|d | ||d  }}|D �]¸  }|j,                  gt/        |j0                  j3                  «       «      z   }|D �]~  }|j4                  j6                  D �]a  }|j8                  dv r’|j8                  dv rdnd}|j:                  j=                  ||f«      |j>                  |   j>                  d      } |||«      r= |||||«      }|j8                  dv r|€"n|dvr ||j8                  |j@                  «       tC        |j8                  tD        «      sŒ¿|j8                  dv r|xjF                  dz  c_#        |j8                  jI                  d«      rŒþ|j8                  dv r�Œ|j8                  |jJ                  vrd|jJ                  |j8                  <   �ŒA|jJ                  |j8                  xx   dz  cc<   �Œd �Œ� �Œ» tM        |jJ                  j3                  «       «      }tM        |j@                  j3                  «       «      }d}||k\  s|dkD  r||z  |jN                  k\  rg g |fS |	s:|r8t%        |«      dk(  r*tQ        ||d      g«      s||d      |dz  k  r
|dk  rg g |fS |tR        v rÑt        j                  «       j                  |¬«      }|D ]¨  }|dk  r|t%        |
«      z   }|dk  s|t%        |
«      k\  rŒ*tQ        |
«      retT        j4                  jV                  jY                  |
|   d¬«      } | |k  sŒitT        j4                  jV                  j[                  | |«       |dz  } n|
|   |k  sŒ£|dz  } n t%        |«      dk(  r|g||fS t%        |«      dk(  r||g||fS g g t        «       fS c c}w )Nc              3   ó,   K  — | ]  }|t         v–— Œ y ­wr  )rx   )r  r‰   s     rY   r  z-TilingSelect.select_tiling.<locals>.<genexpr>g  s   è ø€ ÒH°EˆuÔ/Ô/ÑHùs   ‚r   c              3   ó@   •K  — | ]  }t        |«      d    ‰k(  –— Œ y­w)r   N)rG  )r  Ú	loop_bodyÚ_lowp_fp_dtypes     €rY   r  z-TilingSelect.select_tiling.<locals>.<genexpr>k  s(   øè ø€ ò "
àô # 9Ó-¨aÑ0°NÕBñ"
ùó   ƒr6   re  c                 ó   — t        | d   «      S rH  ©rÔ   ©Úsizess    rY   rÊ  z,TilingSelect.select_tiling.<locals>.<lambda>x  s   € ´#°e¸A±h³-€ rX   ©rW  c                 óL   — ||d      }t        | ||«      }|j                  r|S d S rB  )r%   r™  )r“   rZ  r  r
  rÉ  r`  s         rY   Ú_try_get_stridez3TilingSelect.select_tiling.<locals>._try_get_stride  s4   € ð ' ~°aÑ'8Ñ9�GÜ0°¸ÀÓO�FØ%+×%5Ò%5˜6Ð?¸4Ð?rX   c                 ó2   — | |vrd|| <   y || xx   dz  cc<   y rH  rW   )Ú	node_namerM  s     rY   Ú_update_negative_op_countz=TilingSelect.select_tiling.<locals>._update_negative_op_count‰  s(   € ð !Ð(FÑFØDEÐ6°yÒAà6°yÓAÀQÑFÔArX   c                 ó–   — t        |«      dk(  xr: t        | «      dkD  xr* |d   dk\  r|d   n|d   t        | «      z   t        | «      k  S ©Nr6   r   rZ  )rZ  r
  s     rY   Ú_is_valid_indicesz5TilingSelect.select_tiling.<locals>._is_valid_indices‘  sb   € ô
 ˜NÓ+¨qÑ0ò (Ü ›M¨AÑ-ò(ð  .¨aÑ0°AÒ5ð +¨1Ò-à!/°Ñ!2´S¸³]Ñ!Bä˜h›-ñ(ð	rX   )r6  r9  rø  rï  )r6  r9  r   ©r   r6   )r+  rD  Úmasked_subblock)r3   rí  r0  r;  é#   r©  é
   re  ).rD   rC   rh   rI  r…   rp   rG  r  r   r‰  Ú	nelementsÚ_select_tiling_indicesra   r¤   r   r´  Úenable_tiling_heuristicsr3  rÔ   r/   r   r%  r<  r  r=  rR  ræ  r>  r@  r?  Úindexing_from_argsri  rM  r£   rq   rN  Ú
startswithrL  re   rJ  r)   r„   r5   rý  r*  Úcheck_lt)"r  Úfn_listÚvar_sizes_listÚloop_bodiesÚ
all_dtypesr‰   r  r
  r4  r5  r’  Ústatsr_  rb  re  r  rZ  r“  r©  Úreduction_varsr8  rC  rF  r  Úarg_idxr“   r`  Úop_numÚnon_contig_indexing_op_numÚquantity_thresholdÚfactor_lowpÚtiling_indiceÚ
call_rangerW  s"                                    @rY   Úselect_tilingzTilingSelect.select_tiling^  s  ø€ ô % WÓ-ˆÜ/°Ó<ˆ
ÙÐˆzÜÑH¸ZÔHÔHØ�rœ=›?Ð*Ð*Ü—‘ˆÜ.¨{¸1©~Ó>¸qÑAˆÙœcó "
à(¨¨˜_ô"
ô 
ð #ˆEä#×0Ñ0Ó2×<Ñ<À5Ð<ÓIˆØ×4Ñ4Ø�^ ]ó
ˆò Ü%(ØÑ$?ô&Ñ"ˆE�?ô   ›,¬¨Ó)?Ñ?ˆKä!“OˆEÜ�z‰z×2Ó2ò@òGòô" #¤3 {Ó#3Ó4öàô 3´4·;±;ÀÕBð�ð ô #& e£*�àÐ-˜oÐ.Ø˜_Ð-Ð.ð %�ð )ó -L�EØ"'×"2Ñ"2Ð!3´d¸5¿?¹?×;QÑ;QÓ;SÓ6TÑ!T�JØ%/ó +L˜	Ø%.§_¡_×%:Ñ%:ó *L˜EØ$Ÿ|™|ð 0ñ  ð (-§|¡|Ð7SÑ'Sñ %&à)*ð !(ð
 )2¯©×(IÑ(IØ%)¨>Ð$:ó)"à"'§*¡*¨WÑ"5×":Ñ":¸1Ñ"=ñ)? ñ $5°X¸~Ô#NÙ-<Ø(-¨x¸Èó.& Fð
 ,1¯<©<Ð;WÑ+Wð )/©à-3¸6Ñ-Aá(AØ,1¯L©LØ,1×,PÑ,Pô)*ô  *¨%¯,©,¼Õ<ð $)§<¡<Ð3FÑ#FØ$)×$7Ò$7¸1Ñ$<Õ$7à$)§L¡L×$;Ñ$;Ð<MÕ$NØ',§|¡|Ø'Qó(Rð (-§|¡|¸5×;KÑ;KÑ'KØIJ¨×(8Ñ(8¸¿¹Ó(Fà(-×(8Ñ(8¸¿¹Ó(FÈ!Ñ(KÕ(FòU*Lò+Lð-Lô^ ˜U×-Ñ-×4Ñ4Ó6Ó7�Ü-0Ø×8Ñ8×?Ñ?ÓAó.Ð*ð &(Ð"Ø-Ð1CÒCØ˜Q’JØ2°VÑ;Ø×5Ñ5ò6ð ˜r 5˜=Ð(ñ (ÙÜ˜NÓ+¨qÒ0Ü,à! .°Ñ"3Ñ4ðôð
 ˜n¨QÑ/Ñ0°=À1Ñ3DÒDØ šð ˜r 5˜=Ð(àœÑ%ô *×6Ñ6Ó8×BÑBÈÐBÓO�Ø%3ò �MØ$ qÒ(Ø(5¼¸KÓ8HÑ(H˜Ø$ qÒ(¨M¼SÀÓ=MÒ,MØ Ü'¨Ô4Ü%&§W¡W×%5Ñ%5×%GÑ%GØ'¨Ñ6Àð &Hó &˜
ð &¨Ó3ÜŸG™G×,Ñ,×5Ñ5°jÀ+ÔNØ,7¸1Ñ,<˜MÙ!Ø$ ]Ñ3°kÓAØ(3°qÑ(8˜Ùðô" �>Ó" aÒ'Ø%�¨¸Ð=Ð=Ü�>Ó" aÒ'Ø% }Ð5°~ÀuÐLÐLØ�2”}“Ð&Ð&ùògs   Ä!Sc           	      ó¸  — g }t        ||«      D ]`  \  }}t        j                  |g|¢­Ž }|t        j                  |j
                  |j                  «      D �cg c]  }|j                  ‘Œ c}z  }Œb t        t           «       }	g }
t        t           «       }t        t           «       }|D �]  }|j                  D ]õ  }t        j                  d|j                  «      sŒ$t        |||«      }|dk(  rŒ7|dk(  rO|	j                  t        |j                  dd  «      «       |
j!                  t        |j                  dd  «      «       Œ‹t#        d„ |j                  D «       «      r(|j                  t        |j                  dd  «      «       ŒÏ|j                  t        |j                  dd  «      «       Œ÷ �Œ
 |	|z
  |z
  }t%        |d„ ¬«      \  }}t'        |«      t'        |«      z   }t'        |	«      dk(  r|dz
  gS |rt)        |«      dd  S |	|z  |z
  }t)        |	«      }t'        |«      dk(  r|d   |v r|d   |dz
  k(  r|S t)        ||
j*                  ¬«      dd  S c c}w )	Nz^d\d+$r   r6   c              3   óP   K  — | ]  }t        |t        j                  «      –— Œ  y ­wr  )r   r   ÚSIZE©r  rÎ  s     rY   r  z6TilingSelect._select_tiling_indices.<locals>.<genexpr>-  s   è ø€ ÒS¸!œ¨¬4¯9©9×5ÑSùó   ‚$&c                 ó   — t        | d   «      S rH  rZ  r[  s    rY   rÊ  z5TilingSelect._select_tiling_indices.<locals>.<lambda>2  ó   € ÄsÈ5ÐQRÉ8Ã}€ rX   r]  r€  r   )r  r
   Úextract_read_writesÚ	itertoolsÚchainÚreadsÚwritesr“   r   ro   rÏ  rÊ   rë   rÈ   r%   rt  r›  r  ra   rÔ   ÚsortedÚcount)r  rp  rq  r  Ú	all_indexÚfnÚ	var_sizesÚrwÚdepÚcontig_varsÚcontig_vars_listÚnon_contig_stride_constÚnon_contig_stride_otherr“   r¨   r`  Úcontig_onlyr4  r5  Únum_itervarsÚcontig_and_const_strideÚcontig_vars_sorteds                         rY   rk  z#TilingSelect._select_tiling_indices  sV  € ð ˆ	Ü  ¨.Ó9ò 	U‰MˆB�	Ü×1Ñ1°"ÐA°yÒAˆBØ¬y¯©¸r¿x¹xÈÏÉÓ/SÖT¨˜#Ÿ)›)ÒTÑT‰Ið	Uô !¤‘oÓ'ˆØÐÜ",¬S¡/Ó"3ÐÜ",¬S¡/Ó"3ÐØó 	CˆEØ×)Ñ)ò C�Ü—y‘y ¨C¯H©HÔ5ØÜ,¨U°C¸ÓG�Ø˜Q’;ØØ˜q’[Ø—O‘O¤C¨¯©°°¨Ó$5Ô6Ø$×+Ñ+¬C°·±¸¸°Ó,=Õ>ÜÑS¸v×?RÑ?RÔSÔSØ+×/Ñ/´°C·H±H¸Q¸R°LÓ0AÕBà+×/Ñ/´°C·H±H¸Q¸R°LÓ0AÕBòCð	Cð "Ð$;Ñ;Ð>UÑUˆÜ!$ ^Ñ9TÔ!UÑˆˆÜ˜5“z¤C¨Ó$8Ñ8ˆÜˆ{Ó˜qÒ à  1Ñ$Ð%Ð%ÙÜ˜+Ó& r sÐ+Ð+àÐ1Ñ1Ø#ñ#$Ðô $ KÓ0ÐäÐ"Ó# qÒ(Ø" 2Ñ&Ð*AÑAØ" 2Ñ&¨,¸Ñ*:Ò:à%Ð%ÜÐ(Ð.>×.DÑ.DÔEÀbÀcÐJÐJùòK Us   ÁIN)
r÷   rø   rù   rú   r¤   r  ro   rI  r}  rk  rW   rX   rY   rR  rR  X  s3   „ ñð
u'ð 
ˆt�C‰y˜$˜s™) ]Ð2Ñ	3ó	u'ón.KrX   rR  c                   óÔ   ‡ — e Zd ZU eZee   ed<   eZee   ed<   e	Z
ee	   ed<   ˆ fd„Zd„ Zdefd„Zdefd	„Zd
„ Zd„ Zd„ Zdee   fd„Zd„ Zd„ Zddedz  fd„Zdeded   fd„Zˆ xZS )r5  Ú
kernel_clsÚvec_kernel_clsÚtile2d_kernel_clsc                 óÌ   •— t         ‰| �  |j                  |j                  j                  «       || _        d | _        d | _        t        j                  «       | _
        g | _        y r  )r  r  ri  Úwsr¥  rC  r4  r’  r   r‰  Úpicked_vec_isaÚkernels©r  rC  r  s     €rY   r  zCppKernelProxy.__init__Q  sQ   ø€ Ü‰Ñ˜×*Ñ*¨L¯O©O×,GÑ,GÔHØ(ˆÔØˆŒØˆÔÜ2=×2JÑ2JÓ2LˆÔØ(*ˆ�rX   c                 ó`   — |D ])  }t        |t        «      sJ ‚t        j                  |«       Œ+ y r  )r£   r$   r;   Úpropagate_scheduler_node)r  r>  r  s      rY   Údata_type_propagationz$CppKernelProxy.data_type_propagationY  s1   € Øò 	@ˆEÜ˜e¤]Ô3Ð3Ð3Ü×8Ñ8¸Õ?ñ	@rX   Úscheduler_nodec                 óÌ   — t        |j                  t        «      syt        j                  |«       t        |j                  «      d   d uxr t        |j                  «      d    S )NTr   r6   )r£   r8  r   r;   r£  rG  )r  r¥  s     rY   Úis_lowp_fp_schedulerz#CppKernelProxy.is_lowp_fp_scheduler_  s\   € Ü˜.×.Ñ.´Ô9Øä×4Ñ4°^ÔDä! .×"6Ñ"6Ó7¸Ñ:À$ÐFò CÜ)¨.×*>Ñ*>Ó?ÀÑBÐBð	
rX   rV  c                 óÔ   — dt         j                  j                  fd„}|j                  gt	        |j
                  j                  «       «      z   }|D ]  } ||j                  «       Œ y )NÚ	sub_graphc                 óZ  ‡‡‡‡‡‡‡‡— dt         j                  j                  dt         j                  d z  fd„Šdt         j                  j                  dt         j                  d z  fd„Šdt         j                  j                  dt         j                  fˆfd„Šdt         j                  j                  dt         j                  fˆfd„Šdt         j                  j                  dt         j                  fˆˆfd„}t	        | j
                  «      }g Š|D �]8  }|j                  d	v r¬ ‰|«      xŠt        v rœt        ˆˆfd
„|j                  D «       «      rŒB|j                  d   }| j                  |«      5  | j                  d||t         j                  f¬«      Š|j                  ‰ˆfd„«       t        xj                   dz  c_        d d d «       Œ¾|j                  dk(  r¨ ‰|«      xŠt        v r˜|j                  \  }}}}} ||‰«      rŒùt"        j$                  j'                  |«      Š| j)                  |«      5  | j                  d||‰f¬«      Š|j+                  |‰«       t        xj                   dz  c_        d d d «       �Œu|j                  dk(  r’|j                  \  }Š}}	}
|t        v s�Œ ‰t         j                  t         j,                  t         j.                  t         j0                  fv sJ ‚|‰t        v rt         j                  n‰t         j                  |	|
f|_
        �Œ|j                  dk(  r`|j                  d   t        v rK|j                  \  }}
Št        ˆˆfd„|j                  D «       «      r�Œk||
t         j                  f|_
        �Œ…|j                  dk(  rq|j                  d   t        v r\|j                  \  }}Št        ˆˆfd„|j                  D «       «      r�ŒÚ‰j3                  |«       ||t         j                  f|_
        �Œ|j                  dk(  s�Œ|j                  \  }}Š}|t        v rd |||«      s[| j)                  |«      5  | j                  d|||f¬«      Š|j+                  |‰«       t        xj                   dz  c_        d d d «       ‰t        v s�Œ�t        ˆˆfd„|j                  D «       «      r�Œ¾|j                  d   }| j                  |«      5  | j                  d||t         j                  f¬«      Š|j                  ‰ˆfd„«       t        xj                   dz  c_        d d d «       �Œ; dt         j                  j4                  fˆfd„} || «       y # 1 sw Y   �ŒoxY w# 1 sw Y   �Œ|xY w# 1 sw Y   ŒõxY w# 1 sw Y   �Œ•xY w)Nr  r¹   c                 óZ  — | j                   dk(  r,t        j                  j                  | j                  d   «      S | j                   dk(  r| j                  d   S | j                   dk(  rCt        | j                  «      dkD  r| j                  d   S | j                  j                  dd«      S y)	z6Get input dtype for nodes that may consumes lowp fp dtrï  r6   rŽ  r€  rz  r   r«   N)r@  r5   ræ  rç  ri  rÔ   rž  r  ©r  s    rY   Úget_input_dtypez]CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.get_input_dtypek  sˆ   € à—;‘; 'Ò)ÜŸ7™7×,Ñ,¨T¯Y©Y°q©\Ó:Ð:Ø—[‘[Ð$6Ò6ØŸ9™9 R™=Ð(Ø—[‘[ JÒ.Ü˜4Ÿ9™9“~¨Ò)Ø#Ÿy™y¨™|Ð+à#Ÿ{™{Ÿ™¨{¸DÓAÐAàrX   c                 ó$  — | j                   dk(  rFt        | j                  «      dk(  sJ ‚t        j                  j                  | j                  d   «      S | j                   dv r| j                  d   S | j                   dk(  r| j                  d   S y)	z6Get output dtype for nodes that may produce lowp fp dtrø  r   r6   )rz  r0  r6  r9  r€  rŽ  r   N)r@  rÔ   ri  r5   ræ  rç  r¬  s    rY   Úget_output_dtypez^CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.get_output_dtypey  s€   € à—;‘; &Ò(Ü˜tŸy™y›>¨QÒ.Ð.Ð.ÜŸ7™7×,Ñ,¨T¯Y©Y°q©\Ó:Ð:Ø—[‘[ð %ñ ð  Ÿ9™9 R™=Ð(Ø—[‘[Ð$6Ò6ØŸ9™9 Q™<Ð'àrX   rj  c                 ó.   •— |t         v sJ ‚ ‰| «      |k(  S )z]Check if the given node produces output with expected low precision floating point data type.)r„   )r  rj  r¯  s     €rY   Úis_lowp_fp_sourcez_CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.is_lowp_fp_sourceŠ  s!   ø€ àœ]Ñ*Ð*Ð*Ù'¨Ó-°Ñ3Ð3rX   c                 óX   •— |t         v sJ ‚ ‰| «      x}r||k(  S | j                  dk(  ryy)zZCheck if the given node accept input with expected low precision floating point data type.rz  TF)r„   r@  )r  rj  Úinput_dtyper­  s      €rY   Úis_lowp_fp_sinkz]CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.is_lowp_fp_sink�  s>   ø€ àœ]Ñ*Ð*Ð*Ù"1°$Ó"7Ð7�;Ð7Ø&¨"Ñ,Ð,Ø—[‘[ JÒ.àà rX   c                 óZ   •‡—  ‰| ‰«      xr t        ˆˆfd„| j                  D «       «      S )z¥Check if the node is a lowp fp sources which are all directly fed to ops that accepts lowp fp input
                thus no need to promote to float
                c              3   ó0   •K  — | ]  } ‰|‰«      –— Œ y ­wr  rW   ©r  Úuserrj  r´  s     €€rY   r  z}CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.is_lowp_fp_source_no_promote.<locals>.<genexpr>ž  s   øè ø€ ò ;Ø26‘O D¨"×-ñ;ùó   ƒ©r  Úusers)r  rj  r´  r±  s    `€€rY   Úis_lowp_fp_source_no_promotezjCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.is_lowp_fp_source_no_promoteš  s1   ù€ ñ )¨¨rÓ2ò ´sô ;Ø:>¿*¹*ô;ó 8ð rX   )rø  r6  r9  c              3   ó0   •K  — | ]  } ‰|‰«      –— Œ y ­wr  rW   r·  s     €€rY   r  zWCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.<genexpr>«  ó   øè ø€ ÒM¸™?¨4°×4ÑMùr¹  r   rz  ©ri  c                 ó   •— | ‰uS r  rW   ©r  Úto_type_nodes    €rY   rÊ  zVCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.<lambda>³  s   ø€ °A¸\Ð4I€ rX   r6   rï  r!  r0  r€  c              3   ó0   •K  — | ]  } ‰|‰«      –— Œ y ­wr  rW   r·  s     €€rY   r  zWCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.<genexpr>å  r¾  r¹  c              3   ó0   •K  — | ]  } ‰|‰«      –— Œ y ­wr  rW   r·  s     €€rY   r  zWCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.<genexpr>ë  r¾  r¹  rŽ  c              3   ó0   •K  — | ]  } ‰|‰«      –— Œ y ­wr  rW   )r  r¸  r‰   r´  s     €€rY   r  zWCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.<genexpr>  s   øè ø€ ÒUÀ¡°°e× <ÑUùr¹  c                 ó   •— | ‰uS r  rW   rÁ  s    €rY   rÊ  zVCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.<lambda>  s   ø€ ¸AÀ\Ð<Q€ rX   r©  c                 óV   •— dt         j                  j                  fˆfd„} || «       y )Nr©  c                 óB  •‡— dt         j                  j                  fd„}| j                  D �cg c]  }|j                  dk(  sŒ|‘Œ }}|D �cg c]  } ||«      sŒ||j
                  i‘Œ }}|D ]†  }|j                  «       D ]q  \  Š}‰| j                  v sŒt        ˆfd„|D «       «      s‰‰v sŒ.t        d„ |D «       «      sŒA‰j                  d   }‰j                  |«       | j                  ‰«       Œs Œˆ | j                  €| j                  «        y y c c}w c c}w )NÚto_nodec                 ó:   — t        d„ | j                  D «       «      S )Nc              3   ó:   K  — | ]  }|j                   d k(  –— Œ y­w)rz  N)r@  ©r  Úusrs     rY   r  z®CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node.<locals>._used_by_to.<locals>.<genexpr>*  s   è ø€ Ò"UÀ 3§:¡:°Õ#;Ñ"Uùs   ‚rº  )rÉ  s    rY   Ú_used_by_toz›CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node.<locals>._used_by_to)  s   € Ü"Ñ"UÀwÇ}Á}Ô"UÓUÐUrX   rz  c              3   ó\   •K  — | ]#  }|j                   d    ‰j                   d    k(  –— Œ% y­w©r€  Nr¿  )r  rÍ  r  s     €rY   r  z™CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node.<locals>.<genexpr>5  s&   øè ø€ Ò#SÀc C§H¡H¨R¡L°D·I±I¸b±MÕ$AÑ#Sùs   ƒ),c              3   óF   K  — | ]  }|j                   d    t        v –— Œ y­wrÐ  )ri  r„   rÌ  s     rY   r  z™CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node.<locals>.<genexpr>8  s"   è ø€ ò ,&ØJM¨¯©°©¼Ô(Eñ,&ùr&  r€  )r…   ÚfxÚNoder>  r@  r»  rª  r  Úall_input_nodesÚreplace_all_uses_withÚ
erase_nodeÚowning_moduleÚlint)	r©  rÎ  r  Úall_to_nodesÚall_to_nodes_and_usersÚ
node_usersr»  Úval_nodeÚto_lowp_fp_legalized_nodess	     `     €rY   Ú_eliminate_duplicate_to_nodez†CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype.<locals>._eliminate_duplicate_to_node"  s-  ù€ ðV¬U¯X©X¯]©]ó Vð *3¯©ö$Ø!%¸D¿K¹KÈ:Ó<Ušð$�Lð $ð 8Dö.Ø/3Á{ÐSWÕGX˜˜tŸz™zÒ*ð.Ð*ð .ð '=ò ;˜
Ø+5×+;Ñ+;Ó+=ò ;™K˜D %Ø# y§¡Ò6Ü #Ó#SÈUÔ#SÔ Sà$(Ð,FÒ$FÜ(+ñ ,&ØQVô,&õ )&ð
 ,0×+?Ñ+?ÀÑ+C Ø $× :Ñ :¸8Ô DØ )× 4Ñ 4°TÕ :ñ;ð;ð, !×.Ñ.Ð6Ø!Ÿ™Õ(ð 7ùò9$ùò.s   ¯DÁDÁDÁD)r…   rÒ  ÚGraph)r©  rÞ  rÝ  s     €rY   Úeliminate_to_dtypez`CppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtype.<locals>.eliminate_to_dtype!  s"   ø€ ð')¼E¿H¹H¿N¹Nõ ')ñR -¨YÕ7rX   )r…   rÒ  rÓ  r‰   r  r>  r@  r„   r  r»  ri  Úinserting_afterÚcall_methodrp   rÕ  r   Úcpp_to_dtype_countr5   ræ  rç  Úinserting_beforeÚreplace_input_withrB  rC  rš  r›  rß  )r©  r¼  Úsub_graph_nodesr  r3   rÈ   rÎ   Ú	value_varr«   rˆ   rð   r„  rà  rj  r‰   r­  r¯  r´  r±  rÝ  rÂ  s                @@@@@@@@rY   Úadd_to_dtypezDCppKernelProxy.legalize_lowp_fp_dtype_loopbody.<locals>.add_to_dtypej  sþ  ÿ€ ð ¤e§h¡h§m¡mð  ¼¿¹ÀdÑ8Jó  ð ¤u§x¡x§}¡}ð  ¼¿¹ÀtÑ9Kó  ð"4¬¯©¯©ð 4¼5¿;¹;õ 4ð
	!¤e§h¡h§m¡mð 	!¼¿¹õ 	!ð´5·8±8·=±=ð ÄeÇkÁkö ô # 9§?¡?Ó3ˆOØ)+Ð&Ø(ó {@�à—L‘LÐ$HÑHÙ/°Ó6Ð6˜¼=ÑHô ÔMÀÇÁÔMÔMØ ØŸ*™* Q™-�CØ"×2Ñ2°5Ó9ñ 8Ø'0×'<Ñ'<Ø&¨c°5¼%¿+¹+Ð-Fð (=ó (˜ð ×3Ñ3Ø(Ó*Iôô  ×2Ò2°aÑ7Õ2÷8ð 8ð —L‘L GÒ+Ù.¨uÓ5Ð5˜¼-ÑGà16·±Ñ.�C˜˜q )¨Qá3°I¸rÔBØ ÜŸG™G×-Ñ-¨dÓ3�EØ"×3Ñ3°EÓ:ñ 8Ø'0×'<Ñ'<Ø&¨c°9¸eÐ-Dð (=ó (˜ð ×0Ñ0°¸LÔIä×2Ò2°aÑ7Õ2÷8ñ 8ð —\‘\ [Ò0ð Ÿ
™
ñØØØ!Ø&Øà ¤MÓ1ð  %Ü!ŸK™KÜ!ŸN™NÜ!ŸM™MÜ!ŸK™Kð	)ñ  ð ð  ð  Ø+0´MÑ+AœEŸKšKÀuÜ!ŸK™KØ*Ø!ð&˜ž
ð —\‘\ ZÒ/°E·J±J¸r±NÄmÑ4Sà',§z¡zÑ$�S˜% ÜÔMÀÇÁÔMÔMÙ Ø"% u¬e¯k©kÐ!:�E–JØ—\‘\ ZÒ/°E·J±J¸r±NÄmÑ4Sà#(§:¡:‘L�S˜!˜RÜÔMÀÇÁÔMÔMÙ ð /×5Ñ5°eÔ<Ø"% q¬%¯+©+Ð!6�E–JØ—\‘\Ð%7Ô7Ø9>¿¹Ñ6�S˜) U¨Ið !¤MÑ1ñ  <¸IÀyÔQØ!*×!;Ñ!;¸EÓ!Bñ @Ø/8×/DÑ/DØ$.°c¸9ÀiÐ5Pð 0Eó 0" ð !&× 8Ñ 8¸ÀLÔ Qä '× :Ò :¸aÑ ?Õ :÷@ð ¤Ó-ô  ÔUÈÏÉÔUÖUà"'§*¡*¨Q¡-˜CØ!*×!:Ñ!:¸5Ó!Añ @Ø/8×/DÑ/DØ$.°c¸5Ä%Ç+Á+Ð5Nð 0Eó 0" ð !&× ;Ñ ;Ø$0Ó2Qô!"ô !(× :Ò :¸aÑ ?Õ :÷@ñ @ðg{@ðz*8¬e¯h©h¯n©nõ *8ñX ˜yÕ)÷8ñ 8ú÷$8ñ 8ú÷L@ð @ú÷"@ñ @ús4   Å7AU:È>AVÑAVÓ3AV Õ:V	ÖV	ÖV	Ö V*	)r…   rÒ  rß  r<  r  r=  rR  ræ  )r  rV  rè  rC  rF  s        rY   Úlegalize_lowp_fp_dtype_loopbodyz.CppKernelProxy.legalize_lowp_fp_dtype_loopbodyi  s]   € ðc	*¤E§H¡H§N¡Nó c	*ðJ  ×*Ñ*Ð+¬d°9×3FÑ3F×3MÑ3MÓ3OÓ.PÑPˆ
Ø#ò 	*ˆIÙ˜Ÿ™Õ)ñ	*rX   c                 óª  ‡ — t        ˆ fd„|D «       «      rÛ|D ]Õ  }|j                  j                  gt        |j                  j                  j                  «       «      z   }|D ]‰  }|j                  j                  D ]n  }|j                  dv sŒ|j                  sJ ‚t        j                  |j                  v sJ ‚|j                  t        j                     }|j                  t        v rŒnJ ‚ Œ‹ Œ× y |D ]^  }t        |t        «      sJ ‚t        |j                  t         «      sJ ‚|j                  }|j#                  «       rŒN‰ j%                  |«       Œ` y )Nc              3   ód   •K  — | ]'  }t        |t        «      xr ‰j                  |«      –— Œ) y ­wr  )r£   r$   r§  )r  r  r  s     €rY   r  z8CppKernelProxy.legalize_lowp_fp_dtype.<locals>.<genexpr>T  s3   øè ø€ ò 
àô �uœmÓ,ÒQ°×1JÑ1JÈ5Ó1QÓQñ
ùó   ƒ-0)rø  rï  )r  r8  r<  r  r=  rR  ræ  r>  r@  rX  rB   rW  r‰   r„   r£   r$   r   Úis_memory_copyré  )r  r>  r  rC  rF  Úfx_noderT  r?  s   `       rY   Úlegalize_lowp_fp_dtypez%CppKernelProxy.legalize_lowp_fp_dtypeS  sD  ø€ Üó 
àô
ô 
ð
 ò B�Ø#Ÿk™k×4Ñ4Ð5¼Ø—K‘K×)Ñ)×0Ñ0Ó2ó9ñ �
ð ",ò B�IØ#,§?¡?×#8Ñ#8ò B˜Ø"Ÿ>™>Ð->Ò>Ø#*§<¢<Ð/ <Ü#6×#:Ñ#:¸g¿l¹lÑ#JÐJÐ#JØ;B¿<¹<Ü 3× 7Ñ 7ñ<˜Gð $+§=¡=´MÒ#AÐAÐ#AñBñBð	Bð àò 	;ˆEÜ˜e¤]Ô3Ð3Ð3Ü˜eŸk™k¬8Ô4Ð4Ð4Ø"Ÿ[™[ˆDØ×&Ñ&Õ(Ø×4Ñ4°TÕ:ñ	;rX   c           	      ó  ‡‡‡ ‡!‡"‡#‡$— t        ‰«      t        ‰«      k(  sJ ‚| j                  Š!t        ‰d„ ¬«      \  Š Š"| j                  ‰ ‰"«       ˆ!ˆ#fd„}ˆˆ ˆ"ˆfd„Š# || j                  «      }t
        j                  xj                  |j                  z  c_        t
        j                  xj                  |j                  z  c_        t        j                  |«      | _        | j                  r| j                  s6|g| _        | j                  dd«       | j                  j!                  | «       yt"        j$                  j&                  j)                  d¬«      5  t+        «       }|j-                  ‰‰«      \  }}Š$dt.        fˆ$fd	„}t        |«      t        |«      k(  sJ ‚d}	d}
d}|r©d}|d
   }|dz   }t        | j                  j0                  «      |kD  r{| j                  j0                  |   j2                  }| j                  j0                  |   j2                  }|xr | }	| xr) t5        d„ | j                  j0                  |d D «       «      }
t        |«      dk(  �rt6        xj8                  dz  c_        | j                  j;                  |d
   |d
   ¬«      } || j<                  |d
   |d
   «      }|j>                  |j@                  z
  }|jB                  d
|j@                  fi|_"        t&        jF                  jH                  r( |||d
   «      r || j<                  |d
   |d
   |«      }n|}|jB                  g|_%        |jB                  |j@                  |j>                  fi|_"        ||g| _        |}�n½t        |«      dk(  �r¦|d   t        | j                  «      dz
  k(  r|d
   |d   k(  sJ ‚t6        xj8                  dz  c_        | j                  j;                  |d
   |d
   ¬«      }d
|j@                  f|j@                  |j>                  fdœ}|j>                  |j@                  z
  }| j                  j;                  |d   |d
   ¬«      }d
|j@                  f|j@                  |j>                  fdœ}|j>                  |j@                  z
  } || jL                  |d
   |«      }|jB                  |d   |jB                  |d   i|_"        g }t&        jF                  jH                  rx |||d
   «      rldD ]f  \  }}|dk(  r|nd}|dk(  r|nd} || jL                  |d
   |||«      }|jB                  ||   |jB                  ||   i|_"        |jO                  |«       Œh n¾ || j<                  |d
   |d
   «      }|jB                  |d   |jB                  |d   i|_"        |jB                  g|_%        |jO                  |«       |jB                  |d   |jB                  d
|j>                  fi|_"        |jB                  |jB                  g|_%        |jO                  |«       |g|z   | _        |}n|g| _        | j                  |	xs t        |«      dk(  xr |
|«       | j                  j!                  | «       ddd«       y# 1 sw Y   yxY w)zOGenerate scalar and vectorized C++ kernels with tiling for the given functions.c                 ó   — t        | d   «      S rH  rZ  r[  s    rY   rÊ  z2CppKernelProxy.codegen_functions.<locals>.<lambda>u  r„  rX   r]  c                 óœ   •—  ‰j                   | g|¢­Ž 5 }t        xj                  dz  c_         ‰|«       |cd d d «       S # 1 sw Y   y xY wrH  )Ú
new_kernelr   Úgenerated_kernel_count)r  ri  rG  rC  Úruns      €€rY   Úcodegen_kernelz8CppKernelProxy.codegen_functions.<locals>.codegen_kernely  sL   ø€ Ø(�×(Ñ(¨Ð4¨tÒ4ð ¸ô ×.Ò.°!Ñ3Õ.á�F”Ø÷÷ ò ús   •#AÁAc           	      óT  •— | j                  ‰‰«      \  }}d}t        ‰‰	«      D ]u  \  }}|‰‰ft        t        j                  ‰‰«      «      dffv r|rJ ‚ |||«       Œ;d}|‰dfk(  sJ d|› d‰› d‰› �«       ‚| j                  «       5   ||d«       d d d «       Œw y # 1 sw Y   Œ‚xY w)NFrW   Tzunexpected group: rQ  r�   )r(  r  r¤   r†  r‡  rn  )
rG  r©  ru  Ú	in_suffixr�  rŽ  rp  r4  r5  rq  s
         €€€€rY   rõ  z-CppKernelProxy.codegen_functions.<locals>.runƒ  sí   ø€ Ø#)×#4Ñ#4°U¸OÓ#LÑ ˆD�.ØˆIÜ!$ W¨nÓ!=ò %‘��IØØ˜OÐ,Üœ9Ÿ?™?¨5°/ÓBÓCÀRÐHð!ñ ñ  )Ð(˜=Ù�t˜^Õ,à $�IØ$ØØð)ò ð Vð ,¨I¨;°d¸5¸'ÀÀOÐCTÐUóVð ð
  ×/Ñ/Ó1ñ %Ù˜4 œ÷%ð %ñ%÷%ð %ús   Â	
BÂB'	FN©Úinplace_buffersr¹   c                 ó8  •— t        ‰j                  j                  «       «      }|dkD  rq‰j                  t        ‰j                  j                  «       «      z   |z  ‰j
                  kD  r1t        j                  j                  j                  | «      }d|z  |kD  S y)Nr   r   T)
re   rL  rR  rN  rM  rJ  r5   ræ  rý  r*  )rl  r  rw  Úhint_tail_sizeÚtiling_statss       €rY   Ú_tail_vec_worthwhilez>CppKernelProxy.codegen_functions.<locals>._tail_vec_worthwhile¬  s”   ø€ ô ˜\×4Ñ4×;Ñ;Ó=Ó>�à˜Q’Jà$×2Ñ2Ü˜l×IÑI×PÑPÓRÓSñTð ñ	ð
 #×;Ñ;ò<ô &'§W¡W×%5Ñ%5×%GÑ%GÈ	Ó%R�NØ˜~Ñ-°Ñ=Ð=ØrX   r   r6   c              3   ó4   K  — | ]  }|j                   –— Œ y ­wr  )r›  )r  r@  s     rY   r  z3CppKernelProxy.codegen_functions.<locals>.<genexpr>Ï  s    è ø€ ò Xà ð ×)Õ)ñXùs   ‚)r"  r   ©ÚmainÚtailr  )r   )r  r  )r  r  r  )(rÔ   rC  ra   r(  rš  r5   ræ  Úremoved_buffersÚinplaced_to_remover2  r]  r4  rŸ  rZ  r   Úaggregate_reduction_buffersÚ
set_kernelr…   Ú	_inductorr   ÚpatchrR  r}  rm   r.  r›  rh   r   Úgenerated_cpp_vec_kernel_countÚtiler›  r&  Ú
tiled_sizer¨   r�  r´  Úenable_loop_tail_vecr‘  rœ  r›  )%r  rp  rq  rö  Úscalar_kernelÚtiling_selectÚtiling_factorsr
  rþ  Ú_inner_loop_reduction_outer_notÚ _tiled_loop_reduction_descendantÚ_outer_loopÚinner_loop_reductionÚouter_loop_levelÚinner_loop_levelÚouter_loop_reductionr@  Ú
vec_kernelrl  Útail_kernelÚ
outer_loopr?  r  Ú
inner_loopÚinner_rangesr  Útile2d_kernelÚouter_rÚinner_rÚ_inner_tail_sizeÚ_outer_tail_sizerG  r4  rC  r5  rõ  rý  s%    ``                             @@@@@rY   Úcodegen_functionsz CppKernelProxy.codegen_functionsq  sš  þ€ ä�7‹|œs >Ó2Ò2Ð2Ð2Ø×(Ñ(ˆÜ!$ ^Ñ9TÔ!UÑˆˆà�‰˜˜Ô/õ	÷	%ñ( ' t§¡Ó7ˆÜ	�‰×Ò =×#@Ñ#@Ñ@ÕÜ	�‰×"Ò" m×&FÑ&FÑFÕ"Ü!Ÿ™¨Ó6ˆŒà×"Ò"¨$¯-ª-Ø)˜?ˆDŒLØ×,Ñ,¨U°DÔ9Ø�N‰N×%Ñ% dÔ+Øô �_‰_×#Ñ#×)Ñ)¸%Ð)Ó@ñ ]	,Ü(›NˆMØ;H×;VÑ;VØ˜ó<Ñ8ˆN˜N¨LðÄ$õ ô" �~Ó&¬#¨nÓ*=Ò=Ð=Ð=Ø.3Ð+Ø/4Ð,ØˆKÙØ',Ð$Ø#1°!Ñ#4Ð Ø#3°aÑ#7Ð Ü�t—~‘~×+Ñ+Ó,Ð/?Ò?Ø+/¯>©>×+?Ñ+?Ø(ñ,ç"‘lð )ð ,0¯>©>×+?Ñ+?Ø(ñ,ç"‘lð )ð -ÒIÐ5IÐ1Ið 4ð <PÐ7Oò 8ÔTWñ Xà$(§N¡N×$8Ñ$8Ð9IÐ9JÐ$KôXó UÐ4ô
 �>Ó" aÓ'ä×6Ò6¸!Ñ;Õ6Ø—~‘~×*Ñ*¨>¸!Ñ+<À^ÐTUÑEVÐ*ÓW�Ù+Ø×'Ñ'¨¸Ñ):¸NÈ1Ñ<Mó�
ð !ŸI™I¨¯©Ñ7�	Ø,0¯H©H°q¸$¿/¹/Ð6JÐ+K�
Ô(Ü—:‘:×2Ò2Ñ7KØ˜~¨aÑ0ô8ñ #1Ø×+Ñ+Ø& qÑ)Ø& qÑ)Ø!ó	#‘Kð #0�KØ48·H±H°:�MÔ0Ø-1¯X©X¸¿¹ÈÏÉÐ7SÐ,T�Ô)Ø *¨KÐ8�”Ø"’Ü�^Ó$¨Ó)à" 1Ñ%¬¨T¯]©]Ó);¸aÑ)?Ò?Ø& qÑ)¨^¸AÑ->Ò>ðð?ô
 ×6Ò6¸!Ñ;Õ6Ø!Ÿ^™^×0Ñ0Ø" 1Ñ%¨n¸QÑ.?ð 1ó �
ð  
× 5Ñ 5Ð6Ø'×2Ñ2°J·O±OÐDñ �ð #-§/¡/°J×4IÑ4IÑ"I�Ø!Ÿ^™^×0Ñ0Ø" 1Ñ%¨n¸QÑ.?ð 1ó �
ð  
× 5Ñ 5Ð6Ø'×2Ñ2°J·O±OÐDñ �ð #-§/¡/°J×4IÑ4IÑ"I�Ù .Ø×*Ñ*Ø" 1Ñ%Ø"ó!�ð —N‘N L°Ñ$8Ø—N‘N L°Ñ$8ð/�Ô+ð !�Ü—:‘:×2Ò2Ñ7KØ# ^°AÑ%6ô8ð-ò 3Ñ(˜ ð 07¸&Ò/@™OÀdð )ð 07¸&Ò/@™OÀdð )ñ "0Ø ×2Ñ2Ø*¨1Ñ-Ø*Ø,Ø,ó"˜ð 'ŸN™N¨L¸Ñ,AØ&ŸN™N¨L¸Ñ,Að0˜Ô,ð $×*Ñ*¨6Õ2ñ-3ñ0 "0Ø×+Ñ+¨^¸AÑ->ÀÈqÑ@Qó"�Jð #Ÿ™¨°VÑ(<Ø"Ÿ™¨°VÑ(<ð0�JÔ,ð 2<·±Ð0@�JÔ-Ø×&Ñ& zÔ2à"Ÿ™¨°VÑ(<Ø"Ÿ™¨¨J¯O©OÐ(<ð3�MÔ/ð 5?·N±NÀJÇNÁNÐ3S�MÔ0Ø×&Ñ& }Ô5Ø -˜°Ñ<�”Ø(‘à -˜�”ð ×,Ñ,Ø/ò SÜ˜Ó'¨1Ñ,ÒQÐ1QØôð
 �N‰N×%Ñ% dÔ+÷{]	,÷ ]	,ñ ]	,ús   ÅTZÚZc                 ó‚   — |D ](  }| j                  |«       t        j                  |«       Œ* | j                  ||«       y r  )ré  r;   Úpropagate_loopbodyr!  )r  rr  rq  r?  s       rY   Úcodegen_loop_bodiesz"CppKernelProxy.codegen_loop_bodiesE  s?   € Øò 	9ˆDØ×0Ñ0°Ô6Ü×2Ñ2°4Õ8ð	9ð 	×Ñ˜{¨NÕ;rX   r>  c                 óÎ  — | j                  |«       | j                  |«       t        |«      dk\  sJ ‚d„ }|D �cg c]  }t        j                  ||«      ‘Œ }}t        t        j                  t        «      r2t        j                  j                  rd„ }|D �cg c]
  } ||«      ‘Œ }}|D �cg c]  }|j                  d   ‘Œ }}| j                  ||«       y c c}w c c}w c c}w )Nr6   c                 ó¾   — | j                  «        | j                  «        t        t        j                  t
        «      r | j                  |Ž S | j                  |«      S r  )Údecide_inplace_updateÚmark_runr£   r5   rG  r2   r8  Úcodegen)r  Ú
index_varss     rY   r�  z(CppKernelProxy.codegen_nodes.<locals>.fnQ  sF   € Ø×&Ñ&Ô(Ø�M‰MŒOÜœ!Ÿ(™(Ô$5Ô6Ø!�t—z‘z :Ð.Ð.à—|‘| JÓ/Ð/rX   c                 óR   — t         j                  j                  | «      }| |_        |S r  )r5   rP  Úlocalize_functionÚoriginal_fn)r�  Ú
wrapped_fns     rY   Úwrap_fnz-CppKernelProxy.codegen_nodes.<locals>.wrap_fn`  s+   € Ü×3Ñ3×EÑEØó�
ð *,�
Ô&Ø!Ð!rX   )rï  r¤  rÔ   r5  Úpartialr£   r5   rP  rL   rQ  r4  r!  )r  r>  r�  r  rp  r/  rq  s          rY   Úcodegen_nodeszCppKernelProxy.codegen_nodesK  sÏ   € à×#Ñ# EÔ*Ø×"Ñ" 5Ô)Ü�5‹z˜QŠÐˆò	0ð <AÖA°4”9×$Ñ$ R¨Õ.ÐAˆÐAô ”q×-Ñ-Ô/AÔBÜ×&Ñ&×4Ò4ò"ð .5Ö5 r‘w˜r•{Ð5ˆGÐ5à49Ö:¨D˜$Ÿ*™* Q›-Ð:ˆÐ:Ø×Ñ˜w¨Õ7ùò# Bùò 6ùâ:s   ºCÂCÂ-C"c                 ó>   — | j                  | j                  ||«       y r  )r[  r4  )r  r´   rE  s      rY   r^  zCppKernelProxy.codegen_loopsl  s   € Ø×Ñ §¡°°kÕBrX   c                 óF   — | j                   D ]  }|j                  «        Œ y r  )r   rµ  ©r  rG  s     rY   rµ  z4CppKernelProxy.update_stores_with_parallel_reductiono  s!   € Ø—l‘lò 	;ˆFØ×8Ñ8Õ:ñ	;rX   Nr´   c                 ó(  — |€J ‚d}| j                   D ]q  }t        j                  «       5 }|j                  ||«      r@d}|j	                  |j                  «       «       |j                  |j                  «       «       d d d «       Œs y # 1 sw Y   Œ~xY w)NÚ
C10_LIKELYÚC10_UNLIKELY)r   rÁ   rÂ   r  rÄ   rÅ   r  r¼  )r  r´   Ú	if_prefixrG  rÍ   s        rY   r¼  zCppKernelProxy.gen_bodys  sŠ   € ØÐÐÐØ ˆ	Ø—l‘lò 	3ˆFÜ×%Ñ%Ó'ð 3¨5Ø×,Ñ,¨T°9Ô=Ø .�IØ×'Ñ'¨¯©«Ô6Ø—K‘K §¡Ó 1Ô2÷	3ð 3ñ	3÷3ð 3ús   ªABÂB	Úinner_loop_reduction_outer_notr  Ú	LoopLevelc                 ó¾  ‡ — dˆ fd„}‰ j                   d   }|r|sJ ‚ ||«       nZ|j                  «        ‰ j                  j                  |j                  «       ‰ j                  j                  |j                  «       ‰ j
                  j                  |j
                  «       ‰ j                  j                  |j                  «       ‰ j                  j                  |j                  «       ‰ j                  j                  |j                  «       ‰ j                  j                  |j                  «       ‰ j                  j                  |j                  «       y)zÐ
        CppKernel/CppVecKernel/CppTile2dKernel have reduction buffers themselves.
        Here, we decide how to aggregate them together and place new reduction buffers
        under CppKernelProxy.
        c           
      ó   •— t        ‰j                  «      dk\  sJ ‚‰j                  d   }‰j                  d   }t        |‰j                  «      sJ ‚t	        |«      ‰j
                  u r^|j                  |j                  «       |j                  «        ‰j                  j                  |j                  |j                  z   «       n5|j                  «        ‰j                  j                  |j                  «       t        «       }t        j                  «       5 }|j                  |d| j                  «      r:|j                  |j!                  «       «       |j                  |j"                  «       d d d «       t        j                  «       5 }|j                  |d| j                  «      �r#|j                  |j!                  «       «       t	        |«      ‰j
                  u rÒ|j$                  }|D ]X  }|› d| j                  › dt'        | j(                  «      › d�}t+        |j,                  ||«       t+        |j"                  ||«       ŒZ t/        |j,                  «       |j                  t1        |j"                  | j                  | j                  › d	�| j(                  | j2                  «      «       n|j                  |j"                  «       d d d «       |‰_        y # 1 sw Y   �ŒpxY w# 1 sw Y   |‰_        y xY w)
Nr   r   r€  r6  r7  rØ   z_tail - rä  r¸  )rÔ   r   r£   r›  r  rš  rú  r  r”  r  r8   rÁ   rÂ   r  r¨   rÄ   rÅ   r–  r§  rF   r  rè   r³  rò   rÑ   r&  )	r  Úmain_loop_kernelÚtail_loop_kernelÚ
suffix_bufrÍ   ru  rÈ   rå   r  s	           €rY   Ú!aggregate_reduction_prefix_suffixzUCppKernelProxy.aggregate_reduction_buffers.<locals>.aggregate_reduction_prefix_suffix†  sª  ø€ Ü�t—|‘|Ó$¨Ò)Ð)Ð)Ø#Ÿ|™|¨A™ÐØ#Ÿ|™|¨BÑ/ÐÜÐ.°×0CÑ0CÔDÐDÐDô Ð$Ó%¨¯©Ñ8ð !×:Ñ:Ø$×2Ñ2ôð !×:Ñ:Ô<Ø×%Ñ%×,Ñ,Ø$×5Ñ5Ø&×7Ñ7ñ8õð
 !×:Ñ:Ô<Ø×%Ñ%×,Ñ,Ð-=×-NÑ-NÔOô &›ˆJÜ×%Ñ%Ó'ð I¨5Ø#×6Ñ6Ø ¨j¯n©nôð ×'Ñ'¨
×(9Ñ(9Ó(;Ô<Ø×%Ñ%Ð&6×&GÑ&GÔH÷Iô ×%Ñ%Ó'ð M¨5Ø#×6Ñ6Ø °
·±õð ×'Ñ'¨
×(9Ñ(9Ó(;Ô<ÜÐ,Ó-°·±Ñ@Ø)9×)MÑ)M˜Ø$2ò ˜DØ*.¨¨u°Z·^±^Ð4DÀHÌ[ÐYc×YnÑYnÓMoÐLpÐpqÐ'r˜HÜ,Ð-=×-DÑ-DÀdÈHÔUÜ,Ø 0× AÑ AÀ4Èõðô 5Ð5E×5LÑ5LÔMØ"×)Ñ)Ü6Ø 0× AÑ AØ *§¡Ø#-§>¡>Ð"2°%Ð 8Ø *× 5Ñ 5Ø *§¡óõð #×)Ñ)Ð*:×*KÑ*KÔL÷5Mð6 %/ˆDÕ!÷CIñ Iú÷Mð6 %/ˆDÕ!ús   ÄAK ÆEK-Ë K*Ë-K=r   N)r  r:  )r   rú  r”  r  r–  r—  r˜  r™  rš  rœ  r�  )r  r9  r  r@  Úmain_kernels   `    rY   r  z*CppKernelProxy.aggregate_reduction_buffers}  s  ø€ õ9	/ðv —l‘l 1‘oˆÙ)ÙÐ�:Ù-¨jÕ9à×1Ñ1Ô3Ø×!Ñ!×(Ñ(¨×)EÑ)EÔFØ×!Ñ!×(Ñ(¨×)EÑ)EÔFØ×&Ñ&×-Ñ-¨k×.SÑ.SÔTØ×&Ñ&×-Ñ-¨k×.SÑ.SÔTØ×!Ñ!×(Ñ(¨×)IÑ)IÔJØ×#Ñ#×*Ñ*¨;×+MÑ+MÔNØ×*Ñ*×1Ñ1Ø×5Ñ5ô	
ð 	×*Ñ*×1Ñ1Ø×5Ñ5õ	
rX   r  )r÷   rø   rù   rŠ  rš  r  rû   rœ  r›  r†  rœ  r  r¤  r$   r§  r   ré  rï  r!  r$  r  r1  r^  rµ  r8   r¼  rm   r	   r  rM  rN  s   @rY   r5  r5  F  s´   ø… ð #,€J��Y‘Ó+Ø)5€N�D˜Ñ&Ó5Ø/>Ð�t˜OÑ,Ó>ô+ò@ð
°=ó 
ðh*¸ó h*òT;ò<R,òh<ð8 4¨Ñ#6ó 8òBCò;ñ3˜\¨DÑ0ó 3ðU
Ø.2ðU
Ø@HÈÑ@U÷U
rX   r5  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )rD  c                 óp   •— t         ‰| �  |j                  |j                  j                  «       g | _        y r  )r  r  ri  rž  r¥  rF  r¡  s     €rY   r  zOuterLoopFusedKernel.__init__Ö  s)   ø€ Ü‰Ñ˜×*Ñ*¨L¯O©O×,GÑ,GÔHØ%'ˆ�
rX   c           
      ó   — g }| j                   D �cg c]  }|j                  «       ‘Œ }}|D ]h  }|j                  }|€J ‚|j                  |j	                  t        t        |«      |j                  z
  |j                  ¬«      |«      j                  «       Œj t        t        |j                  t        |«      «      |j                  ¬«      S c c}w )Nrg  )rF  r4  r’  r›  rL  rô   rÔ   rö   rõ   r`   ra   )r  rM  rD  Úkernels_parallel_depthr4  Únested_kernelsrG  r’  s           rY   rL  z*OuterLoopFusedKernel.decide_parallel_depthÚ  sÞ   € Ø!#Ðà48·J±Jö+
Ø'0ˆI× Ñ Õ"ð+
ˆð +
ð %ò 	ˆFð !×,Ñ,ˆKØÐ*Ð*Ð*Ø"×)Ñ)Ø×,Ñ,Ü!ä Ó,Ð/A×/MÑ/MÑMà$6×$BÑ$Bô	ð ó÷ !‘.õ
ð	ô  ÜØ"×1Ñ1´3Ð7MÓ3Nóð +×6Ñ6ô	
ð 	
ùò)+
s   ‘C)r÷   rø   rù   r  rL  rM  rN  s   @rY   rD  rD  Õ  s   ø„ ô(ö
rX   rD  c                   ó   — e Zd ZdZdZdZy)ÚReasonFusedNodesÚsame_vars_reduceÚcompatible_reductionÚcompatible_ranges_no_reductionN)r÷   rø   rù   ÚSAME_VARS_REDUCEÚCOMPATIBLE_REDUCTIONÚCOMPATIBLE_RANGES_NO_REDUCTIONrW   rX   rY   rH  rH  ø  s   „ Ø)ÐØ1ÐØ%EÑ"rX   rH  c                   ó¶  ‡ — e Zd ZU eZee   ed<   dZ ee	j                  e	j                  g«      Zedej                  dee	   fd„«       Zˆ fd„Zdefd„Zd	„ Zd
„ Zd„ Zd„ Zd„ Zd„ Zdedz  fd„Zd„ Zd„ Zd„ Zdededefd„Z d„ Z!d„ Z"d„ Z#d„ Z$de%e&   fd„Z'de(fd„Z)de(e*z  e&z  fd„Z+dedefd „Z,d!ed"e-e   d#e-e   fd$„Z.d%„ Z/d&„ Z0d'„ Z1d+d(„Z2d)„ Z3d+d*„Z4ˆ xZ5S ),ÚCppSchedulingÚkernel_proxy_clsiô  Údevicer¹   c                 ó   — | j                   S r  )Úbackend_features)r  rR  s     rY   Úget_backend_featuresz"CppScheduling.get_backend_features  s   € à×#Ñ#Ð#rX   c                 óV   •— t         ‰| �  |«       |r| j                  «        d| _        y r�  )r  r  Úreset_kernel_groupÚ_ready_to_flush)r  r  r  s     €rY   r  zCppScheduling.__init__  s'   ø€ Ü‰Ñ˜Ô#ÙØ×#Ñ#Ô%Ø$ˆÕrX   Ústatusc                 ó   — || _         y r  ©rX  )r  rY  s     rY   Ú_set_flush_statuszCppScheduling._set_flush_status  s
   € Ø%ˆÕrX   c                 ó&   — t        d„ |D «       «      S )Nc              3   óŠ   K  — | ];  }t        t        t        j                  j                  j
                  |«      «      –— Œ= y ­wr  )r¤   rÌ  r5   ræ  rý  Úsimplifyr�  s     rY   r  z)CppScheduling.group_fn.<locals>.<genexpr>  s,   è ø€ ÒMÀ!”Uœ3œqŸw™w×/Ñ/×8Ñ8¸!Ó<×=ÑMùs   ‚AA)r¤   )r  r\  s     rY   Úgroup_fnzCppScheduling.group_fn  s   € ÜÑMÀuÔMÓMÐMrX   c                 ó"   — t        «       | _        y r  )ÚKernelGrouprC  r  s    rY   rW  z CppScheduling.reset_kernel_group  s   € Ü'›MˆÕrX   c                 óT  — t        |t        «      rœt        |j                  «      dkD  sJ |j                  «       ‚d }t	        t
           «       }|j                  D ]D  }| j                  |«      \  }}|€|}||k(  sJ |||j                  f«       ‚|j                  |«       ŒF |t        |«      fS t        |t        «      sJ ‚|j                  }t        |t        j                  «      sJ ‚|j                  «       \  }}	}|	j                  t        |	j                  j!                  «       «      fS rB  )r£   r"   rÔ   Úsnodesr   r   Ú_get_indexing_ranges_exprsÚupdater  r$   r  r   ÚComputedBufferÚget_default_sizes_bodyrÕ  Úindexing_exprsrR  )
r  r  rÕ  ri  ÚsnodeÚvÚexprsÚcomp_bufferrÎ   r?  s
             rY   re  z(CppScheduling._get_indexing_ranges_exprs!  s  € Ü�dÔ.Ô/Ü�t—{‘{Ó# aÒ'Ð4¨¯©Ó4Ð'ØˆJÜ'¬™_Ó.ˆNØŸ™ò -�Ø×:Ñ:¸5ÓA‘��5ØÐ%Ø!"�JØ! Q’ÐD¨°Q¸¿¹Ð(DÓD�Ø×%Ñ% eÕ,ð-ð œt NÓ3Ð3Ð3ä˜$¤Ô.Ð.Ð.Ø—i‘iˆÜ˜+¤r×'8Ñ'8Ô9Ð9Ð9Ø ×7Ñ7Ó9‰
ˆˆ4�Ø�‰¤ T×%8Ñ%8×%?Ñ%?Ó%AÓ BÐBÐBrX   c                 óü   — t        |t        «      r||j                  «       fgS t        |t        «      sJ ‚g }|j                  D ]5  }t        |t        «      sJ ‚|j                  ||j                  «       f«       Œ7 |S r  )r£   r$   Úsnapshot_loop_stater"   rd  r›  )r  r  Ú	snapshotsrj  s       rY   Ú_snapshot_node_loop_statesz(CppScheduling._snapshot_node_loop_states4  s‚   € Ü�dœMÔ*Ø˜4×3Ñ3Ó5Ð6Ð7Ð7ä˜$Ô 2Ô3Ð3Ð3Øˆ	Ø—[‘[ò 	CˆEÜ˜e¤]Ô3Ð3Ð3Ø×Ñ˜e U×%>Ñ%>Ó%@ÐAÕBð	Cð ÐrX   c                 óB  — t        |t        t        f«      sJ ‚t        |t        t        f«      sJ ‚|j                  \  }\  }}|j                  \  }\  }}|dk(  r|dk(  s	J ||f«       ‚t	        |«      t	        |«      k  r|n|}t	        |«      t	        |«      k  r|n|}	t        |t        «      sJ ‚| j                  |	«      }
|j                  |
¬«       |j                  \  }\  }}|j                  \  }\  }}||k(  ry| j                  |«      }t        |	t        «      r|	j                  |¬«       nGt        |	t        «      sJ ‚|	j                  D ]&  }t        |t        «      sJ ‚|j                  |¬«       Œ( |j                  \  }\  }}|j                  \  }\  }}||k(  S )NrW   )Úextra_indexing_constraintsT)r£   r$   r"   r4  rÔ   re  Úrecompute_size_and_bodyrd  )r  rþ   rÿ   rÎ   Úvars1Úreduce1Úvars2Úreduce2Únode_to_recompÚref_nodeÚref_indexing_constraintsÚ#node_to_recomp_indexing_constraintsrj  s                rY   Ú_align_compatible_range_nodesz+CppScheduling._align_compatible_range_nodes?  s²  € Ü˜%¤-Ô1CÐ!DÔEÐEÐEÜ˜%¤-Ô1CÐ!DÔEÐEÐEà#Ÿk™kÑˆÑˆE�7Ø#Ÿk™kÑˆÑˆE�7Ø˜"Š} ¨B¢ÐB°¸'Ð0BÓBÐ.ä"% e£*¬s°5«zÒ"9™¸uˆÜ ›J¬¨U«Ò3‘5¸ˆÜ˜.¬-Ô8Ð8Ð8à#'×#BÑ#BÀ8Ó#LÐ Ø×.Ñ.Ø'?ð 	/ô 	
ð Ÿ™‰ˆ‰:ˆE�1ØŸ™‰ˆ‰:ˆE�1Ø�EŠ>Øà.2×.MÑ.MØó/
Ð+ô �h¤Ô.Ø×,Ñ,Ø+Nð -õ ô ˜hÔ(:Ô;Ð;Ð;Ø!Ÿ™ò �Ü! %¬Ô7Ð7Ð7Ø×-Ñ-Ø/Rð .õ ðð Ÿ™‰ˆ‰:ˆE�1ØŸ™‰ˆ‰:ˆE�1Ø˜‰~ÐrX   c                 óL  — |j                  «       s|j                  «       rt        j                  ||«      S |j                  «       r(|j                  «       rJ ‚t	        j                  ||«      S | j                  ||«      t        j                  k(  rE| j                  ||«      sJ |j                  |j                  f«       ‚t	        j                  ||«      S | j                  ||«      r't        j                  ||| j                  ||«      «      S t	        j                  ||«      S r  )Ú
is_foreachr!   r  Úis_templater"   Ú_why_fuse_nodesrH  rN  r}  r4  Úcan_fuse_vertical_outer_looprý   Ú_get_outer_loop_fusion_depth©r  rþ   rÿ   s      rY   r  zCppScheduling.fuseh  s  € Ø×ÑÔ ×!1Ñ!1Ô!3Ü-×2Ñ2°5¸%Ó@Ð@Ø×ÑÔ Ø×(Ñ(Ô*Ð*Ð*Ü%×*Ñ*¨5°%Ó8Ð8ð ×$Ñ$ U¨EÓ2Ü#×BÑBòCð ×9Ñ9¸%ÀÔGð Ø—K‘KØ—K‘KðJó ÐGô *×.Ñ.¨u°eÓ<Ð<Ø×2Ñ2°5¸%Ô@Ü2×7Ñ7Ø˜5 $×"CÑ"CÀEÈ5Ó"Qóð ô *×.Ñ.¨u°eÓ<Ð<rX   Nc                 óþ   — |j                   \  }\  }}|j                   \  }\  }}||k(  r||k(  rt        j                  S |dk(  r|||z   k(  rt        j                  S | j	                  ||«      rt        j
                  S y )NrW   )r4  rH  rL  rM  Ú&_can_fuse_nodes_with_compatible_rangesrN  )r  rþ   rÿ   rÎ   ru  rv  rw  rx  s           rY   r�  zCppScheduling._why_fuse_nodes  s   € Ø#Ÿk™kÑˆÑˆE�7Ø#Ÿk™kÑˆÑˆE�7à�EŠ>˜g¨Ò0Ü#×4Ñ4Ð4Ø�bŠ=˜U e¨g¡oÒ5Ü#×8Ñ8Ð8Ø×6Ñ6°u¸eÔDÜ#×BÑBÐBàrX   c                 ó€  — |j                   \  }\  }}|j                   \  }\  }}|dk(  xr |dk(  }t        j                  |«      t        j                  |«      k(  }	t        |«      dk(  xs t        |«      dk(  }
|r|	r|
syt        |«      t        |«      k  r|n|}t        |«      t        |«      k  r|n|}t	        |t
        «      ryt	        |t        «      sJ ‚t	        |j                  t        j                  «      ryt	        |j                  t        j                  «      sJ ‚|j                  j                  j                  «       }d }t	        |t
        «      ròt        t        t        df      «       }|j                   D ]œ  }t	        |t        «      sJ ‚t	        |j                  t        j                  «      r ndt	        |j                  t        j                  «      sJ ‚|j#                  t        |j                  j                  j                  «       «      «       Œž t        |«      dk7  ryt%        t'        t)        |«      «      «      }n\t	        |t        «      sJ ‚t	        |j                  t        j                  «      sJ ‚|j                  j                  j                  «       }||k7  ry| j+                  |«      }|j-                  | j+                  |«      «       	 | j/                  ||«      t1        |«      D ]  \  }}|j3                  |«       Œ S # t1        |«      D ]  \  }}|j3                  |«       Œ w xY w)NrW   r6   F.)r4  r“  rf   rÔ   r£   r"   r$   r  r   ÚTemplateBufferrg  ÚdataÚget_sizer   r¤   r   rd  rt  r  ÚnextÚiterrq  r  r}  r3  Úrestore_loop_state)r  rþ   rÿ   rÎ   ru  rv  rw  rx  Úc1Úc2Úc3ry  rz  Úranges2Úranges1Ú
ranges_setrj  rp  r  Ústates                       rY   r†  z4CppScheduling._can_fuse_nodes_with_compatible_rangesŒ  sµ  € ð $Ÿk™kÑˆÑˆE�7Ø#Ÿk™kÑˆÑˆE�7à˜‰]Ò,˜w¨"™}ˆÜ�Y‰Y�uÓ¤§¡¨5Ó!1Ñ1ˆÜ�‹Z˜1‰_Ò/¤ E£
¨a¡ˆÙ‘r™bØä"% e£*¬s°5«zÒ"9™¸uˆÜ ›J¬¨U«Ò3‘5¸ˆô �nÔ&8Ô9Øô ˜.¬-Ô8Ð8Ð8Ü�n×)Ñ)¬2×+<Ñ+<Ô=ØÜ˜.×-Ñ-¬r×/@Ñ/@ÔAÐAÐAð !×%Ñ%×*Ñ*×3Ñ3Ó5ˆØˆÜ�hÔ 2Ô3Ü#¤E¬#¨s¨(¡OÑ4Ó6ˆJØ!Ÿ™ò B�Ü! %¬Ô7Ð7Ð7Ü˜eŸj™j¬"×*;Ñ*;Ô<ÙÜ! %§*¡*¬b×.?Ñ.?Ô@Ð@Ð@Ø—‘œu U§Z¡Z§_¡_×%=Ñ%=Ó%?Ó@ÕAðBô �:‹ !Ò#Øäœ4¤ ZÓ 0Ó1Ó2‰Gä˜h¬Ô6Ð6Ð6Ü˜hŸm™m¬R×->Ñ->Ô?Ð?Ð?Ø—m‘m×(Ñ(×1Ñ1Ó3ˆGà�gÒØà×3Ñ3°NÓCˆ	Ø×Ñ˜×8Ñ8¸ÓBÔCð	/Ø×5Ñ5°e¸UÓCä'¨	Ó2ò /‘��eØ×'Ñ'¨Õ.ñ/øœx¨	Ó2ò /‘��eØ×'Ñ'¨Õ.ñ/ús   Ë!L Ì&L=c                 ó¾   — t        |t        t        t        f«      sJ ‚t        |t        t        f«      sJ ‚t	        d„ ||fD «       «      ry| j                  ||«      d uS )Nc              3   óH   K  — | ]  }t        |t        t        f«      –— Œ y ­wr  )r£   rý   r    r  s     rY   r  z:CppScheduling._can_fuse_horizontal_impl.<locals>.<genexpr>Ð  s&   è ø€ ò 
àô �tÔ9Ô;TÐU×Vñ
ùs   ‚ "F)r£   r"   r$   r    rh   r�  r„  s      rY   Ú_can_fuse_horizontal_implz'CppScheduling._can_fuse_horizontal_implË  sp   € ÜØÔ&¬Ô7PÐQô
ð 	
ð 
ô ˜%Ô"4´mÐ!DÔEÐEÐEÜñ 
à ˜ô
ô 
ð Ø×#Ñ# E¨5Ó1¸Ð=Ð=rX   c                 ó  — |j                  «       s|j                  «       ryt        |j                  «       «      t        |j                  «       «      z   t        j                  j
                  kD  ry| j                  ||«      S r�  )r€  rÔ   r  r   r´  Úmax_horizontal_fusion_sizer—  r„  s      rY   Úcan_fuse_horizontalz!CppScheduling.can_fuse_horizontal×  sf   € Ø×ÑÔ %×"3Ñ"3Ô"5Øä�—‘Ó!Ó"¤S¨¯©Ó):Ó%;Ñ;Ü�j‰j×3Ñ3ò4ð à×-Ñ-¨e°UÓ;Ð;rX   rþ   rÿ   c                 óp  — |j                  «       x}r¤t        |j                  t        j                  «      xr~ t        |j
                  t        j                  «      xrX t        |j
                  j                  «      dk(  xr4 |j
                  j                  d   j                  «       |j                  k(  S y)Nr6   r   F)Úget_template_noder£   Úlayoutr   ÚMultiOutputLayoutr  ÚMultiOutputrÔ   ÚinputsrT  rÈ   )r  rþ   rÿ   Útemplate_bufs       rY   Úcan_fuse_multi_outputs_templatez-CppScheduling.can_fuse_multi_outputs_templateâ  s›   € ð !×2Ñ2Ó4Ð4ˆ<Ð4ä˜<×.Ñ.´×0DÑ0DÓEò IÜ˜uŸz™z¬2¯>©>Ó:òIä˜Ÿ
™
×)Ñ)Ó*¨aÑ/òIð —J‘J×%Ñ% aÑ(×1Ñ1Ó3°|×7HÑ7HÑHð	ð rX   c                 óì  — d}t        d„ ||fD «       «      s|S t        |t        «      r|j                  «       d   n|}t        |t        t
        f«      sJ ‚t        |t        «      r|j                  «       d   n|}t        |t        t
        f«      sJ ‚|j                  \  }\  }}|j                  \  }\  }	}
|dk(  r|	dk(  r|dk7  r|
dk7  r|S t        d„ ||fD «       «      r'|j                  |j                  k(  r|j                  S |S t        t        |«      t        |	«      «      }|dk\  rI|d | |	d | k(  r>t        d„ ||fD «       «      r(t        |«      t        u r|n|}|j                  |k(  r|S |S |S |S )Nr   c              3   óT   K  — | ]   }t        |«      t        t        t        fv –— Œ" y ­wr  )r  rý   r"   r$   r  s     rY   r  z=CppScheduling._get_outer_loop_fusion_depth.<locals>.<genexpr>ð  s/   è ø€ ò 
ð ô �‹JÜ+Ô-?ÄÐOôPñ
ùr  r€  rW   c              3   ó>   K  — | ]  }t        |«      t        u –— Œ y ­wr  r
  r  s     rY   r  z=CppScheduling._get_outer_loop_fusion_depth.<locals>.<genexpr>	  r  r  r6   c              3   ó>   K  — | ]  }t        |«      t        u –— Œ y ­wr  r
  r  s     rY   r  z=CppScheduling._get_outer_loop_fusion_depth.<locals>.<genexpr>  s   è ø€ ò Ø>B”�T“
Ô9Ô9ñùr  )r  r£   rý   r  r"   r$   r4  r  r`   rÔ   rh   r  )r  rþ   rÿ   ÚDISABLE_OUTER_LOOP_FUSIONÚ_node1Ú_node2rÎ   ru  rv  rw  rx  r  Ú_compare_nodes                rY   rƒ  z*CppScheduling._get_outer_loop_fusion_depthî  sÇ  € Ø$%Ð!Üñ 
ð  ˜ô
ô 
ð
 -Ð,ô ˜%Ô!<Ô=ð ×!Ñ!Ó# BÒ'àð 	ô
 ˜&Ô#5´}Ð"EÔFÐFÐFô ˜%Ô!<Ô=ð ×!Ñ!Ó# AÒ&àð 	ô
 ˜&Ô#5´}Ð"EÔFÐFÐFà$Ÿl™lÑˆÑˆE�7Ø$Ÿl™lÑˆÑˆE�7Ø�BŠ;˜5 Bš;¨7°bª=¸WÈº]à,Ð,ÜÑTÀeÈUÀ^ÔTÔTð ×0Ñ0°E×4QÑ4QÒQð ×-Ñ-ðð /ðô
 #&¤c¨%£j´#°e³*Ó"=Ðà# qÒ(ØÐ.Ð.Ð/°5Ð9QÐ:QÐ3RÒRäñ ØGLÈeÀnôô ô " %›[Ô,GÑG‘EÈUð ð !×8Ñ8Ð<SÒSà2Ð2à4Ð4ð /Ð.Ø(Ð(rX   c                 ó  — |j                  «        xro |j                  «        xr\ |j                  «       |j                  z  xr= | j                  ||«      xr |j	                  «         xr | j                  ||«      dk\  S rH  )r€  Úget_operation_namesÚ	ancestorsr—  r›  rƒ  r„  s      rY   r‚  z*CppScheduling.can_fuse_vertical_outer_loop$  s”   € à×!Ñ!Ó#Ð#ò EØ×%Ñ%Ó'Ð'òEà×)Ñ)Ó+¨e¯o©oÑ=òEð ×.Ñ.¨u°eÓ<ò -Ø×*Ñ*Ó,Ð,ðòEð ×1Ñ1°%¸Ó?À1ÑDð		
rX   c                 ó*   — | j                  ||«      ryyrd  )r‚  r„  s      rY   Úget_fusion_pair_priorityz&CppScheduling.get_fusion_pair_priority0  s   € Ø×,Ñ,¨U°EÔ:ààrX   c                 ó   — |j                  «       ry|j                  «       r%t        ||g«      \  }}|j                  «        xr |S | j                  ||«      xr |j                  «        xs | j	                  ||«      S r�  )r€  rO   r›  r—  r‚  )r  rþ   rÿ   Útemplate_fusion_supportedrÎ   s        rY   Úcan_fuse_verticalzCppScheduling.can_fuse_vertical7  s‰   € Ø×ÑÔàØ×ÑÔÜ+SØ˜�wó,Ñ(Ð% qð ×)Ñ)Ó+Ð+ÒIÐ0IÐIà×*Ñ*¨5°%Ó8ÒUÀ×ASÑASÓAUÐ=Uò=à×.Ñ.¨u°eÓ<ð	=rX   r>  c                 óŒ  ‡‡‡‡‡— t        d„ |D «       «      r|S dŠdŠd}d}d}d}g }|D �]­  }t        |j                  t        j                  «      sJ ‚|j                  j                  «       }|j                  ||f«       |\  \  }	}
}}
|j                  j                  «       D �]0  \  Š}t        |t        j                  «      sŒ"|j                  t        «      D ]÷  Št        ˆfd„|j                  D «       «      r‰|k7  r‰}|dz  }|dkD  r|c c c S t        ‰j                  d   t        j                  j                   j"                  «      sŒv‰j                  d   |j                  v sŒ’‰€Œ•t%        ˆˆfd„|j                  j                  «       D «       «      sŒÃ‰j                  d   dkD  sŒÖ‰j                  d   Š‰j                  d   Šd	}|	}Œù �Œ3 �Œ° |s|S |€J ‚t'        |«      }|D ]0  \  }\  \  }	}
}}
‰|j                  vr|c S t'        |	«      |k7  sŒ.|c S  ˆˆfd
„}d}t)        t*           «       }|D ]g  \  }
} ||Ž \  }
}}
|€|j,                  }||j,                  k(  sJ ||j,                  f«       ‚|j/                  |j                  j1                  «       «       Œi |€J ‚|t3        |«      f}|D �cg c]  }||j5                  «       f‘Œ }}|D ]  }|j7                  ||¬«       Œ |d   j8                  d   Št        ˆfd„|dd D «       «      r$t;        |«      D ]  \  }}|j=                  |«       Œ |S c c}w )aI  
        Apply loop split optimization.
        When one of the indexing_exprs contains a division, we eliminate the division by splitting the loop
        to avoid non-contiguous loads, subject to the following conditions:
            1. No reduction and no mudular index for all nodes.
            2. The indexing_exprs of all nodes contain only one (or more, but all the same) division,
               where the divisor is an integer and not too small (the divisor > 8), the dividend is
               one of the iter_vars, and this var, i.e. the dimension that needs to be split, is
               contiguous in all other indexing_exprs.

        For example, if the node's var_ranges: {z0: 2, z1: 9216, z2: 960} and indexing_exprs:
        {'index0': 8847360*z0 + 960*z1 + z2, 'index1': 32*z0 + (z2//30), 'index2': z2},
        we will split z2 -> 30*z2 + z3, then the node's var_ranges will be changed to
        {z0: 2, z1: 9216, z2: 32, z3: 30} and indexing_exprs will be changed to
        {'index0': 8847360*z0 + 960*z1 + 30*z2 + z3, 'index1': 32*z0 + z2, 'index2': 30*z2 + z3}.
        c              3   óÄ   K  — | ]X  }t        |j                  d    d    «      dk7  xs4 t        d„ |j                  j                  j                  «       D «       «      –— ŒZ y­w)r6   r   c              3   óF   K  — | ]  }|j                  t        «      –— Œ y ­wr  )r  r   )r  r†  s     rY   r  z9CppScheduling.try_loop_split.<locals>.<genexpr>.<genexpr>Y  s   è ø€ ò Ø.2�—‘œ×)ñùr&  N)rÔ   r4  rh   r8  ri  rR  r  s     rY   r  z/CppScheduling.try_loop_split.<locals>.<genexpr>W  sc   è ø€ ò 
ð
 ô	 �—
‘
˜1‘˜aÑ Ó! QÑ&ò Üñ Ø6:·j±j×6OÑ6O×6VÑ6VÓ6Xôó óñ
ùs   ‚AA Nr   Fc              3   ó@   •K  — | ]  }‰j                  |«      –— Œ y ­wr  )r  )r  r¨   Údiv_exprs     €rY   r  z/CppScheduling.try_loop_split.<locals>.<genexpr>t  s   øè ø€ ÒQ°#˜HŸL™L¨×-ÑQùrX  r6   c              3   ód   •K  — | ]'  \  }}|‰k7  rt        |‰j                  d    «      dv –— Œ) y­w)r   rf  N)r%   ri  )r  Úname_Úexpr_r·  rÈ   s      €€rY   r  z/CppScheduling.try_loop_split.<locals>.<genexpr>  s9   øè ø€ ò  á ,  uØ$¨š}ô 0°°x·}±}ÀQÑ7GÓHÈFÔRñ ùrì  r’  Tc                 ó‚  •— | \  }}|\  }}|j                  ‰«      }|j                  «       }||   ‰z  ||<   |j                  |dz   ‰«       t        j                  ||d¬«      \  \  }	}
}|	j                  «       }|j                  |dz   «      }‰||   z  |z   ||<   t        j                  |||g||	|«      }||f||	|ffS )Nr6   ré  )r:  )r“   ÚcopyÚinsertr
   Úindex_vars_no_squeezeÚpopr   r   )r\  r?  r©  Ú
index_sizeÚreduce_sizer*  Úreduce_varsÚ	split_idxÚnew_index_sizeÚnew_index_varsrÎ   rÕ  Ú	iter_varsÚdivisor_varÚsplit_numberÚ	split_vars                 €€rY   Ú
loop_splitz0CppScheduling.try_loop_split.<locals>.loop_splitœ  sô   ø€ Ø&+Ñ#ˆJ˜Ø&*Ñ#ˆJ˜Ø"×(Ñ(¨Ó3ˆIØ'Ÿ_™_Ó.ˆNØ(2°9Ñ(=ÀÑ(MˆN˜9Ñ%Ø×!Ñ! )¨a¡-°Ô>Ü.:×.PÑ.PØ °Cô/Ñ+Ñˆ^˜Q ð '×+Ñ+Ó-ˆIØ#Ÿ-™-¨	°A©Ó6ˆKØ#/°)¸IÑ2FÑ#FÈÑ#TˆI�iÑ Ü—;‘;Ø�y +Ð.°
¸NÈKóˆDð   Ð-ØØ Ð-ðð rX   )rs  Úrecompute_sizes_body_funcc              3   óB   •K  — | ]  }|j                   d    ‰k7  –— Œ y­w©r6   N)r4  )r  r  r4  s     €rY   r  z/CppScheduling.try_loop_split.<locals>.<genexpr>Ï  s   øè ø€ Ò<¨$ˆt�z‰z˜!‰} Õ%Ñ<ùs   ƒ)rh   r£   r  r   rg  rh  r›  ri  rª  r:  r’  Úfindr   rÆ  ri  ÚcoreÚnumbersr;  r  rÔ   r   r   rÕ  rf  rR  r  ro  rt  r4  r3  r�  )r  r>  Únum_divÚ	div_expr_Ú	match_divÚmatched_index_sizeÚnode_bodiesr  Ú
sizes_bodyrÀ  rÎ   Úoriginal_bodyr†  Úmatched_num_dimsrÊ  Úextra_indexing_rangesÚextra_indexing_exprsÚ
split_bodyrs  rp  r”  r·  r4  rÈ   rÈ  rÉ  s                        @@@@@rY   Útry_loop_splitzCppScheduling.try_loop_splitD  s—  ü€ ô& ñ 
ð
 ô
ô 
ð ˆLàˆ	ØˆØˆØˆ	Øˆ	Ø!Ðð .0ˆàó 	8ˆDÜ˜dŸi™i¬×):Ñ):Ô;Ð;Ð;ØŸ™×9Ñ9Ó;ˆJØ×Ñ  jÐ1Ô2Ø0:Ñ-‰OˆZ˜˜]¨AØ+×:Ñ:×@Ñ@ÓBó 8‘
��dÜ! $¬¯
©
Ô3ØØ $§	¡	¬(Ó 3ò 8�HäÓQ¸×9PÑ9PÔQÔQØ$¨	Ò1à$,˜	Ø 1™˜Ø ’{Ø$žä" 8§=¡=°Ñ#3´U·Z±Z×5GÑ5G×5OÑ5OÕPØ$ŸM™M¨!Ñ,°×0GÑ0GÒGØ Ñ,Üô  à0=×0LÑ0L×0RÑ0RÓ0Tô õ ð
 %ŸM™M¨!Ñ,¨qÓ0à$,§M¡M°!Ñ$4˜	Ø'/§}¡}°QÑ'7˜Ø$(˜	Ø-7Ñ*ò/8ò8ð	8ñD ØˆLð "Ð-Ð-Ð-ÜÐ1Ó2Ðà9Dò 	Ñ5ˆDÑ5‘?�J  M°1Ø × 7Ñ 7Ñ7Ø’Ü�:‹Ð"2Ó2Ø’ð		õ	ð, !%ÐÜ)¬#™Ó0ÐØ(ò 	L‰MˆAˆzÙ)¨:Ð6ÑˆAˆz˜1Ø$Ð,Ø(2×(=Ñ(=Ð%Ø(¨J×,AÑ,AÒAð Ø%Ø×%Ñ%ðDó ÐAð !×'Ñ'¨
×(AÑ(A×(HÑ(HÓ(JÕKð	Lð %Ð0Ð0Ð0à!ÜÐ%Ó&ð&
Ð"ð
 EJÖJ¸D�d˜D×4Ñ4Ó6Ò7ÐJˆ	ÐJØò 	ˆDØ×(Ñ(Ø+EØ*4ð )õ ð	ð �a‘—‘˜qÑ!ˆÜÓ<°%¸¸°)Ô<Ô<Ü'¨	Ó2ò /‘��eØ×'Ñ'¨Õ.ð/ð ˆùò Ks   Ê=Mr  c                 óX  ‡ ‡‡‡	— ‰ j                   Št        j                  }g Šg Š	t        |t        «      sJ ‚dt        fˆˆˆ	ˆ fd„} ||«      sÑ|t        _        ‰j                  «        ‰	j                  «        t        j                  j                  j                  d¬«      5  |j                  «       D ]^  }t        |t        t        f«      sJ ‚|j                  «       }‰ j                  ‰«      }|j                  |«       ‰j!                  ||«       Œ` 	 ddd«       yy# 1 sw Y   yxY w)a  
        Generate the code for the outer loop fused scheduler node.
        1. Codegen with fused outer loop: depends on the analysis of
            the outer loop fused scheduler node, with or without the local buffer.
        2. If failed, fallback to standard codegen.
        r  c           	      óÆ  •‡ ‡‡‡‡‡— t        ‰ t        «      sJ ‚‰j                  «        ‰j                  «        dt        fd„Šg }i Št	        ˆˆ fd„‰ j                  «       D «       «      �rt        «       Š‰ j                  «       D �]í  Št        ‰t        «      sJ ‚‰j                  ‰j                  «       «       ‰j                  «       st        ‰j                  «       «      dk7  rŒb‰j                  «       d   Št	        ˆ fd„‰j                  D «       «      sŒ”‰j                  }t        |t         j"                  «      sJ ‚|j%                  «       }‰ j&                  t         ‰‰«      «      z
  }ˆˆfd„}|j)                  «       r |«       s�Œ	g }|j*                  d   } ‰‰«      |d	 }t-        |«      D ]  }	|j/                  d|«       ||	z  }Œ t!        j0                  |j2                  |j4                  ||«      }
ˆˆfd
„}d} ||
|«      }|sEt!        j6                  |› dt        |«      › �|
¬«      }|j9                  |«       g ‰|j:                  <   ‰|j:                     j9                  |«       �Œð t=        ‰j>                  «      5 }t        |«      dkD  r4|D ]/  }|j:                  €J ‚|jA                  |‰|j:                     «       Œ1 ‰ j                  «       D ]z  }t        |tB        t        f«      sJ ‚‰jE                  ‰«      }|jG                  |j                  «       «       ‰j9                  |«       ‰j9                  |j                  «       «       Œ| ‰ jI                  ‰‰ j&                  «      sD|jJ                  D ]+  }tL        jN                  jJ                  jQ                  |«       Œ- 	 d	d	d	«       ytR        jT                  j9                  tS        jV                  t        ‰«      t        |jX                  «      ¬«      «       ‰ j[                  ‰«      }‰j]                  |g t^        j`                  jc                  ‰«      ¢«       d	d	d	«       y# 1 sw Y   yxY w)zN
            Codegen code with fused outer loop and local Buffer.
            r  c                 óÀ   — t        | t        t        f«      sJ ‚| j                  «       }t	        |d„ ¬«      j
                  \  }\  }}t        |«      t        |«      z   }|S )Nc                 ó4   — t        | j                  «       «      S r  )ro   r›  r�  s    rY   rÊ  z~CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.get_call_ranges.<locals>.<lambda>ñ  s   € ¬¨Q¯^©^Ó-=Ó)>€ rX   r]  )r£   r$   r"   r  ra   r4  r¤   )r  r>  rÎ   r4  r5  r’  s         rY   Úget_call_rangeszlCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.get_call_rangesí  s`   € Ü! $¬Ô8JÐ(KÔLÐLÐLØ-1¯^©^Ó-=�Ü.1ØÑ>ô/ç‘%ñ ,�Ñ+�E˜?ô $ E›l¬U°?Ó-CÑC�Ø"Ð"rX   c              3   ó`   •K  — | ]%  }t         ‰|«      «      ‰j                  d z   k(  –— Œ' y­wrÍ  )rÔ   r  )r  r  rá  r  s     €€rY   r  zfCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.<genexpr>ù  s3   øè ø€ ò àô ‘O EÓ*Ó+¨t×/KÑ/KÈaÑ/OÕOñùs   ƒ+.r6   r   c              3   óV   •K  — | ]   }|j                   ‰j                  «       v –— Œ" y ­wr  )r  r  )r  r¸  r  s     €rY   r  zfCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.<genexpr>  s&   øè ø€ ò Ø:>˜Ÿ	™	 T§^¡^Ó%5Ô5ñùó   ƒ&)c                  óB  •‡‡— dŠd} t        ‰j                  j                  j                  «       «      D ]  \  }}‰| |z  z  Š| |z  } Œ ‰j                  j	                  ‰j                  «       «      }ˆfd„Š ‰|«      xr t        ˆˆfd„‰j                  D «       «      S )Nr   r6   c                 ó   •— | ‰k(  S r  rW   )r„  Úcontiguous_index_exprs    €rY   Úis_contiguous_indexz–CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.is_all_write_read_contiguous.<locals>.is_contiguous_index%  s   ø€ Ø'(Ð,AÑ'AÐ ArX   c              3   óÈ   •K  — | ]Y  }t        |j                  t        «      xr9  ‰|j                  j                  j	                  ‰j                  «       «      «      –— Œ[ y ­wr  )r£   r  r$   r8  Úget_read_exprrT  )r  r¸  rè  Úscheduler_buffers     €€rY   r  zŒCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.is_all_write_read_contiguous.<locals>.<genexpr>(  s\   øè ø€ ò Qð %)ô !+¨4¯9©9´mÓ Dò !"Ù$7Ø$(§I¡I§O¡O×$AÑ$AØ(8×(AÑ(AÓ(Có%&ó%"ó!"ñQùs   ƒAA")r3  r8  rÕ  rª  Úget_write_exprrT  r  r»  )r`  r¨   r3  Úwrite_index_exprrç  rè  rë  r¥  s       @@€€rY   Úis_all_write_read_contiguouszyCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.is_all_write_read_contiguous  s·   ú€ Ø45Ð1Ø%&˜FÜ.6à .× 4Ñ 4× ?Ñ ?× EÑ EÓ Gó/ò 0¡
  Uð !6¸À#¹Ñ EÐ 5Ø &¨%¡¡ð0ð 0>×/CÑ/C×/RÑ/RØ 0× 9Ñ 9Ó ;ó0Ð,ôBñ $7Ð7GÓ#Hò $ÌSô Qð -=×,BÑ,BôQó Nð rX   r€  Nc                 ó~   •— |D ]7  }| |j                   k(  sŒt        ˆfd„‰|j                     D «       «      sŒ5|c S  y )Nc              3   óÐ   •K  — | ]]  }|j                   �Ot        ˆfd„t        j                  j                  j
                  |j                      j                  D «       «      –— Œ_ y ­w)Nc              3   óV   •K  — | ]   }|j                   j                  «       ‰v –— Œ" y ­wr  )r  rT  )r  r¸  Úvisited_scheduler_nodess     €rY   r  z�CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.try_share_local_buffer.<locals>.<genexpr>.<genexpr>J  s,   øè ø€ ò (&à,0ð )-¯	©	×(:Ñ(:Ó(<Ð@WÔ(Wñ(&ùrä  )rÈ   r  r5   ræ  r  Úname_to_bufr»  )r  Úglobal_bufferrò  s     €rY   r  z†CppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.try_share_local_buffer.<locals>.<genexpr>I  s`   øè ø€ ò S"ð )6ð (5×'9Ñ'9Ð'Eô %(ó (&ä45·G±G×4EÑ4E×4QÑ4QØ,9×,>Ñ,>ñ5*ç*/©%ô	(&÷ %&ñS"ùs   ƒA#A&)r�  r  rÈ   )Úlocal_buffer_layoutrQ  Ú	local_bufÚlocal_to_global_buffersrò  s      €€rY   Útry_share_local_bufferzsCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_buf.<locals>.try_share_local_bufferG  sS   ø€ Ø-:ò 5 	Ø#6¸)×:JÑ:JÓ#JÌsó S"ð :QØ(1¯©ñ:&ôS"õ P"ð ,5Ò$4ð5ð $(rX   Úlocal_buffer_datarÎ   )rÈ   r�  F)Úlocal_buffer_numberT)2r£   rý   Úclearr   r  r  r   r  r$   rt  rT  r›  rÔ   Úget_outputsr»  r  r   rg  rS  r  Úis_contiguousr`  r3  r½  ÚFixedLayoutrR  r‰   ÚBufferr›  rÈ   rL   ri  Úadd_local_bufferr"   rQ  r1  r@  r  r5   ræ  Úremover   Ú!cpp_outer_loop_fused_inner_countsÚCppOuterLoopFusedCountrQ  rK  Úfinalize_kernelr†  r‡  Úfrom_iterable)r  rQ  rô  Úglobal_buffer_layoutÚsize_offsetrî  Úlocal_buffer_strider`  Úlocal_buffer_sizeÚszrõ  rø  Úlocal_buf_prefixÚlocal_buffer_usedÚscoperU  r  r>  Úremoved_bufferÚouter_fusion_cpp_kernel_proxyrá  r÷  rë  r¥  rò  r<  rC  Ú
nodes_listr  s   `                   @@@@@€€€€rY   Ú$try_outer_loop_fusion_with_local_bufzSCppScheduling.codegen_outer_loop_node.<locals>.try_outer_loop_fusion_with_local_bufå  sO  þ€ ô ˜dÔ$?Ô@Ð@Ð@Ø!×'Ñ'Ô)Ø×ÑÔð#Ô&7ó #ð .0ˆMàBDÐ#Üô à!×1Ñ1Ó3ôõ ô <F»<Ð'Ø&*§n¡nÓ&6ó f�Nä% n´mÔDÐDÐDØ+×/Ñ/°×0GÑ0GÓ0IÔJà&×3Ñ3Ô5Ü˜~×9Ñ9Ó;Ó<ÀÒAà à'5×'AÑ'AÓ'CÀAÑ'FÐ$Üó ØBR×BXÑBXôõ ð )9×(=Ñ(=˜Ü)¨-¼×9JÑ9JÔKÐKÐKØ/<×/GÑ/GÓ/IÐ,Ø&*×&BÑ&BÄSÙ+¨NÓ;óFñ '˜õð8 1×>Ñ>Ô@Ù <Ô >á$à9;Ð+Ø!5×!<Ñ!<¸RÑ!@˜Ù,;¸NÓ,KØ'˜Lð-Ð)ô #+Ð+<Ó"=ò )˜BØ/×6Ñ6°q¸&ÔAØ" b™L™Fð)ô /1¯n©nØ0×7Ñ7Ø0×6Ñ6Ø-Ø/ó	/Ð+õ(ð" ,?Ð(á,BØ/°ó-Ð)ñ  1ä02·	±	Ø(8Ð'9¸¼3¸}Ó;MÐ:NÐ%OØ':ô1Ð-ð *×0Ñ0Ð1BÔCØNPÐ3Ð4E×4JÑ4JÑKà/Ð0A×0FÑ0FÑG×NÑNØ)öðIfôP $ L×$5Ñ$5Ó6ð "¸%Ü�}Ó%¨Ò)Ø(5ò ˜Ø+×0Ñ0Ð<Ð<Ð<Ø×.Ñ.Ø(Ð*AÀ,×BSÑBSÑ*Tõðð
 "×1Ñ1Ó3ò 9�EÜ% eÔ.@Ä-Ð-PÔQÐQÐQØ'+×'<Ñ'<¸\Ó'JÐ$Ø$×2Ñ2°5·?±?Ó3DÔEØ)×0Ñ0Ð1AÔBØ×%Ñ% e§o¡oÓ&7Õ8ð9ð ×>Ñ>Ø)¨4×+GÑ+Gôð +0×*?Ñ*?ò G˜ô Ÿ™×/Ñ/×6Ñ6°~ÕFðGð !÷+"ð "ô, ×9Ñ9×@Ñ@Ü×2Ñ2ÜÐ1Ó2Ü,/°×0CÑ0CÓ,Dôôð 15×0OÑ0OØ)ó1Ð-ð ×,Ñ,Ø1Ø@”i—o‘o×3Ñ3°JÓ?Ð@ô÷?"ðH ÷I"ðH ús   ÊD'QÎ<BQÑQ Frù  N)rC  r   r	  r£   rý   rû  r…   r  r   r  r  r"   r$   r  rQ  r1  r  )
r  r  r	  r  r  Ú_nodesr>  r<  rC  r  s
   `      @@@rY   Úcodegen_outer_loop_nodez%CppScheduling.codegen_outer_loop_nodeÕ  s%  û€ ð ×(Ñ(ˆÜ)0×)OÑ)OÐ&Ø=?ÐØ02ˆ
Ü˜$Ô ;Ô<Ð<Ð<ði	Ô7R÷ i	ð i	ñV 4°DÔ9à5SŒGÔ2Ø!×'Ñ'Ô)Ø×ÑÔô —‘×'Ñ'×-Ñ-¸eÐ-ÓDñ KØ!×1Ñ1Ó3ò K�EÜ% eÔ.@Ä-Ð-PÔQÐQÐQØ27·/±/Ó2C�FØ'+×'<Ñ'<¸\Ó'JÐ$Ø$×2Ñ2°6Ô:Ø ×0Ñ0Ð1AÀ6ÕJñK÷Kð Kð :÷Kð Kús   Â#A2D Ä D)c                 ót  — | j                   }t        |t        «      r| j                  |«       nU|j	                  «       }| j                  |«      }| j                  |«      }|j                  |«       |j                  ||«       | j                  «       }|t        j                  kD  r| j                  d«       yy)zB
        Turn a set of pre-fused nodes into a C++ kernel.
        TN)rC  r£   rý   r  r  rÜ  rQ  r1  r  Ú_get_scheduled_num_argsrP  ÚMAX_FUSED_KERNEL_ARGS_NUMr\  )r  r  rC  r>  r>  Úargs_nums         rY   Úcodegen_nodezCppScheduling.codegen_node   s£   € ð ×(Ñ(ˆä�dÔ7Ô8Ø×(Ñ(¨Õ.à)-¯©Ó)9ˆEØ×'Ñ'¨Ó.ˆEØ#×4Ñ4°\ÓBÐØ×*Ñ*¨5Ô1Ø×(Ñ(Ð)9¸5ÔAà×/Ñ/Ó1ˆØ”m×=Ñ=Ò=Ø×"Ñ" 4Õ(ð >rX   c                 ón   — t        |t        «      xr$ t        |j                  t        j                  «      S r  )r£   r$   r  r   ÚCppTemplateBuffer)r  r  s     rY   Úis_cpp_templatezCppScheduling.is_cpp_template¶  s,   € Ü˜$¤Ó.ò 
´:Ø�I‰I”r×+Ñ+ó4
ð 	
rX   Útemplate_nodeÚepilogue_nodesÚprologue_nodesc                 óì  — |rJ ‚|D �cg c]  }t        |t        t        f«      r|‘Œ }}t        d   dxx   dz  cc<   t        d   dxx   t	        |«      z  cc<   | j                  |«      sJ d«       ‚t        t        |«      }|j                  \  }\  }}|dk(  sJ ‚t        t        j                  |j                  «      }|D �cg c]  }|j                  ‘Œ }	}t        d„ |	D «       «      sJ d«       ‚d	„ }
 |
||j                  |	«      }|j                  |||	¬
«      \  }}|5  t        |j                  «      s|j                  «        |D ]  }|j                  «        Œ  |«       }ddd«       t!        j"                  |«      5  |g|¢}| j%                  ||j&                  «      }ddd«       t        |j                  «      r­t	        |j(                  «      dk(  sJ d«       ‚|j(                  d   j*                  D ]r  }t        |j                  t,        «      sJ d«       ‚t        |j                  j                  t        j.                  «      sJ d«       ‚|j                  j                  «        Œt | j1                  «       |j3                  ||«       t         j4                  xj6                  |j6                  z  c_        | j9                  «        yc c}w c c}w # 1 sw Y   �ŒxxY w# 1 sw Y   �ŒExY w)zG
        Codegen a CPP template, possibly with fused epilogues
        ÚinductorÚcpp_templated_kernel_counterr6   Úcpp_epilogue_fusion_counterzlTemplate node passed to CppScheduler.codegen_template must be a SchedulerNode that wraps a CppTemplateBufferrW   c              3   óP   K  — | ]  }t        |t        j                  «      –— Œ  y ­wr  )r£   r   rg  )r  r  s     rY   r  z1CppScheduling.codegen_template.<locals>.<genexpr>Ø  s   è ø€ ÒO¸”:˜a¤×!2Ñ!2×3ÑOùr‚  z9Epilogue nodes must all be instances of ir.ComputedBufferc                 ó–   ‡— ‰sy| j                  «       |v sJ ‚|| j                  «          j                  }t        ˆfd„|D «       «       S )NFc              3   ó†   •K  — | ]8  }t        |j                  t        «      xr |j                  j                  ‰v –— Œ: y ­wr  )r£   r  r   )r  r¸  r  s     €rY   r  zZCppScheduling.codegen_template.<locals>.template_buffer_has_other_users.<locals>.<genexpr>ä  sA   øè ø€ ò ð ô ˜4Ÿ9™9Ô&7Ó8ò 5Ø—I‘I—N‘N nÐ4ó5ñùs   ƒ>A)rT  r»  r  )Útemplate_bufferÚoutputs_by_namer  r»  s     ` rY   Útemplate_buffer_has_other_userszGCppScheduling.codegen_template.<locals>.template_buffer_has_other_usersÜ  s\   ø€ ñ "Øà"×+Ñ+Ó-°Ñ@Ð@Ð@Ø# O×$<Ñ$<Ó$>Ñ?×EÑEˆEÜó ð "ôó ð ð rX   )Ú$flag_template_buffer_has_other_usersr  NzSMulti outputs template should be with 1 output template buffer of MultiOutputLayoutr   z?Multi outputs template should be with ExternKernelSchedulerNodez7Multi outputs template has multi users with MultiOutput)r£   r$   r"   r   rÔ   r  r   r4  r   r  r  r  r'  Úmake_kernel_renderr*   r(  r5   Úset_kernel_handlerÚdefine_kernelri  Úoutputsr»  r    rŸ  Úcodegen_commentÚcall_kernelræ  r  Úfree_buffers_in_scheduler)r  r  r  r  Úepilogue_noderÎ   ÚrnumelÚctbr  Úepilogue_ir_nodesr(  r)  rG  Úrenderr  Úsrc_codeÚnode_scheduleÚkernel_namer¸  s                      rY   Úcodegen_templatezCppScheduling.codegen_template»  sô  € ñ "Ð!Ð!ð
 "0ö
àÜ˜-¬-Ô9KÐ)LÔMò ð
ˆð 
ô 	�ÑÐ;Ó<ÀÑAÓ<Ü�ÑÐ:Ó;¼sÀ>Ó?RÑRÓ;Ø×#Ñ# MÔ2ð 	
Øzó	
Ð2ô œ]¨MÓ:ˆØ&×,Ñ,‰ˆ‰;ˆAˆvØ˜Š|Ðˆ|Ü$(¬×)=Ñ)=¸}×?QÑ?QÓ$RˆØHVÖ7WÀ1¸¿»Ð7WÐÐ7WÜÑOÐ=NÔOÔOð 	
ØGó	
ÐOò	ñ 0OØ�×.Ñ.Ð0Aó0
Ð,ð ×/Ñ/ØØ1UØ,ð 0ó 
‰ˆ�ð
 ñ 	 Ü,¨]×-?Ñ-?Ô@Ø×&Ñ&Ô(Ø&ò  �Ø—‘•ð á“xˆH÷	 ô ×!Ñ! &Ó)ñ 	SØ*Ð<¨^Ð<ˆMØ×,Ñ,¨X°}ÀfÇkÁkÓRˆK÷	Sô % ]×%7Ñ%7Ô8ô �}×,Ñ,Ó-°Ò2ð ØeóÐ2ð &×-Ñ-¨aÑ0×6Ñ6ò %�Ü! $§)¡)Ô-FÔGð ØUóÐGô " $§)¡)§.¡.´"·.±.ÔAð ØMóÐAð —	‘	×"Ñ"Õ$ð%ð 	×Ñ˜]¨KÔ8Ø×Ñ˜;¨Ô,Ü	�‰×Ò 6×#9Ñ#9Ñ9ÕØ×&Ñ&Õ(ùòQ
ùò  8X÷6	 ñ 	 ú÷	Sñ 	Sús$   ‰KÃKÄ"AKÆ#K)ËK&Ë)K3c                 ó6   — | j                   j                  «       S r  )rC  Úget_num_argsr  s    rY   r  z%CppScheduling._get_scheduled_num_args  s   € Ø× Ñ ×-Ñ-Ó/Ð/rX   c                 ó   — | j                   S r  r[  r  s    rY   Úready_to_flushzCppScheduling.ready_to_flush  s   € Ø×#Ñ#Ð#rX   c                  ó   — y r  rW   r  s    rY   Úcodegen_synczCppScheduling.codegen_sync  s   € ØrX   c                 ó€  — t         j                  j                  }||j                  v r|j                  |   }|S t        j
                  j                  r$t        |t        j
                  j                  «      nd}dj                  d||j                  «       g«      }||j                  |<   t         j                  j                  r|nd}|j                  t        t        j                  «      |«      }|j                  t        t        j                  «      |«      }|j                  dd«      }|j!                  d«      }|j#                  d|«      }	t$        r|j#                  d|	d	z   «      }	|||	d	z    › d
�}
t'        «       }|€| j(                  j*                  n|}|j-                  «       \  }}}t         j                  j                  s|j/                  d|›d�«       |j1                  |d¬«       t         j                  j                  s|j/                  d«       |j3                  ||j5                  «       d|
¬«       |S )NrT   rÎ   r´  rG  z#pragma CMTz//z
extern "C"r—   r6   z;
zasync_compile.cpp_pybinding(z, r'''T)Ústripz''')F)ÚgpuÚcpp_definition)r5   ræ  Úwrapper_codeÚsrc_to_kernelr   r´  Údescriptive_namesr(   r|  Únext_kernel_suffixÚcpp_wrapperrí  rq   r-   ÚKERNEL_NAMEÚDESCRIPTIVE_NAMEÚrfindrÎ  rV   r>   rC  ri  Úcpp_argdefsrÃ   r  r,  Úgetvalue)r  r6  r>  Úkernel_argsr¦  r8  Ú
fused_nameÚkernel_decl_nameÚ
first_charÚ	last_charÚkernel_definitionÚcompile_wrapperri  rÎ   Ú	arg_typess                  rY   r,  zCppScheduling.define_kernel  s  € Ü—'‘'×&Ñ&ˆØ�w×,Ñ,Ñ,Ø!×/Ñ/°Ñ9ˆKðR ÐôK —:‘:×/Ò/ô & e¬V¯Z©Z×-IÑ-IÔJàð ð
 Ÿ(™( E¨:°w×7QÑ7QÓ7SÐ#TÓUˆKØ.9ˆG×!Ñ! (Ñ+Ü./¯g©g×.AÒ.A™{ÀxÐØ×'Ñ'¬¬K×,CÑ,CÓ(DÐFVÓWˆHØ×'Ñ'¬¬K×,HÑ,HÓ(IÈ;ÓWˆHð  ×'Ñ'¨°tÓ<ˆHð "Ÿ™¨Ó5ˆJØ Ÿ™ c¨:Ó6ˆIÝà$ŸM™M¨#¨y¸1©}Ó=�	Ø#+¨J¸ÀQ¹Ð#GÐ"HÈÐ LÐä,Ó.ˆOØ-8Ð-@�4×$Ñ$×)Ò)ÀkˆDØ"×.Ñ.Ó0‰OˆAˆq�)Ü—7‘7×&Ò&Ø×)Ñ)Ø2°9°-¸vÐFôð ×"Ñ" 8°4Ð"Ô8Ü—7‘7×&Ò&Ø×)Ñ)¨&Ô1Ø×!Ñ!ØØ×(Ñ(Ó*ØØ0ð	 "ô ð ÐrX   c                 ó  — | j                   j                  «       }|�rB| j                  || j                   j                  «      }| j	                  | j                   j                  |«       t
        j                  j                  rft        j                  j                  j                  «        t        j                  j                  j                  || j                   j                  «       | j                   j                  t        j                  j                  |«       t
        j                  j                  r(t        j                  j                  j                  «        | j                  «        | j!                  d«       y r�  )rC  Úcodegen_groupr,  Úscheduled_nodesr.  r   r´  Úenable_kernel_profiler5   ræ  rD  Ú write_kernel_context_guard_beginÚwrite_kernel_context_guardr/  Úwrite_kernel_context_guard_endrW  r\  )r  r6  r8  s      rY   ÚflushzCppScheduling.flushH  s  € Ø×$Ñ$×2Ñ2Ó4ˆÚØ×,Ñ,Ø˜$×+Ñ+×;Ñ;óˆKð × Ñ  ×!2Ñ!2×!BÑ!BÀKÔPÜ�z‰z×/Ò/Ü—‘×$Ñ$×EÑEÔGÜ—‘×$Ñ$×?Ñ?ØØ×%Ñ%×5Ñ5ôð ×Ñ×)Ñ)¬!¯'©'×*>Ñ*>ÀÔLÜ�z‰z×/Ò/Ü—‘×$Ñ$×CÑCÔEà×ÑÔ!Ø×Ñ˜uÕ%rX   c                 ót   — t         j                  j                  }t        ||«      }|j	                  ||«       y r  )r5   ræ  rD  r   Úwrite_provenance_debug_handle)r  r7  r8  r¦  Údebug_handles        rY   r.  zCppScheduling.codegen_comment\  s3   € ä—'‘'×&Ñ&ˆÜ>Øàó
ˆð
 	×-Ñ-¨k¸<ÕHrX   r  )6r÷   rø   rù   r5  rQ  r  rû   r  r   r7   ÚINPLACE_BUFFERSÚREDUCE_TO_SINGLE_ELEMENTrT  rL  r…   rR  rU  r  rm   r\  r`  rW  re  rq  r}  r  rH  r�  r†  r—  rš  r   r¢  rƒ  r‚  r¯  r²  r  r$   rÜ  rý   r  r"   r  r  r   r9  r  r=  r?  r,  r]  r.  rM  rN  s   @rY   rP  rP  þ  s”  ø… ð .<Ð�d˜>Ñ*Ó;ð !$ÐÙ!à×*Ñ*Ø×3Ñ3ð	
óÐð ð$¨%¯,©,ð $¸:ÀnÑ;Uò $ó ð$ô%ð&¨ó &òNò*òCò&	ò'òR=ð.Ð/?À$Ñ/Fó ò=/ò~
>ò	<ð
Ø&ð
Ø/@ð
à	ó
ò4)òl

òò=ðO D¨Ñ$7ó OðbIKà)óIKðV)à)Ð,>Ñ>ÀÑNó)ð,
Ð$5ð 
¸$ó 
ð
T)à(ðT)ð !Ð!2Ñ3ðT)ð !Ð!2Ñ3ó	T)òl0ò$òó,ò\&÷(IrX   rP  c                   óD   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Zddefd„Zd„ Z	ˆ xZ
S )	rb  c                 ó  •— t         ‰| �  «        t        «       | _        t	        «       | _        t        | j
                  «      | _        t        j                  «       | _
        | j                  j                  | j                  «       g | _        y r  )r  r  r@   ri  r8   Ú
loops_codeÚWorkSharingrž  rÁ   rÂ   rÍ   rÄ   rX  )r  r  s    €rY   r  zKernelGroup.__init__h  s^   ø€ Ü‰ÑÔÜ“LˆŒ	Ü&›.ˆŒÜ˜dŸo™oÓ.ˆŒÜ×)Ñ)Ó+ˆŒ
Ø�
‰
× Ñ  §¡Ô)Ø!ˆÕrX   c                 ó:   —  || j                   t        «       g|¢­Ž S r  )ri  r,   )r  r  ri  s      rY   ró  zKernelGroup.new_kernelq  s   € Ù�4—9‘9Ô2Ó4Ð<°tÒ<Ð<rX   c                 ó‚   — | xj                   |z  c_         | j                  }| j                  }|j                  ||«       y r  )rX  re  rž  r^  )r  ró  r>  r´   rž  s        rY   r  zKernelGroup.finalize_kernelt  s5   € Ø×Ò Ñ%ÕØ�‰ˆØ�W‰WˆØ× Ñ   rÕ*rX   c                 óX   — | j                   j                  «       \  }}}t        |«      }|S r  )ri  rL  rÔ   )r  Úarg_defsÚ
_call_argsÚ
_arg_typesr  s        rY   r;  zKernelGroup.get_num_argsz  s)   € Ø+/¯9©9×+@Ñ+@Ó+BÑ(ˆ�*˜jÜ�x“=ˆØˆrX   r¹   c                 óà  — | j                   j                  «        | j                  syt        «       }t        j
                  j                  xr t        j                  dv }|r|j                  dg«       |j                  d«       |€t        t        j                  «      n|}|€t        t        j                  «      n|}| j                  j!                  «       \  }}}dj#                  d«      j%                  |«      }t'        «       }t        j
                  j(                  rdnd}	|j                  d|› d	|	› d
|› d|› d�	«       |j+                  «       5  |j                  d«       |j                  d«       |rNt,        j.                  j0                  }
|
�dt        |
«      z   dz   nd}|j                  d||z   › d||z   › d�g«       | j                  j3                  «       D ]  \  }}|j                  d|› d|› d�«       Œ |j5                  | j6                  «       |j                  d«       |j                  d«       d d d «       |j9                  «       S # 1 sw Y   |j9                  «       S xY w)NrT   )ÚlinuxrR   z3#include <torch/csrc/inductor/aoti_runtime/utils.h>z+#include <torch/csrc/inductor/cpp_prefix.h>z,
é   ÚC10_ALWAYS_INLINE_ATTRIBUTEzextern "C" z void r™   ro  r—   z3std::atomic<int> inductor_cpu_integer_div_error{0};zFinductor_cpu_integer_div_error_flag = &inductor_cpu_integer_div_error;Úgraph_rÎ   z9torch::aot_inductor::RAIIAtenRecordFunctionHandle record_z_("z", nullptr);rÖ   r»   r¼   z.inductor_cpu_integer_div_error_flag = nullptr;zHinductor_cpu_throw_if_integer_div_error(inductor_cpu_integer_div_error);)rÍ   r>  rX  r8   r   r´  rY  ÚsysÚplatformrß   rÃ   rq   r-   rI  rJ  ri  rL  Úljustr|  rZ   Úforce_inline_kernelrÅ   r5   ræ  Úgraph_idÚaliasesr  re  rM  )r  rÈ   r´   rY  rP  r8  rj  rÎ   Úfunc_export_declÚinline_attrrv  r:  ÚoldÚnews                 rY   rW  zKernelGroup.codegen_group  sF  € Ø�
‰
×ÑÔØ×#Ò#ØÜ‹~ˆô !'§
¡
× @Ñ @ò !
ÄSÇ\Á\ð V
ð F
Ðñ !Ø�O‰OÐRÐSÔTØ�‰ÐDÔEð <@¸<œ3œ{×6Ñ6Ô7ÈTÐØ;?¸<”cœ+×6Ñ6Ô7ÈTˆØŸ™×.Ñ.Ó0‰ˆ�!�QØ—;‘;˜r“?×'Ñ'¨Ó1ˆÜ1Ó3Ðä-3¯Z©Z×-KÒ-KÑ)ÐQSð 	ð 	�‰ØÐ*Ð+¨6°+°¸aÐ@PÐ?QÐQRÐS[ÐR\Ð\]Ð^ô	
ð
 �[‰[‹]ñ 	Ø�N‰NÐPÔQØ�N‰NØXôñ %ÜŸ7™7×+Ñ+�Ø;CÐ;O˜¤C¨£MÑ1°CÒ7ÐUW�Ø—‘ð&Ø&,¨{Ñ&:Ð%;¸3¸vÈÑ?SÐ>TÐT`ðbðôð !ŸI™I×-Ñ-Ó/ò 7‘��SØ—‘  s e¨3¨s¨e°1Ð5Õ6ð7à�K‰K˜Ÿ™Ô(Ø�N‰NÐKÔLØ�N‰NØZô÷)	ð. �}‰}‹Ð÷/	ð. �}‰}‹Ðús   ÅC*IÉI-c                 ój   — | j                   j                  «       \  }}}|j                  ||d|¬«       y )NF)ÚtritonrU  )ri  rL  Úgenerate_kernel_call)r  r¦  r8  rÎ   Ú	call_argsrU  s         rY   r/  zKernelGroup.call_kernelµ  s:   € Ø"&§)¡)×"7Ñ"7Ó"9Ñˆˆ9�iØ×$Ñ$ØØØØð	 	%õ 	
rX   r  )r÷   rø   rù   r  ró  r  r;  rq   rW  r/  rM  rN  s   @rY   rb  rb  g  s)   ø„ ô"ò=ò+òñ
4¨#ó 4öl
rX   rb  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zy)rf  c                 ó`   — || _         d| _        d | _        t        j                  «       | _        y r�  )r´   Úin_parallelr¥  rÁ   rÂ   rÍ   )r  r´   s     rY   r  zWorkSharing.__init__À  s)   € ØˆŒ	Ø ˆÔØˆÔÜ×)Ñ)Ó+ˆ�
rX   c                 óV  — | j                   r|| j                  k7  r| j                  «        | j                   sò|| _        d| _         d}t        j                  j
                  dk\  rd}n|t        j                  «       k(  rd}nd}|st        j                  j                  r| j                  j                  d«       n| j                  j                  d|› d�«       | j                  j                  | j                  j                  «       «       | j                  j                  d«       y y )NTFr6   z#pragma omp parallelz!#pragma omp parallel num_threads(r—   zint tid = omp_get_thread_num();)r‚  r¥  r>  r   r´  rD  ÚosÚ	cpu_countr¬  r´   rÃ   rÍ   rÄ   rÅ   )r  rD  Úuse_dynamics      rY   r0  zWorkSharing.parallelÆ  sè   € Ø×Ò ¨4×+;Ñ+;Ò ;à�J‰JŒLØ×ÒØ&ˆDÔØ#ˆDÔàˆKÜ�z‰z×!Ñ! QÒ&à#‘ØœBŸL™L›NÒ*à"‘ð $�áœfŸj™j×8Ò8Ø—	‘	×#Ñ#Ð$:Õ;à—	‘	×#Ñ#Ð&GÈÀyÐPQÐ$RÔSØ�J‰J×$Ñ$ T§Y¡Y×%5Ñ%5Ó%7Ô8Ø�I‰I×ÑØ1õð)  rX   c                 óh   — | j                   r| j                  j                  d«       | j                   S )Nz#pragma omp single)r‚  r´   rÃ   r  s    rY   rO  zWorkSharing.singleâ  s*   € Ø×ÒØ�I‰I×ÑÐ 4Ô5Ø×ÑÐrX   c                 óF   — | j                   j                  «        d| _        y r�  )rÍ   r>  r‚  r  s    rY   r>  zWorkSharing.closeç  s   € Ø�
‰
×ÑÔØ ˆÕrX   c                 ó:   — | j                   j                  «        | S r  )rÍ   rZ  r  s    rY   rZ  zWorkSharing.__enter__ë  s   € Ø�
‰
×ÑÔØˆrX   c                 ó>   — | j                   j                  |||«       y r  )rÍ   r`  r\  s       rY   r`  zWorkSharing.__exit__ï  s   € Ø�
‰
×Ñ˜H g¨vÕ6rX   N)	r÷   rø   rù   r  r0  rO  r>  rZ  r`  rW   rX   rY   rf  rf  ¿  s    „ ò,òò8 ò
!òó7rX   rf  c                   ó”  — e Zd ZU dZej
                  dz  ed<   dZej
                  dz  ed<   ej                  j                  Z
ej
                  ed<   ej                  j                  Zej
                  ed<   ej                  j                  Zej
                  ed<   dZeed<   d	Zeed
<   d	Zeed<   d	Zeed<   d	Zeed<   d„ Zd„ Zd„ Zy)r:  Nr¨   r&  r'  r  r(  r   r0  FÚsimd_ompÚsimd_vecÚ	collapsedr›  c                 ój   — t        j                  «       }|r|j                  «       | _        y d| _        y rB  )r   r‰  rj  Úsimd_nelements)r  rŸ  s     rY   Ú__post_init__zLoopLevel.__post_init__  s-   € ô .9×-EÑ-EÓ-GˆÙAO >×#;Ñ#;Ó#=ˆÕÐUVˆÕrX   c                 ó  — t        j                  |«      }t        | j                  | j                  «      }||_        d|_        t        |j                  |«      |z  |_        | j                  |_	        d|_
        | j                  |_        |S )NTF)r:  r;  r:  r¨   r&  r(  r�  r   r  r0  rŽ  r›  )r  r"  Úsympy_factorr@  s       rY   r
  zLoopLevel.tile  sn   € Ü—}‘} VÓ,ˆÜ˜Ÿ™ 4§9¡9Ó-ˆØ!ˆŒ
ØˆŒÜ" 4§9¡9¨lÓ;¸lÑJˆŒØŸ™ˆŒØˆŒØ ×-Ñ-ˆÔØˆrX   c                 ó   — t        | j                  «      }t        | j                  «      }t        j                  j
                  r||k(  ry | j                  r| j                  dkD  rd| j                  › d�nd}| j                  rFd}| j                  dkD  r|d| j                  › d�z  }| j                  r\|j                  dd|› �«      }nF| j                  rd}n7| j                  rd	|› �}n%| j                  st        j                  «       rd
}nd}t        › d| j                  › d|› �}| j                  › d|› �}| j                   j"                  r%| j                  › dt        | j                   «      › �}n;| j                  › dt        | j                   «      › dt        | j                   «      › d�}d|› d|› d|› d�}| j$                  s|s|gS ||gS )Nr6   zsimd simdlen(z) rT   z#pragma omp forz
 collapse(r—   z for z#pragma omp z#pragma GCC ivdepr™   r¸  r  r·  z+=(z == 0 ? 1 : zfor(r¾   )rF   r'  r&  r   r´  Úno_redundant_loopsrŒ  r�  r0  rí  r�  r›  r   r¹  rK   r¨   r(  r™  rŽ  )	r  Úoffset_exprÚ	size_exprÚsimdÚline1Ú
offset_strrà  Ú	steps_strÚline2s	            rY   rI  zLoopLevel.lines  s½  € Ü! $§+¡+Ó.ˆÜ §	¡	Ó*ˆ	Ü�:‰:×(Ò(¨[¸IÒ-EØð �}Š} ×!4Ñ!4°qÒ!8ð ˜D×/Ñ/Ð0°Ñ3àð 	ð
 �=Š=à%ˆEØ�}‰}˜qÒ Ø˜: d§m¡m _°AÐ6Ñ6�Ø�}Š}ØŸ™ g°°t°f¨~Ó>‘Ø�]Š]Ø‰EØ�]Š]Ø" 4 &Ð)‰EØ×"Ò"¤{×'9Ñ'9Ô';Ø'‰EàˆEÜ"�| 1 T§X¡X J¨a°¨}Ð=ˆ
Ø—h‘h�Z˜q  Ð,ˆØ�:‰:×ÒØŸ8™8˜* B¤{°4·:±:Ó'>Ð&?Ð@‰Ið
 —8‘8�*˜C¤¨D¯J©JÓ 7Ð8ð 9Ü" 4§:¡:Ó.Ð/¨qð2ð ð �z�l " X J¨b°°¸1Ð=ˆØ�>Š>¡Ø�7ˆNØ�uˆ~ÐrX   )r÷   rø   rù   r¨   r:  r’  rû   r&  ÚSÚZeror'  r  ÚOner(  r0  ro   rŒ  rm   r�  rŽ  r›  r‘  r
  rI  rW   rX   rY   r:  r:  ó  s®   … à!€Cˆ�‰�dÑ	Ó!Ø"€Dˆ%�*‰*�tÑ
Ó"ØŸ™Ÿ™€FˆE�J‰JÓ%ð #ŸW™WŸ\™\€J�—
‘
Ó)ØŸ™Ÿ™€Eˆ5�:‰:Ó#Ø€HˆcÓØ€HˆdÓØ€HˆdÓØ€IˆtÓØ€L�$Óò
Wò	ó'rX   r:  c                   ó–   — e Zd ZU dZdZee   dz  ed<   dZe	dz  ed<   e
de	fd„«       Zd„ Zed„ «       Zd„ Zd	„ Zd
e	fd„Zd„ Zdefd„Zy)r2  aV  
    A loop-nest-like structure. It is built with the `build` method
    as a loop nest and then will perform loop-tiling at some depth.

    A typical case is for vectorization, where we typically do loop-tiling
    at the innermost loop level. A more complicated case is when we do
    2D tiling at both the innermost and outer levels.
    Nr.  rG  c                 ó$  — | j                   }| j                  }| j                  }|€J ‚d}t        t	        ||«      «      D ]B  \  }\  }}t        ||«      }|s|g}n|j                  |«       ||k\  sŒ2| j                  |_        ŒD t        |«      }	|	S )z4Build a LoopNest with the given `kernel` as the leafN)	rZ  r9  r“  rÆ   r  r:  r›  r›  r2  )
rG  rZ  r9  r“  r.  Úloop_idxr¨   r&  r@  r4  s
             rY   r]  zLoopNest.buildW  sŸ   € ð —?‘?ˆØ—‘ˆØ ×0Ñ0ˆØÐ*Ð*Ð*à(,ˆÜ%.¬s°8¸VÓ/DÓ%Eò 	8Ñ!ˆH‘k�s˜DÜ˜S $Ó'ˆDÙØ˜‘à—‘˜TÔ"Ø˜?Ó*Ø$*×$7Ñ$7�Õ!ð	8ô ˜U“Oˆ	ØÐrX   c                 ó,   — t        | j                  «      S r  )rm   r.  r  s    rY   Ú__bool__zLoopNest.__bool__l  s   € Ü�D—J‘JÓÐrX   c                 óä  — | j                   €t        dd¬«      S d}d}| j                   d   j                  }t        j                  d«      }| j                   D ];  }|j                  |k7  r n*|t        |j                  |j                  «      z  }|dz  }Œ= d„ } || j                   «      }dt        fd„}|t        | j                   «      k  �rt        |t        j                  «      rýt        | j                   |   j                  t        j                  «      rÌ|dz  t        | j                   |   j                  | j                   |   j                  «      k  rŒ|�&||kD  r!| j                   |   j                  r || «      sd|}d}| j                   |   j                  }t        |t        | j                   «      «      D ]%  }	| j                   |	   j                  |k7  r n|dz  }Œ' t        ||¬«      S )a‰  
        Maximal allowed depth for parallelism: All reduction or non-reduction levels.
        When the range of the first inner loop beyond the maximum parallel depth is much
        larger than the range of all outer loops within the maximum parallel depth,
        change the starting depth of parallelism to the first inner loop and recalculate
        the maximum parallel depth.
        r   rg  r6   c                 óJ   — t        | «      D ]  \  }}|j                  sŒ|c S  y r  )rÆ   r�  )r.  rç   r@  s      rY   Úget_simd_vec_depthz7LoopNest.max_parallel_depth.<locals>.get_simd_vec_depth…  s+   € ä$ UÓ+ò ‘��4Ø—=“=Ø’Hðð rX   r4  c                 ó†   — t        | j                  t        «      sJ ‚t        d„ | j                  j                  D «       «      S )Nc              3   ó>   K  — | ]  }t        |t        «       –— Œ y ­wr  )r£   rœ  )r  rG  s     rY   r  zILoopNest.max_parallel_depth.<locals>.has_scalar_kernel.<locals>.<genexpr>�  s"   è ø€ ò àô ˜v¤|Ó4Ô4ñùr  )r£   rG  r5  rh   r   )r4  s    rY   Úhas_scalar_kernelz6LoopNest.max_parallel_depth.<locals>.has_scalar_kernelŽ  s>   € Ü˜i×.Ñ.´Ô?Ð?Ð?Üñ à'×.Ñ.×6Ñ6ôó ð rX   r1  )r.  rô   r›  r:  r;  r   r&  r(  r2  rÔ   r£   r3  )
r  rö   Ú	max_depthr›  Ú	num_stepsr@  r§  Úsimd_vec_depthrª  rç   s
             rY   rM  zLoopNest.max_parallel_deptho  s¸  € ð �:‰:ÐÜ °¸qÔAÐAàˆØˆ	Ø—z‘z !‘}×1Ñ1ˆÜ—M‘M !Ó$ˆ	Ø—J‘Jò 	ˆDØ× Ñ  LÒ0ÙØ!¤H¨T¯Y©Y¸¿
¹
Ó$CÑCˆIØ˜‰N‰Ið		ò	ñ ,¨D¯J©JÓ7ˆð	¬ó 	ð œ˜DŸJ™J›Ó'Ü˜9¤e§m¡mÔ4Ü˜4Ÿ:™: iÑ0×5Ñ5´u·}±}ÔEØ˜C‘Ü�t—z‘z )Ñ,×1Ñ1°4·:±:¸iÑ3H×3NÑ3NÓOòPð Ð*Ø Ò.Ø—J‘J˜yÑ)×6Ò6Ù% dÔ+ð $ˆKØˆIØŸ:™: kÑ2×?Ñ?ˆLÜ˜;¬¨D¯J©J«Ó8ò �Ø—:‘:˜a‘=×-Ñ-°Ò=ÙØ˜Q‘‘	ðô ¨IÀ;ÔOÐOrX   c                 óÚ  — |j                   | j                  «       j                   k  sJ d«       ‚| j                  €J ‚t        | j                  «      |j                   k\  sJ ‚| j                  |j                     }|j                   |_        |j                  rt        xj                  dz  c_        t        |j                  dz   |j                   «      D ]  }d| j                  |   _
        Œ y )Nz?Parallel depth cannot exceed the maximal allowed parallel depthr6   T)rõ   rM  r.  rÔ   rö   r0  r›  r   Úparallel_reduction_countr3  rŽ  )r  r2  r@  rç   s       rY   rN  zLoopNest.mark_parallel®  sÏ   € Ø×'Ñ'¨4×+BÑ+BÓ+D×+SÑ+SÒSð 	
ØMó	
ÐSð �z‰zÐ%Ð%Ð%Ü�4—:‘:‹ )×":Ñ":Ò:Ð:Ð:Ø�z‰z˜)×/Ñ/Ñ0ˆØ!×0Ñ0ˆŒØ×Òä×,Ò,°Ñ1Õ,Ü�y×,Ñ,¨qÑ0°)×2JÑ2JÓKò 	+ˆAØ&*ˆD�J‰J�q‰MÕ#ñ	+rX   c                 ó’   — | j                   sJ ‚| j                   |   j                  |«      | j                   |<   | j                   |   S )zÝ
        Do loop-tiling at the `depth` level with `factor`.
            for (x0 = 0; x0 < x0_end; x0++)
            ->
            for (x0 = 0; x0 < x0_end; x0 += factor)
        See details in Note [tiled_size].
        )r.  r
  )r  r<  r"  s      rY   r
  zLoopNest.tile¼  sA   € ð �zŠzÐˆzØ ŸJ™J uÑ-×2Ñ2°6Ó:ˆ�
‰
�5ÑØ�z‰z˜%Ñ Ð rX   r¹   c                 ó6   — | j                   sJ ‚| j                   S r  ©rG  r  s    rY   r4  zLoopNest.get_kernelÈ  s   € Ø�{Š{Ðˆ{Ø�{‰{ÐrX   c                 ó   — || _         y r  r²  r4  s     rY   r  zLoopNest.set_kernelÌ  s	   € Øˆ�rX   Úlevelc                 óÐ   — | j                   sJ ‚t        | j                   «      |k\  sJ ‚|t        | j                   «      k(  rd n| j                   |d  }t        || j                  «      S r  )r.  rÔ   r2  rG  )r  r´  r.  s      rY   rE  zLoopNest.from_loop_levelÏ  sV   € Ø�zŠzÐˆzÜ�4—:‘:‹ %Ò'Ð'Ð'Ø¤ T§Z¡Z£Ò0‘°d·j±jÀÀÐ6HˆÜ˜˜tŸ{™{Ó+Ð+rX   )r÷   rø   rù   rú   r.  r  r:  rû   rG  rŠ  r‘  r]  r¤  r&   rM  rN  r
  r4  r  ro   rE  rW   rX   rY   r2  r2  I  s‡   … ñð %)€Eˆ4�	‰?˜TÑ!Ó(Ø#€FˆI˜ÑÓ#àð�iò ó ðò( ð ñ<Pó ð<Pò|+ò
!ð˜Ió òð, Sô ,rX   r2  )NNN)¬rÁ   rO  r5  r†  r“  r7  r„  rÊ   rr  rA  Úcollections.abcr   r   Úenumr   Útypingr   r   r   r	   r:  r…   Útorch.fxÚtorch._inductorr
   Útorch._prims_commonr   r   Útorch.utils._ordered_setr   Útorch.utils._sympy.functionsr   r   r   Útorch.utils._sympy.symbolr   r   r   Ú_dynamo.utilsr   rT   r   r   r   r   r   Údebugr   rV  r   r  r   r   r    r!   r"   r#   r$   rý  r%   Úutilsr&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   Úvirtualizedr2   r3   r4   r5   Úcommonr7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   Ú	cpp_utilsrC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rs  rV   ÚcacherZ   Ú_loggingÚgetArtifactLoggerr÷   Úschedule_logÚNATIVE_OMP_RTYPESÚRTYPE_TO_CPPrÔ  ÚPYTHON_TO_CPPÚCONTAINER_PYTHON_TO_CPPrB  rC  r„   rA  rp   rm   rD  rE  r  rš  rF  rG  rx   r  r‰   rû   rŽ   r’   r’  r±   r¥   rq   rÑ   ro   rã   rè   rò   Ú	dataclassrô   rý   rP  rk  rm  Ú_initialize_pointwise_overridesr•  r�  r„  rŠ  rœ  r†  r¤   rG  rI  rR  r5  rD  rH  rP  rb  rf  r:  r2  rW   rX   rY   ú<module>rÏ     s’  ðä Û Û Û Û Û Û 	Û 	Û 
Û ß .Ý ß 0Ó 0ã ã Û Ý (ß @Ý /ß KÑ Kß OÑ Oå %ß <Õ <Ý ;Ý  ÷÷ ñ õ +÷÷ ÷ ó ÷ >Ó =÷÷ ÷ ó ÷÷ ÷ ÷ ñ ð& �l‰l˜gÑ%€ð ‡�ñ:ó ð:ð �~‰~×/Ñ/°¸*ÓE€áÒBÓCÐ àØØØØØØØØØ ñ€ñ !òóÐ ð  ØØØØØ#Ø&ØØØñ€ð ØñÐ ð 
‡N�NØ	‡M�Mð€ð 
‡M�MØ	‡K�KØ	‡N�NØ	‡M�MØ	‡J�JØ	‡K�KØ	‡J�JØ	‡K‚KØ	‡K‚KØ	×ÒØ	×Òð*Ð �T˜%Ÿ+š+Ñ&ó ò)òDð Ø-1Øñ2)ð
 �:Š:˜Ñ# dÑ*ó2)òjð-Ø
ð-à�jŠjð-ð ð-ð —
’
ð	-ð
 �jŠjð-ð ó-ð`Ø�;Ñðàðð ðð �;Š;ð	ð
 
ˆs‰óðBV˜^ð V°3ð VÀ#ó Vð/¨ó /ð2 ×Ò÷ð ó ðô^!Ð"4ô ^!÷B!ñ !òB&ôM"�;ô M"ð` × ,Ò ,¨UÔ 3ôo7�lô o7ðd × /Ò /°Ô 9Ø × %Ò %Ô 'ô:˜ô :ôg�ô gôTS�9ô Sôlb.�lô b.ðJ*$ ð *$¨e°E·K²KÀ$Ñ4FÈÐ4LÑ.Mó *$ðZ ×Ò÷ð ó ð÷kKñ kKô\L

�Yô L

ô^ 
˜9ô  
ôFF�tô FôfI�Nô fI÷RU
ñ U
÷p17ñ 17ðh ×Ò÷Rð Ró ðRðj ×Ò÷I,ð I,ó ñI,rX   