^jx@ddlZddlZddlZddlmZddlmZmZmZm Z m Z m Z ddl m Z ddlmZddlmZddlmZddlZddlZddlZddlmZdd lmZdd lmZdd lmZmZm Z m!Z!dd l"m#Z$dd l%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,de-e.ej^j`jbedeffdejdjfde4e5ej^j`jbfde6fdZ7ddddddedede e8e8fde6de6dejdjfdzde&e-e8e8fdzde6fd Z9dejdjfd!eejdjtd"ed#ede e-ejdjte-edfdzff d$Z;e d%Gd&d'Z<Gd(d)Z=d*ej|de8fd+Z?dejdjfde8dzfd,Z@d-ed.ejdjtdzdefd/ZA d4d-ejdjfd.ejdjtdzde-e6e-edfe4e5efffd0ZBdejdjfde6fd1ZCdejdjfde8dzfd2ZDdejdjfde6fd3ZEy)5N) defaultdict)Callable Collection Container GeneratorIterableMapping) dataclass)partial)chain)Any)enable_python_dispatcher) control_deps)FakeTensorMode)compute_unbacked_bindingsrebind_unbackedstatically_known_truesym_eq)_pytree) OrderedSet)tree_map) flop_registry)Vpattern.nodemodulesreturnct|jdk(ryt|jdtjj r$t|tjj sy|jdj dk7ryt|jdjtsy|jdj|vryt||jdj|dury|j dk7r|j dk7ry|j|dk7ryt|jdjdkDryy)NrF call_module call_function call_methodrT) lenargs isinstancetorchfxNodeoptargetstrtypeusers)rrrs c/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_inductor/fx_utils.pymatches_module_function_patternr/*s   499~ diilEHHMM 2* ehhmm; yy|-' diil))3 / yy|') GDIIaL'' (); ww/!dgg&> {{gaj  499Q<  " T check_strides check_storager recursive_idsnewoldexisting_storagesr2r3r4cd}t|t|uryt|tjs'||duSt|trv t t |t |f}|v} j|| xs;t|t|k(xr"tfdt||DSt|tjjrr|jjjt!j"|jj$|jj$t j&k(S||k(S|j(|j(k7s||j*|j*sy|j,|j,k7ryrC|j(tj.k(r&||j1|j1sysyt3|j5|j5k(rt7|t7|k7ryd} t7|dk(rt7|vr | syy)aValidate that two FakeTensors (or iterables thereof) are the same, including storage locations if enabled. check_strides: disabling this flag will remove checks for striding. check_storage: disabling this flag will remove checks for storage offset and location. This is useful for subgraph argument and output updating, where storage location can change without invalidating the subgraph. recursive_ids: This is only for use when recursing through collections, and must not be supplied by users of this function.c,tt||SN)rr)r5r6s r.is_intlist_samez-_is_fake_tensor_same..is_intlist_same]s$VC%566r0FNc 3HK|]\}}t||yw)r1N)_is_fake_tensor_same).0new_iold_ir3r2r7rr4s r. z'_is_fake_tensor_same..ts> %u))&3&3!&3 s"Tc t|jdtjsy|jD]}t|j tj js<|j tjjjjusyt|j tj jrt|\}}}|sytj5t!5t#j$5}tjj&x}|j)|j+|j |i|}dddddddddttjsyt-|t-|jdk(syy#1swY_xYw#1swYcxYw#1swYgxYw)NvalTF)r%metar&Tensorr-r*_ops OperatorBase _inductor fx_passes reinplace_generalized_scatterHigherOrderOperatorget_fake_args_kwargsr fake_moder contextlib ExitStack shape_env enter_contextignore_fresh_unbacked_symbols get_storage)ruseris_validr$kwargsstackrQnew_fake_tensors r.any_user_may_aliasz0_is_fake_tensor_same..any_user_may_aliass$))E*ELL9JJ% D4;; (?(?@;;??,,66KKL$++uzz'E'EF &:$%? "HdF  ?(* ?$$& ?+0"#!6!66IC'' (O(O(QR"-$++t">v"> ? ? ?ou||<?+{499U;K/LLK% N% ? ? ? ? ? ?s=5 GG AG #G +GG G  G GG" r)r,r%r&rErridaddr#allziptypes py_sym_typesrrQ_maybe_evaluate_staticsympyEqexprtruelayoutshapedevicestridedstriderstorage_offsetrT) r5r6r7r2r3rr4r;id_pairvisitedrZs ````` r.r=r=Is(7 CyS ! c5<< ( ;$;  c: &$ * #w3(G.G   g & CCH$  ),C     c5;;33 4""99HHSXX]]CHHMM::: cz zzSZZsyy#))'L zzSZZ  JJ%-- ' cjjl;    2 2 44  S [- -,d +c*+q0  $5 5"4( r0valid_subgraphsr$rWc /6Kdtjjdttjdffd}dtj dt fd}|jtjjjur(|dgd|d Dd |d D|d fy|jtjjjur!t|d }|d |f|d |fy|jtjjjur|d }||}|d d} || } ttj dt|d d| dd D} |dj |dj k(rFt| d k(r8|| j!r"t#dt|dd| dd DsJd|g|dd|df|d dg| dd |dfy|jtjjj$ur1|d} || } |d}||}|dd} || } ttj dt| d d|d d| dd D} | dj |dj k(ri|dj |dj k(rJt| d k(r<|| j!r&t#dt| dd|dd| dd DsJd| vr| g| dd|df|vr|g|dd|df| vr| g| dd |dfyy|jtjjj&tjjj(tjjj*fvr|dt|d dfy|jtjjj,ur|dt|d dfy|jtjjj.ur|dgd|d D|d fy|jtjjj0ur!|dg|d d|d D|d fy|jtjjj2tjjj4fvr g|d |d }|d|f|d |fy|jt6urR|rJd|d }t|d d}|vr||ft9|tjjrt;j<tjjfd |rt?tA|jBjEd!"|d#$|jBjFDcgc]}|jHd%k(s|}}t|d k(sJ|d}d&tJdtJffd' }t;jL||jN|jPf\}}tS|g|i|Ed{yyytUjVd(|jd)t;jX||f\}}fd*|DEd{ycc}w7]7 w)+aAHOPs that invoke subgraphs take a number of different forms. This function regularizes them, yielding subgraphs from the args and kwargs coupled with appropriate args to update those subgraphs. If the second yielded value is None, this function was unable to determine what args to pass to the subgraph.subgraphr.c`tfdjjdDS)Nc36K|]}t|ywr:)get_fake)r>nrps r.rAzI_extract_subgraphs_and_args..get_subgraph_args..s &'HQ ! s placeholderr))tuplegraph find_nodes)rps`r.get_subgraph_argsz6_extract_subgraphs_and_args..get_subgraph_argss1 +3>>+D+D +D+V   r0dc:|j xr |j Sr:)is_floating_point is_complex)r{s r. is_integerz/_extract_subgraphs_and_args..is_integers&&&;q||+;;r0rc3&K|] }|d ywrNr>as r.rAz._extract_subgraphs_and_args..01!A$0rc3&K|] }|d ywrrrs r.rAz._extract_subgraphs_and_args..s3JQAaD3Jrc34K|]}|jywr:dtypers r.rAz._extract_subgraphs_and_args..s5 AGG5 Nc3TK|] }t|jdk("ywrr#sizers r.rAz._extract_subgraphs_and_args..s'AFFH "&(z/flex_attention subgraph arg format has changed! c34K|]}|jywr:rrs r.rAz._extract_subgraphs_and_args..%s5  GG5 rc3TK|] }t|jdk("ywrrrs r.rAz._extract_subgraphs_and_args..2s'AFFH "rz8flex_attention_backward subgraph arg format has changed! c3&K|] }|d ywrrrs r.rAz._extract_subgraphs_and_args..Lrrc3&K|] }|d ywrrrs r.rAz._extract_subgraphs_and_args..Ps#:QAaD#:rzfSubgraph arguments can be renamed, so we cannot consistently handle kwargs at this point in the stack.c |vSr:r)rprns r.z-_extract_subgraphs_and_args..es X8r0rurvTstrictr!itemctt|tjjrj ||S|Sr:)r%r&r'r(get)rplaceholder_to_args r.replace_placeholderz8_extract_subgraphs_and_args..replace_placeholderus-dEHHMM2-11$== r0z1Please add support for subgraph args to function !c3vK|]0}t|tjjr |vr|df2ywr:)r%r&r' GraphModule)r>srns r.rAz._extract_subgraphs_and_args..s8 !UXX112qO7KI s69)-r&r'rrwrErboolr*ops higher_orderassociative_scancondflex_attentionrr r#popr]flex_attention_backward foreach_mapinvoke_quant_packed invoke_quantinvoke_subgraphmap_implscan while_loopwhile_loop_stack_outputrr%pytree tree_any_onlydictr^rxrynodesr)r rr$rW_extract_subgraphs_and_argswarningswarn tree_flatten)rrnr$rWrzr subgraph_argsscore_subgraphscore_subgraph_args mask_subgraphmask_subgraph_argsinteger_arg_dtypes fw_subgraphfw_subgraph_argsjoint_subgraphjoint_subgraph_argscontrol_deps_subgraphcontrol_deps_argsrt wrapped_nodes wrapped_noder wrapped_argswrapped_kwargsflat_args_kwargs_rs ` @r.rrs ((&& u||S ! /3>d1g>>>  ..FF F1g ,[9a/?Q .}=' 45  1%#Aa("2A&5   Q  % %a 6#A&,,Q =&'1,-1134$Ra('+&r* F F F  / )A!1"1!5ARAA A _ , "G$7$;"Gd2h"GG G O +!E#5bq#9!EDH!EE E ,  ** 22 ++  1guT!"X&&  ..>> >1guT!"X&&  ..77 71g;0Q0;47;;;  ..33 31gEaE#:$q'#:ET!WEEE  )) 66 -$q',DG, 1g}$$1g}$$  $  8 z!%Q!$qr(O O 3'):: :  !588#7#7 "" HH 8  "&)//::m:L%" 166<<@WM}%* **(+L # #  ,2??#""L$7$78, (L.3o0<@N  7  B  ? }A N %114.A! %   5  s8X/\3\\ A+\7\8A\ \ \\)frozenceZdZUejj ed<ejjjed<e e dfed<e e dfed<y) _FxNodeHashrr*. arg_hashes kwarg_hashesN) __name__ __module__ __qualname__r&r'r(__annotations__rTargetrwintrr0r.rrs@ ((-- HHMM  c3hS/!r0rceZdZdZdej j ddfdZdej jde fdZ de fdZ y) FakeTensorUpdatera` The main idea here is that it's difficult to maintain accurate fake tensors (our primary form of metadata) for each node in our graph as we transform it. The most reliable way to obtain this information is by rerunning faketensor propagation. However, in general, faketensor propagation is fairly expensive. So, instead we'd like to only rerun faketensor propagation on nodes that have changed. In order to detect which nodes have changed, we first hash its node, target, and argument lists (which are immutable in FX). Then, whenever we call incremental_update, we check which FX nodes have a new hash, and recompute the faketensor metadata for that node. Then, we continue to recursively compute the faketensors for all users until the fake tensors stop changing. Since this runs in the context of Inductor, we assume that the input and output semantics for the outermost graph are not subject to change after class initialization, but we allow striding changes for subgraphs. Any other changes will result in errors or undefined behavior. gmrNc|tt|_||_ddlm}||jDcic]$}t |j|x}t|&c}|_|jjjD],}|jj|j|.ycc}w)Nr)_get_subgraph_names) rrprocessed_hashesrtorch._inductor.compile_fxrgetattrrsubgraph_updatersrxrr\ hash_node)selfrr subgraph_namerprs r.__init__zFakeTensorUpdater.__init__s *; 7 9 C"5TWW!=Q !-8 8X;LX;V VQ  GGMM'' .get_hash_or_ids..get_hash_or_ids(!7N !a5L!s $$c3.K|] }|ywr:r)r>rtrs r.rAzGFakeTensorUpdater.hash_node..get_hash_or_ids..s;q*;)objectrrw)rrs @r.get_hash_or_idsz4FakeTensorUpdater.hash_node..get_hash_or_idss)  !& !S ! ;F;; ;r0) rr rwrrr*r$r from_iterablerWitems)rrrs r.rzFakeTensorUpdater.hash_nodesf .should_process_nodes-%L  -JKK  r0r$rWc:|jdk(xr|jtjjj tjjj fvxr4tjtjjfd||fS)Nr!c |jvSr:)r)rrs r.rzUFakeTensorUpdater.incremental_update..node_invokes_subgraph..sa4#9#99r0) r)r*r&rrauto_functionalizedauto_functionalized_v2rrr'r)rr$rWrs r.node_invokes_subgraphzCFakeTensorUpdater.incremental_update..node_invokes_subgraphs?* KK II**>>II**AA  ((HH((96N r0rFNrurvTr)r3z*subgraph args must have consistent values!)r2r3ziA subgraph argument other than striding has been modified; FakeTensorUpdater cannot update this argument!rCrunbacked_bindingsc32K|]}t|ywr:)r[)r>rUs r.rAz7FakeTensorUpdater.incremental_update..s>4bh>s)$rrrrxrget_node_storager&r'r(rr rrr\rrMrr[rrr^ryr=rsrD output_nodeincremental_updaterrNrr*rrQrupdater-)rr7rstoragerrcurrent_graph_hashes nodes_updated to_processsubgraph_updatingsrrVr$rWinvokes_subgraphany_output_updatedrprupdate_subgraphprp_fakerorig_output_argsnew_output_argsrYrQsymbol_to_paths` r.r z$FakeTensorUpdater.incremental_updatesV4?s3CGGMM'' 0D*400w0!'*a/* 0 ehhmm   ((-- (+ 7:   0 *+68 _& ?AGGMM''| ?D $ $T^^D-A%AT B%9$%H "HdF*?t)Ud)Uf)UU%UD111tHJ.&t,%*"/J$000370;A0C+Hm'/6H&HO&7<*84 %0$'$NN555G)#'%!3DAq$8 !*21h*? ? 1.3 $ (7!"$P!" (<$%$*$52727 (" !"%0 !"("12u - 2 C!3F'1E$NN668(2.+Q&)?)?$*,,./ 1E$NN668(1-?A 4+,-  <@.x8+J.@.J'CCT*etyy.@ ?68 ?"-$++t">v"> ? ? !&:5!13D4' AKK114 I.DIIe  Q M[[222 2";I"WWW2@ -.*400w0!'*a/*   >4::> >y| ?|!57 ? ? ? ?s$ O(O&O(O% !O((O1 ) rrr__doc__r&r'rrr(rrrr rr0r.rrsM0 <588// .s$ )* 1ehhmm$ s.0FT)rr rsr$rWanyrarg_tree_leaves)r#rr$rWs r.rMrMsiGH4qvvqxx6HILD&  .4.D.Dd.Uf.U dF"" v r0cddlmmdtjj dt ffd |rydtjj dt ffd tfd|jDryy ) zReturns true if a node is always realized when lowered to inductor IR. NOTE: This may return some false negatives. e.g. it doesn't handle buffers realized heuristically during lowering, or buffers realized indirectly through view ops. r) fallbacksneeds_realized_inputsrrc|jdk(r1|jtjur|jdS|jdvxs|jvS)Nr!r)ruoutput)r)r*operatorgetitemr$)rr, is_buffers r.r2z#is_node_realized..is_buffersS 77o %$++9I9I*I TYYq\* *ww33Ot{{i7OOr0TcB|jdk(xs|jvS)Nr/)r)r*)rr-s r.realizes_inputsz)is_node_realized..realizes_inputss!ww("Jdkk5J&JJr0c3.K|] }|ywr:r)r>rUr4s r.rAz#is_node_realized..s 8T?4 8rF) torch._inductor.loweringr,r-r&r'r(rr)r-)rr,r2r-r4s @@@@r.is_node_realizedr7smJP P$PKehhmmKK 8TZZ 88 r0c t|rt|jtryt d5t |\}}}|r|jt jjjt jjjfvrOtj|j}|.||i|d|jjdicdddSt jjj!d5}|j|i|dddj#}|cdddS dddy#1swY.xYw#1swYyxYw)NT)allow_non_fake_inputsout_valrCF)display) countable_fxr%r*r+rrMr&rrrrrrrDutils flop_counterFlopCounterModeget_total_flops)rsuccessr$rW flop_formulaflop_counter_mode counted_flopss r.count_flops_fxrEs=  DKK!= d 3! 4T :v {{ &&55 &&>> -00= +'VVuAUV!!))99: -" T,V, - .==?M )!! !*  - -!* s*B,E)*EE&EE EE(ct|tjjsJt |dsy|j }t |ds|t vS|j}|t vS)z> Whether or not we can count the flops of an FX node. r*Foverloadpacket)r%r&r'r(rr*rrG)rr*packets r.r<r<s^ dEHHMM ** * 4 " [[F 6+ ,&&  " "F ] ""r0r:)FrOr0r collectionsrcollections.abcrrrrrr dataclassesr functoolsr itertoolsr typingr rbr&torch.fxtorch._dispatch.pythonr.torch._inductor.fx_passes.control_dependenciesrtorch._subclasses.fake_tensorr%torch.fx.experimental.symbolic_shapesrrrr torch.utilsrrtorch.utils._ordered_setrtorch.utils._pytreertorch.utils.flop_counterr virtualizedrrwr,nnrModuler'r(rr+rr/rr=rrrrrErTr rsrMr7rEr<rr0r.r[s#" ;G8 */(2  4((//0(382DD E ((--#uxx''... / H!%8<S S SsCx(S  S  S ((--$ SeCHo.5S Sln ((--n uxx334n  n  n uUXX))5c?T+AABC n b $"""nnb&5<<&C&)588==)S4Z)  --4  &9=  xx}} ((..5  4sCx$sCx. 01 588==T@3:6 #uxx}} # #r0