
    ^j͘                     d   d Z ddlZddlZddlZddlmZ ddlmZmZ ddl	m
Z
 ddlmZ ddlZddlZ	 ddlZddlmZ 	 ddlZddlmZ 	 dd	lmZ dd
lmZ ddlmZ  e       Z	 	 	 	 ddddeee   z  dej<                  j>                  dee   dej@                  eej@                     z  dee   deeee!ef   f   dz  de"de"de!dedz  de#defdZ$	 d dedee   dej@                  eej@                     z  de"def
dZ% G d d      Z&y# e$ r dZdZY w xY w# e$ r dZdZY w xY w# e$ r dZY w xY w)!z/ONNX export, simplification, and OnnxOptimizer.    N)OrderedDict)MappingSequence)deepcopy)PathLike)shape_inference)G_LOGGER)fold_constants)CustomOpSymbolicRegistry)
get_logger)notes
output_dirmodelinput_namesinput_tensorsoutput_namesdynamic_axesbackbone_onlyverboseopset_versionvariant_namer   returnc
                   |	rIt         j                  j                  t         j                  j                  |	            d   }	|r|	 dn|	}n|rdnd}t         j                  j	                  | | d      }t        |d      r|j                          i }dt        j                  t        j                  j                        j                  v rd|d<   t        j                  j                  |||f||d	dd	|||d
| |
t        t        j                  |      }t        |
t              r|
nt        j                   |
d      }t#        d |j$                  D        d      }|||_        n(|j$                  j)                         }d|_        ||_        t        j,                  ||       t.        j1                  d|        |S )a"  Export a model to ONNX.

    Args:
        output_dir: Directory where the ONNX file will be written.
        model: Model to export.
        input_names: Names of model inputs in ONNX graph.
        input_tensors: Example model input tensor(s) for tracing.
        output_names: Names of model outputs in ONNX graph.
        dynamic_axes: Optional dynamic axis configuration for ONNX export.
        backbone_only: Whether to export backbone-only graph naming.
        verbose: Whether ONNX exporter should emit verbose logs.
        opset_version: ONNX opset version.
        variant_name: Model variant identifier (e.g. ``"rfdetr-medium"``).
            When provided, the exported file is named ``{variant_name}.onnx`` or ``{variant_name}-backbone.onnx`` (when
            ``backbone_only=True``) instead of the generic ``inference_model.onnx`` or ``backbone_model.onnx``.
        notes: Optional user-defined metadata (string, dict, list, or any
            JSON-serialisable value) to embed in the exported ONNX model under the ``"rfdetr_notes"`` metadata property.
            Ignored when ``None``. String values are stored verbatim; all other types are JSON-encoded, so consumers
            must call ``json.loads()`` to recover a dict or list.

    Returns:
        Path to the exported ONNX model.
    r   z	-backbonebackbone_modelinference_model.onnxexportdynamoFT)r   r   export_paramskeep_initializers_as_inputsdo_constant_foldingr   r   r   N)	allow_nanc              3   @   K   | ]  }|j                   d k(  s|  yw)rfdetr_notesN)key).0ps     g/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/rfdetr/export/_onnx/exporter.py	<genexpr>zexport_onnx.<locals>.<genexpr>   s     Yq.AXYs   r$   z#
Successfully exported ONNX model: )ospathsplitextbasenamejoinhasattrr   inspect	signaturetorchonnx
parametersload
isinstancestrjsondumpsnextmetadata_propsvalueaddr%   saveloggerinfo)r   r   r   r   r   r   r   r   r   r   r   export_nameoutput_fileexport_kwargs
onnx_modelnotes_valueexistingmetas                     r(   export_onnxrH   4   s   J ww''(8(8(FGJ4Ai0|*7&=N'',,zk]%+@AK uhM7$$UZZ%6%67BBB #(h	JJ  !$) #!  T- YY{+
  *%5e4::eW\;]YJ$=$=Y[_`(HN,,002D%DH$DJ		*k*
KK6{mDE    onnx_dirforcec                    ddl }ddl}| j                  dd      }t        j                  j                  |      r|s|S t        |t        j                        r|g}t        j                  d|         t        |       }|j                  d       |j                          |j                  d       |j                  |       t        ||      D 	ci c]2  \  }}	||	j                         j!                         j#                         4 }
}}	|j%                  |d|
d	
      \  }}|r |j&                  ||       nt)        d      t        j                  d|        |S c c}	}w )ad  Optimize and simplify an ONNX graph.

    Args:
        onnx_dir: Path to the source ONNX model.
        input_names: Input names matching the exported graph.
        input_tensors: Input tensor sample(s) used by the simplifier.
        force: Whether to overwrite an existing simplified model.

    Returns:
        Path to the simplified ONNX model.
    r   Nr   z	.sim.onnxzstart simplify ONNX model: zModel: originalzModel: optimized   F)check_n
input_datadynamic_input_shapezFailed to simplify ONNX model.z$Successfully simplified ONNX model: )r3   onnxsimreplacer*   r+   isfiler6   r2   Tensorr?   r@   OnnxOptimizer
common_opt	save_onnxzipdetachcpunumpysimplifyr>   RuntimeError)rJ   r   r   rK   r3   rQ   sim_onnx_diroptnametensor
input_dict	model_optcheck_oks                r(   onnx_simplifyre      s1   " ##G[9L	ww~~l#E-.&
KK-hZ89

!CHHNNHH MM,JMk[hJij,$$++-3355jJj!**<zot*uIx		)\*;<<
KK6|nEF ks   7Ec                       e Zd ZddZd ZdefdZdefdZddZddZ	dd	Z
d d
edefdZd d
edefdZddZd Zd Zd Zd Zd Zd!dZd!dZd!dZd!dZd Zd Zd Zd Zd Zy)"rU   Nc                    g }t         |j                  d       t        |j                  d       t        t        |j                  d       t
        |j                  d       |r dj                  |      }t        d| d      |t        j                  }t        |t              r| j                  |      }n|}t        j                  |      | _        || _        | j                  |       y )Nr3   zonnx.shape_inferenceonnx_graphsurgeonz-polygraphy.backend.onnx.loader.fold_constantsz, z&ONNX export dependencies are missing (z)). Install with: pip install rfdetr[onnx])r3   appendr   gsr	   r
   r.   ImportErrorINFOr6   r7   	load_onnximport_onnxgraphseverityset_severity)selfinputrp   missing_depsmissing_str
onnx_graphs         r(   __init__zOnnxOptimizer.__init__   s    <'" 67:) 34! OP))L1K8Eno  }}HeS!.JJ^^J/
 (#rI   c                     |t         _        y N)r	   rp   )rr   rp   s     r(   rq   zOnnxOptimizer.set_severity   s
    $rI   	onnx_pathc                     t         j                  j                  |      s
J d|        t        j                  |      }t        j                  d|        |S )zLoad onnx from file.znot found onnx file: zload onnx file: )r*   r+   rS   r3   r5   r	   r@   rr   rz   rv   s      r(   rm   zOnnxOptimizer.load_onnx   sM    ww~~i(M,A)*MM(YYy)
(45rI   c                     t        j                  | j                        }t        j                  d|        t        j                  ||       y )Nzsave onnx file: )rj   rH   ro   r	   r@   r3   r>   r|   s      r(   rW   zOnnxOptimizer.save_onnx   s6    ^^DJJ/
(45		*i(rI   c                 X   t        j                  | dt        | j                  j                         dt        | j                  j                         j                                dt        | j                  j                         dt        | j                  j                         d
       y )Nz .. z nodes, z
 tensors, z	 inputs, z outputs)	r	   r   lenro   nodestensorskeysinputsoutputs)rr   prefixs     r(   r@   zOnnxOptimizer.info   s    hd3tzz//014::%%',,./0
4::$$%&iDJJ4F4F0G/HR	
rI   c                     | j                   j                         j                          |rt        j                  | j                         S y ry   )ro   cleanuptoposortrj   rH   )rr   return_onnxs     r(   r   zOnnxOptimizer.cleanup   s6    

%%'>>$**-- rI   c                     |D cg c]  }| j                   j                  |    c}| j                   _        |r2t        |      D ]#  \  }}|| j                   j                  |   _        % y y c c}w ry   )ro   r   	enumerater`   )rr   keepnamesoir`   s         r(   select_outputszOnnxOptimizer.select_outputs   se    =ABdjj003B

$U+ 24-1

""1%*2  Cs    A,r`   r   c                     t        |j                        D ]/  \  }}t        |t              r|j                  |k(  r|}(||k(  s.|}1 dk\  sJ d| d| d       |S )Nr   
not found (z) in node.inputs)r   r   r6   r7   r`   rr   noder`   r<   r   inpindexs          r(   find_node_inputzOnnxOptimizer.find_node_input   sl    , 	FAs$$T)9		
 zFZvQug5EFFzrI   c                     t        |j                        D ]/  \  }}t        |t              r|j                  |k(  r|}(||k(  s.|}1 dk\  sJ d| d| d       |S )Nr   r   r   z) in node.outputs)r   r   r6   r7   r`   r   s          r(   find_node_outputzOnnxOptimizer.find_node_output  sl    - 	FAs$$T)9		
 zGZvQug5FGGzrI   c                 t   t         j                  D ]  } ||        | j                           t        t	        j
                  | j                        d      }|j                         dkD  rt        d      t        j                  |      }t	        j                  |      | _        | j                          |r|S y )NF)!allow_onnxruntime_shape_inferencel        z-ERROR: model size exceeds supported 2GB limit)r   
_OPTIMIZERr   r
   rj   rH   ro   ByteSize	TypeErrorr   infer_shapesrn   )rr   r   fnrv   s       r(   rV   zOnnxOptimizer.common_opt  s    *55 	BtHLLN	 $BNN4::$>bgh
 :-KLL(55jAJ^^J/
 rI   c           
         d}| j                   j                  D ]`  }|j                  dk(  st        |j                        dk(  s-|j
                  dz   }|j                         j                         j                  d      }|j                         }t        j                  |dz   t        j                  dg      }t        j                  d	|d
z   |j                  d   g|g      }t        j                  |dz   t        j                  dgt        j                              }t        j                  |dz   t        j                  dgt        j                              }	t        j                  |dz   t        j                  dgt        j                              }
t        j                  |dz   t        j                  dg      }t        j                  d|dz   ||	|
|g|g      }t        j                  |dz   t        j                  dg      }t        j                  d	|dz   |j                  d   g|g      }t        j                  |dz   t        j                  dg      }t        j                  d|dz   |||	|g|g      }t        j                  |dz   t        j                  dg      }t        j                  d|dz   ddi||g|g      }t        j                  j!                         }t        j                  d|dz   |j"                  |j                  d   |||g|j                  d   g      }| j                   j                  j%                  ||||||g       g |_        g |_        |dz  }c | j'                          |S )a  This function loops through the graph looking for Resize nodes that uses scales for resize (has 3 inputs).

        It substitutes found Resize with Resize that takes the size of the output tensor instead of scales. It adds
        Shape->Slice->Concat         Shape->Slice----^     subgraph to the graph to extract the shape of the output
        tensor. This fix is required for the dynamic shape support.
        r   ResizerM   /   shape_hw_out   )r`   dtypeshapeShapeshape_hwopr`   r   r   
const_zeror   r`   values	const_two   
const_fourslice_hw_outSliceslice_hwshape_bc_outshape_bcslice_bc_outslice_bcconcat_bchw_outConcatconcat_bchwaxisr   r`   attrsr   r   resize_bchw)ro   r   r   r   r   r`   r   r   rj   Variablenpint64Noder   Constantarrayemptyr   extendr   )rr   resized_node_countr   r`   add_nodediv_noder   r   r   r   r   r   r   r   r   r   r   r   r   none_varr   s                        r(   
resize_fixzOnnxOptimizer.resize_fix  s$    JJ$$ =	(Dww("s4;;'71'<yy3668::<>>!,668!{{~0ERXX^_]`a77TJ%6@P@PQR@S?T_k^l  [[d\.A"((TUSV^`^f^fJgh
KKTK-?RSQT\^\d\dHef	[[d\.A"((TUSV^`^f^fJgh
!{{~0ERXX^_]`a77
*()ZL)N	  "{{~0ERXX^_]`a77TJ%6@P@PQR@S?T_k^l  "{{~0ERXX^_]`a77
*(*iL)N	 #%++4:K3KSUS[S[decf"g gg-!1+(,7,- ;;,,. gg-** KKNHhP!\\!_- 

  ''8XxQ\^i(jk !"a'"{=	(~ 	!!rI   c                 N   d}| j                   j                  D ]y  }|j                  dv st        |j                  d   t
        j                  j                  j                        sN|j                  d   }|j                  d   }||g|_        |dz  }{ | j                          |S )Nr   )Addr   )
ro   r   r   r6   r   rj   irra   r   r   )rr   adjusted_add_node_countr   ra   biass        r(   adjustAddNodezOnnxOptimizer.adjustAddNodec  s    "#JJ$$ 	-Dww'!jQAVAV&WQ{{1~%tn'1,'	- 	&&rI   c                 $   d}| j                   j                  D ]  }|j                  dk(  s|j                  dz   }|j                  d   }|j
                  d   }t        j                  |dz         }t        j                  d|dz   dd	gi|g|g
      }t        j                  |dz         }t        j                  d|dz   i ||g|g
      }	t        j                  |dz         }
t        j                  |dz   t        j                  dgt        j                              }t        j                  d|dz   i ||g|
g
      }t        j                  |dz         }t        j                  d|dz   dd	gi|
g|g
      }t        j                  |dz         }t        j                  |dz   t        j                  |j                  d   gt        j                              }t        j                  d|dz   i ||g|g
      }t        j                  |dz         }t        j                  d|dz   i |g|g
      }t        j                  |dz         }t        j                  d |d!z   i ||g|g
      }t        j                  d"t        |      z   t        j                  |j                  d#   j                  d   j                  d$   j                   j#                  d#d%d#                  }t        j                  d&t        |      z   t        j                  |j                  d'   j                  d   j                  d$   j                   j#                  d#d%d#                  }t        j                  |d(z         }t        j                  d)|d*z   i ||g|g
      }t        j                  d|d+z   i ||g|g
      }| j                   j                  j%                  ||	|||||||g	       g |_        g |_        |d#z  } | j'                          |S ),Nr   InstanceNormalizationr   mean_out)r`   
ReduceMean	mean_nodeaxesr   sub_outSubsub_nodepow_out	pow_constg       @r   r   Powpow_node	mean2_out
mean2_nodeepsilon_outepsilon_constepsilonr   epsilon_nodesqrt_outSqrt	sqrt_nodediv_outDivr   zInstanceNormScaleV-r   r<       zInstanceBiasV-r   mul_outMulmul_noder   )ro   r   r   r`   r   r   rj   r   r   r   r   r   float32r   r7   ascontiguousarrayr   reshaper   r   )rr   removed_instance_norm_countr   r`   input_tensoroutput_tensorr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   constant_scaleconstant_biasr   r   r   s                              r(   decompose_instancenormsz%OnnxOptimizer.decompose_instancenormsp  s   &'#JJ$$ E	1Dww11yy3#{{1~ $Q;;D:,=>GG#+!B4.(>%J	 ++4)+;<774*#4BV^G_jqir ++4)+;<KKTK-?RUQV^`^h^hHij	774*#4BQZG[fmen KKTK-?@	WW#,!B4.#9&K
 !kktm/CD "/$**YBWAX`b`j`j8k!  "ww.%}5(M  ;;D:,=>GGD;$6b+ai`j	 ++4)+;<774*#4BQYGZeldm "$)C0K,LL((Q)>)>q)A)G)G)P)W)W)_)_`acegh)ij" !#$s+F'GG((Q)>)>q)A)G)G)P)W)W)_)_`acegh)ij! ++4)+;<774*#4BQ_G`krjs 774*#4BQ^G_jwix 

  ''(JiYackmuv !!+q0+KE	1N 	**rI   c                    d}| j                   j                  D ]@  }|j                  dk(  s|j                  g k7  s$|j	                         j                  dk(  sB|j	                  d      j                  dk(  sa|j	                         j	                         |j	                  d      k(  s|j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j                  dk(  s=|j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j                  dk(  st        |j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j                  d   j                  j                        dk(  s|j                  d   }|j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         j	                         }|j                  D cg c]0  }t        |t        j                  j                  j                        2 c}j                  d	      }t        j                   t#        |j                  |   j                  j%                               t        j&                  
      }t        j                  dt)        |      z   t        j*                  |j-                  d                  }|j	                         }	|	j                  D cg c]0  }t        |t        j                  j                  j                        2 c}j                  d	      }t        j                   t#        |	j                  |   j                  j%                               t        j&                  
      }
t        j                  dt)        |      z   t        j*                  |
j-                  d                  }|j	                         j	                         j	                         j	                         j	                         j                  d   j                  j%                         d   }|	j	                         j                  dk(  r!d	}|	j	                         j	                         }nd}|	}|j	                         j                  dk(  r|j	                         }|||g}t        j.                  dt)        |      z   t        j0                  t        j2                        |j                        }t        j4                  ddt)        |      z   ||gt7        d|fdt9        |      fg            }| j                   j                  j;                  |       | j                   j                  D ]U  }|j                  d   |j                  v s|j                  j                  |j                  d         }||j                  |<   W g |_        g |_        |dz  }C | j=                          |S c c}w c c}w )Nr   Reshaper   r   r   r   r   rM   Tr   zgroupNormGamma-r   zgroupNormBeta-SigmoidFCastzGroupNormV-	GroupNormzGroupNormN-r   bSwish)r   r   r   )ro   r   r   r   r   r   r   r   r   r6   rj   r   ra   r   r   r   r   r   tolistr   r7   r   r   r   r   float16r   r   intri   r   )rr   group_norm_plugin_countr   r   
gamma_noder   r   gammaconstant_gamma	beta_nodebetaconstant_betar   	use_swish	last_node
input_listgroup_norm_vgroup_norm_nr   s                      r(   insert_groupnorm_pluginz%OnnxOptimizer.insert_groupnorm_plugin  sS   "#JJ$$ ;	-D9$LLB&FFHKK</FF1ILLE)FFHJJLDFF1I-FFHJJLNN$&&(**,..0224668::<>>@BBDGG5PFFHJJLNN$&&(**,..0224668::<>>@BBDFFHKKuT

(**,..0224668::<CCAFMMSSTXYY  ${{1~!VVXZZ\^^-//1335779;;=??ACCEGGIKKM
KUK\K\]CC)>)>?]ccdhi**;*;E*B*I*I*P*P*R!S[][e[ef!#%,C(DDbFZFZ[`[h[hik[lFm" 'LLN	KTK[K[\CC)>)>?\bbcghxx)9)9%)@)G)G)N)N)P QY[YcYcd "$s+B'CCREYEYZ^ZfZfgiZjEk! &&(**,..*,,.00299!<CCJJLQO;;=##y0 $I ) 1I %I )I;;=##v- )I*NMJ
!{{!C(?$@@"((2::BVXdXjXj   "ww!C(?$@@%)N%	7';hI=W&XY  

  ''5 $

 0 0 >H ((+x> ( 5 5i6G6G6J K1=.> !$&	!'1,'w;	-z 	&&] ^ ]s   )5]/%5]4c           	      &   d}| j                   j                  D ]Z  }|j                  dk(  s|j                         j                  dk(  s2|j                         j                  d   |j                  d   k(  s`|j                         j                  d      j                  dk(  s|j                         j                  d      j                  dk(  s|j                         j                  d      j                         j                  dk(  s|j                         j                  d      j                         j                         j                  dk(  s?|j                         j                  d      j                         j                         j                         j                  dk(  s|j                         j                  d      j                         j                         j                         j                         j                  dk(  s|j                         j                  d      j                         j                         j                         j                         |j                         j                  d      k(  sv|j                         j                  d      j                         j                         j                         j                         j                         j                  d	k(  s|j                         j                  d      j                         j                         j                         j                         j                         j                         j                  dk(  slt        |j                         j                  d      j                         j                         j                         j                         j                         j                  d   j                  j                        dk(  s |j                         j                  dk(  r|j                  d   }n|j                         j                  d   }|j                         j                         j                         j                         j                         j                         j                         }|j                  D cg c]0  }t        |t        j                  j                  j                        2 c}j                  d
      }t        j                   t#        |j                  |   j                  j%                               t        j&                        }t        j                  dt)        |      z   t        j*                  |j-                  d                  }|j                         }	|	j                  D cg c]0  }t        |t        j                  j                  j                        2 c}j                  d
      }t        j                   t#        |	j                  |   j                  j%                               t        j&                        }
t        j                  dt)        |      z   t        j*                  |
j-                  d                  }|||g}t        j.                  dt)        |      z   t        j0                  t        j&                        |j                        }t        j2                  ddt)        |      z   |t5        dg      |g      }| j                   j                  j7                  |       |dz  }|	j8                  d   | j                   j8                  v rM| j                   j8                  j                  |	j8                  d         }|| j                   j8                  |<   |	j                         j                  dk(  r|	j                         }n|	}| j                   j                  D ]U  }|j8                  d   |j                  v s|j                  j                  |j8                  d         }||j                  |<   W g |_        ] | j;                          |S c c}w c c}w )Nr   r   r   r   r   r   r   r   r   Tr   zLayerNormGamma-r   zLayerNormBeta-zLayerNormV-	LayerNormzLayerNormN-)r   gh㈵>)r   r   r   r   )ro   r   r   r   r   r   r   r   r   r6   rj   r   ra   r   r   r   r   r   r  r   r7   r   r   r   r   r   r   ri   r   r   )rr   layer_norm_plugin_countr   r   r  r   r   r  r  r	  r
  r  r  layer_norm_vlayer_norm_nr  r   s                    r(   insert_layernorm_pluginz%OnnxOptimizer.insert_layernorm_plugin  s   "#JJ$$ =	+D<'FFHKK5(FFHOOA&$++a.8FFHJJqM$$-FFHJJqM$$-FFHJJqMOO%((L8FFHJJqMOO%''),,5FFHJJqMOO%'')++-00F:FFHJJqMOO%'')++-//144=FFHJJqMOO%'')++-//1TVVXZZ]BFFHJJqMOO%'')++-//133588EAFFHJJqMOO%'')++-//1335779<<E

1)++-//1335779@@CJJPPQUVV668;;%'#';;q>L#'668??1#5L!VVXZZ\^^-//1335779;;=
KUK\K\]CC)>)>?]ccdhi**;*;E*B*I*I*P*P*R!S[][e[ef!#%,C(DDbFZFZ[`[h[hik[lFm" 'LLN	KTK[K[\CC)>)>?\bbcghxx)9)9%)@)G)G)N)N)P QY[YcYcd "$s+B'CCREYEYZ^ZfZfgiZjEk! +NMJ
!{{!C(?$@@"((2::BVXdXjXj   "ww!C(?$@@%%':&;<)N  

  ''5'1,'$$Q'4::+=+== JJ..44Y5F5Fq5IJE0<DJJ&&u- {{}''61$-KKM	$-	$(JJ$4$4 B$,,Q/8??B$,OO$9$9):K:KA:N$OE5AHOOE2B )+I%{=	+~ 	&&U ^ ]s   5b	5bc                    |j                   d   j                  }|j                   d   j                  }|j                  d   }|}	|j                  d   |	z  }
t        j                  |j                  ||	|
      |j                  ||	|
      g      j                  |d|	z  |
z        }|j                   d   }|j                  d   }t        j                  dj                  |      t        j                  |            }t        j                  ddj                  |      ||g|g      }| j                  j                  j                  |       ||j                  |      j                   d<   ||j                  |      j                   d<   t!        d|      D ]R  }|j                         j                   j#                          |j                         j                   j#                          T |j                  j#                          |j                  j#                          |j                   j#                          |j                   j#                          | j%                          |S )Nr   r   r   zWeights_KV_{}MatMulzMatMul_KV_{}r   r   r   r   r   dstackr   r   rj   r   formatr   r   ro   r   ri   r   rangeclearr   )rr   node_knode_vfused_kv_idxheadsnum_dynamic	weights_k	weights_vchannel_count	num_headshead_dim
weights_kvr   output_tensor_kconstant_weights_kvfused_kv_noder   s                    r(   fuse_kvzOnnxOptimizer.fuse_kvB  s   MM!$++	MM!$++	!*	??1%2 YY!!-HE!!-HE

 '-Y!9
: 	 }}Q' ..+ kk/*@*@*NPRPdPdeoPpq &&|4 "56$%	
 	

. +:$$Q'*9$$Q'q+& 	&AHHJ##%HHJ##%	&
 	rI   c                    |j                  |      j                         j                  d   }|j                         j                  d   }|j                  d   }	|j                  d   j                  d   j                  j                          |j                  d   j                  d   j                  j                          |j                  |      j                         j                  j                          t	        d|      D ]H  }
|j                  |
      j                         j                  d      j                  j                          J |j                  d   j
                  }|j                  d   |dz  z  }t        j                  dj                  |      t        j                  t        j                  dd|d|gt        j                                    }t        j                  dj                  |      t        j                  t        j                         d       }t        j"                  ddj                  |      ||g|g	      }| j$                  j&                  j)                  |       t        j"                  d
dj                  |      ||g|	g	      }| j$                  j&                  j)                  |       ||j                  |      j                  d<   |dkD  rt        j                  dj                  |      t        j                  t        j                        d       }t        j"                  ddj                  |      |j                  d   g|g      }| j$                  j&                  j)                  |       ||j                         j                  d<   |j                  j                          | j+                          y )Nr   r   r   zShape_KV_{}r   zReshapeKV_{}r   
Reshape_{}r   fMHCAzfMHCA_{}zReshape2_fmhca{}_outr   zReshape2_fmhca{}_shaper   r   )r   r   r   r  r  r   r   rj   r   r  r   r   r   r   r   r   r  r   ro   r   ri   r   )rr   node_qnode_kvfinal_tranposemhca_idxr"  r#  output_q	output_kvoutput_final_tranposer   r)  dims_per_headr   output_reshaper   fmhcareshape2_input1_outreshape2_input1_shapes                      r(   insert_fmhcazOnnxOptimizer.insert_fmhcav  s    88K(**,33A6IIK&&q)	 . 6 6q 9 	""1%,,224""1%,,224!((..0q+& 	0AHHQKMMOa ''--/	0 ^^A&--
"((+	:   *  1a=*IQSQYQY!Z[
 ^%:%:8%DbhhrzzFZ\`a''|228<iQVEWbpaq
 	

( ""8,n-*+	
 	

& ,4%%a(?"$++.D.K.KH.UWYW_W_`b`h`hWiko"p$&GG(//9a(),-	%! JJ##$9:+>N%%a( 	$$&rI   c           	         |j                   d   j                  }|j                   d   j                  }|j                   d   j                  }	|j                  d   }
|}|j                  d   |z  }t        j                  |j                  |
||      |j                  |
||      |	j                  |
||      g      j                  |
d|z  |z        }|j                   d   }|j                  d   }t        j                  dj                  |      t        j                  |            }t        j                  ddj                  |      ||g|g      }| j                  j                  j                  |       ||j                  |      j                   d<   ||j                  |      j                   d<   ||j                  |      j                   d<   t!        d|      D ]z  }|j                         j                   j#                          |j                         j                   j#                          |j                         j                   j#                          | |j                  j#                          |j                  j#                          |j                  j#                          |j                   j#                          |j                   j#                          |j                   j#                          | j%                          |S )Nr   r   rM   zWeights_QKV_{}r  zMatMul_QKV_{}r   r  )rr   r2  r  r   fused_qkv_idxr"  r#  	weights_qr$  r%  r&  r'  r(  weights_qkvr   r*  constant_weights_qkvfused_qkv_noder   s                      r(   fuse_qkvzOnnxOptimizer.fuse_qkv  sy   MM!$++	MM!$++	MM!$++	 "*	??1%2 ii!!-HE!!-HE!!-HE
 '-Y!9
: 	 }}Q' ..+!{{+;+B+B=+QSUSgSghsStu  ''6 "67$%	
 	

/ +:$$Q'*9$$Q'*9$$Q'q+& 	&AHHJ##%HHJ##%HHJ##%	& 	rI   c                 Z   |j                         j                  d   }|j                  d   }|j                  d   j                  d   j                  j                          |j                  d   j                  d   j                  j                          |j                  d   j                  d   j                  j                          |j                  d   j                  }|j
                  d   |dz  z  }	t        j                  dj                  |      t        j                  t        j                  dd|d|	gt        j                                    }
t        j                  dj                  |      t        j                  t        j                        d       }t        j                   dd	j                  |      ||
g|g
      }| j"                  j$                  j'                  |       t        j                   ddj                  |      |g|g
      }| j"                  j$                  j'                  |       |dkD  rt        j                  dj                  |      t        j                  t        j                        d       }t        j                   ddj                  |      |j                  d   g|g      }| j"                  j$                  j'                  |       ||j                         j                  d<   |j                  j                          | j)                          y )Nr   r   r   rM   zShape_QKV_{}r   zReshapeQKV_{}r   r/  r   fMHA_V2zfMHA_{}zReshape2_{}_outr   zReshape2_{}_shaper1  )r   r   r   r  r   r   rj   r   r  r   r   r   r   r   r   r  r   ro   r   ri   r   )rr   node_qkvr4  mha_idxr"  r#  
output_qkvr8  rB  r9  r   output_shaper   fmhar<  r=  s                   r(   insert_fmhazOnnxOptimizer.insert_fmha  s   ZZ\((+
 . 6 6q 9 	##A&--335##A&--335##A&--335ooa(//#))!,; !!'*  1a=*IQSQYQY!Z[
 {{?#9#9'#BBHHRZZDXZ^_''|227;ZQVDWbnao
 	

( wwy//8,ZoYp
 	

%?"$++.?.F.Fw.OQSQYQYZ\ZbZbQcei"j$&GG,33G<hooVWFXEYdwcx%! JJ##$9:+>N%%a( 	$$&rI   c                 &   |j                   dk(  rt        |j                        dk(  rh|rBt        |j                  d   j                        dkD  r|j	                         j                   dk(  s)|s!t        |j                  d   j                        dk(  r|j                         j                   dk(  rM|j                  d      j                   dk(  r#|j                  d      j                   dk(  rdnd}nd}|r|n|dz   }nd}d}|j                  |      }|j                   dk(  rl|j                         j                   d	k(  rN|j                         j                         j                   dk(  r"|j                         j                         j                         j                   dk(  r|j                         j                         j                         j	                  d      j                   d
k(  r|j                         j                         j                         j	                  d      j                   dk(  rV|j                         j                         j                         j	                  d      j	                         j                   dk(  r|j                         j                         j                         j	                  d      j	                         j	                         j                   dk(  r|j                         j                         j                         j	                  d      j	                         j	                         j	                  d      j                   dk(  r&|j                         j                         j                         j	                  d      j	                         j	                         j	                  d      j                   d	k(  r|j                         j                         j                         j	                  d      j	                         j	                         j	                  d      j	                         j                   dk(  r0|j                         j                         j                         j	                  d      j	                         j	                         j	                  d      j	                         j	                         j                   d	k(  r|j                         j                         j                         j	                  d      j	                         j	                         j	                  d      j	                         j	                         j	                         j                   dk(  r|j                         j                         j                         j	                  d      j	                         j	                         j	                  d      j	                         j	                         j	                         j	                         j                   dk(  rV|j                  |j                         j                         j                         j	                  d      j	                         j	                         j	                  d      j	                         j	                         j	                         j	                         j                  k7  r|j                         j                         j                         j	                  d      j	                         j	                         j	                  d      j	                         j	                         j	                         }|j                         j                         j                         j	                  d      j	                         j	                         j	                  d      j	                         j	                         j	                         j	                         }|}|j                         j                         j                         j                  |      j                         }	|j                   dk(  r|	j                   d	k(  r	d||||||	fS y)Nr  r   r   r   r   r   rM   r   	TransposeSoftmaxr   T)Fr   r   NNNN)r   r   r   r   r   r   r`   )
rr   r   mhanum_dynamic_kvnum_dynamic_qr   r2  r  r   r4  s
             r(   mha_mhca_detectedzOnnxOptimizer.mha_mhca_detected!  sv   
 GGxDLL!Q&T[[^223a7DFFHKK5<PDKKN$9$9 :a ? vvx{{g%66!9<<7**.&&),,'*AQqN%&N25>A;M!" !~&A	!CCEHH+CCEGGILLI-CCEGGIKKM$$0CCEGGIKKMOOA&))Y6CCEGGIKKMOOA&))Y6CCEGGIKKMOOA&((*--6CCEGGIKKMOOA&((*,,.11X=CCEGGIKKMOOA&((*,,.00366)CCCEGGIKKMOOA&((*,,.00366+ECCEGGIKKMOOA&((*,,.003557::iGCCEGGIKKMOOA&((*,,.00355799;>>+MCCEGGIKKMOOA&((*,,.00355799;==?BBiOCCEGGIKKMOOA&((*,,.00355799;==?AACFF(RII!3!5!5!7!9!9!;!=!=a!@!B!B!D!F!F!H!J!J!L!N!N!P!U!UU +--/11355a8::<>>@BBD+--/11355a8::<>>@BBDFFH!"!?!A!A!C99(^->->+-MPVXfff2rI   c           	         | j                   j                  }t        |      D ]  \  }}|dz   t        |      kD  s|dz   t        |      kD  r)| j	                  ||   d      \  }}}	}
}}}|sJ|dk(  s
||	dz   k(  sJ |dk(  r#|
j
                  d   j                  d   |z  dk(  r| j                  |||||	      }| j                  |
|||||        y y)	Nr   r   FrQ  r   K      T)	ro   r   r   r   rT  r   r   r-  r>  )rr   r"  
mhca_indexsmr   idx_detectedrS  rR  r2  r  r   r4  r3  s                  r(   fuse_kv_insert_fmhcaz"OnnxOptimizer.fuse_kv_insert_fmhcaT  s    

  & 	FCQwU#sQwU'; _c^t^tc
 _u _[Hm^VVV^ $)]nq>P-PPP8a 0 6 6q 9U Bc I,,vvz5.Y!!&'>:uVcd%	& rI   c           	      D   | j                   j                  }t        |      D ]|  \  }}|dz   t        |      kD  s|dz   t        |      kD  r)| j	                  ||   d      \  }}}}	}
}}|sJ||k(  sJ | j                  |	|
||||      }| j                  |||||        y y)Nr   r   TrV  F)ro   r   r   r   rT  rE  rM  )rr   r"  	mha_indexr   r[  r\  r]  rS  rR  r2  r  r   r4  rH  s                 r(   fuse_qkv_insert_fmhaz"OnnxOptimizer.fuse_qkv_insert_fmhal  s    

  & 	FCQwU#sQwU'; _c^t^tc
 _u _[Hm^VVV^ $666==ESab  >9e^\	  rI   c                 b    d}| j                  |||      r|dz  }| j                  |||      r|S Nr   r   )r^  )rr   r'  rZ  rY  s       r(   insert_fmhca_pluginz!OnnxOptimizer.insert_fmhca_plugin  s<    
''	:rB!OJ ''	:rBrI   c                 ^    d}| j                  ||      r|dz  }| j                  ||      r|S rc  )ra  )rr   r'  r`  s      r(   insert_fmha_pluginz OnnxOptimizer.insert_fmha_plugin  s8    	''	9=NI ''	9=rI   ry   ) F)NN)r   )__name__
__module____qualname__rw   rq   r7   rm   rW   r@   r   r   r  r   r   rV   r   r   r   r  r  r-  r>  rE  rM  rT  r^  ra  rd  rf   rI   r(   rU   rU      s    $2%3 )3 )

.
2# S 3 c H"T'J+X@'DB'H2h<|:x/b13f0*rI   rU   )FT   Nrh  )'__doc__r0   r8   r*   collectionsr   collections.abcr   r   copyr   r   r[   r   r2   r3   r   rk   rh   rj   onnx_graphsurgeon.logger.loggerr	   polygraphy.backend.onnx.loaderr
   rfdetr.export._onnx.symbolicr   rfdetr.utilities.loggerr   r?   r7   nnModulerT   r  boolobjectrH   re   rU   rl  rI   r(   <module>rz     s   6   	 # -    $
"8
= B .	  #W Whsm#W88??W #W <<(5<<"88	W
 3-W #wsCx001D8W W W W *W W 	W| 	((#( <<(5<<"88( 	(
 	(`M MK  DO  	BH  Ns4   
D 
D D% 	DD	D"!D"%D/.D/