
    ^j o                        d dl Z d dlmZ d dlmZ d dlmZ d dlmZm	Z	m
Z
 d dlmZmZ d dlmZmZ eeeef   z  Z eej$                  j&                  dej(                        Zedddd	eez  d
efd       Zedddded
efd       Zedddded
efd       Zeddddeded
efd       Zedddded
efd       Zeej6                  ddddeded
eeef   fd              Zedddded
efd       Zeej6                  ddddeded
eeef   fd              Zedddded
efd       Zeddddeded
eeef   fd       Z eej6                  dddded
efd              Z!eej6                  ddddeded
efd              Z"edddded
efd       Z#edddddede$d
efd       Z%eddddeded
eeef   fd       Z&edddddedede$d
eeef   fd        Z'edddded!ed
efd"       Z(edddded!ed
efd#       Z)edddded!eded
eeeef   fd$       Z*eddddded!edede$d
eeeef   f
d%       Z+e	 d3dddded!ed&ed
efd'       Z,e	 d3dddded!ed&ed
efd(       Z-e	 d3dddded!eded&ed
eeeef   f
d)       Z.e	 d3dddded!eded&ed
eeeef   f
d*       Z/edddded!ed
efd+       Z0edddded!eded
eeeef   fd,       Z1edddded!ed
efd-       Z2eej6                  dddded!eded
eeeef   fd.              Z3edddded!ed
efd/       Z4edddded!eded
eeeef   fd0       Z5de#e%eeed1Z6de&e'eee d1Z7e(e)e,e-e2e4e0d2Z8e*e+e.e/e3e5e1d2Z9y)4    N)Tuple)partial)Float32Boolean
const_expr)Tdsl_user_op)llvmnvvm)src_c	calc_funclocipareturnc          
          t        t        j                  t        j                         t        |       j                  ||      gdddd            S )Nr   ztanh.approx.f32 $0, $1;z=f,fF)has_side_effectsis_align_stack)r   r
   
inline_asmr   f32ir_value)r   r   r   s      i/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torch/_vendor/quack/activation.pytanhr      sJ    EEGQZ  SR 01%" 	
	 	    xc                   t        t        | t                     rddt        d| z        z  z   S t        j
                  j                  d|       }t        |d         t        |d         f}t        j
                  j                  |dd      S )N      ?r   r   r      r   
isinstancetupler   cutearchmul_packed_f32x2fma_packed_f32x2)r   r   r   x_halftanh_x_halfs        r   sigmoid_tanhr*   %   su    jE**+S4a=(((++J:F1IVAY8yy))+z:NNr   c                   t        t        | t                     rEt        j                  j                  dt        j                  j                  |  d      z   ||      S t        j                  t        j                        }t        j                  j                  | | | f      }t        j                  j                  |d   d      t        j                  j                  |d   d      f}t        j                  j                  |d      }t        j                  j                  |d         t        j                  j                  |d         fS )N      ?Tfastmathr   r   r    r,   r,   )r   r"   r#   r$   r%   
rcp_approxmathexplog2er&   exp2add_packed_f32x2)r   r   r   log2_eneg_x	exp_neg_xdenoms          r   sigmoidr;   0   s    jE**+yy##C$))--T-*J$JPSXZ#[[466"		**1w.@AIINN58dN3IINN58dN3
	 		**9jAyy##E!H-tyy/C/CE!H/MMMr   outdoutc                    || | | z  z
  z  S N )r<   r=   r   r   s       r   dsigmoid_from_outputrA   ?   s     3s?##r   c                8   t        t        | t                     r)t        j                  j                  | t        d            S t        j                  j                  | d   t        d            t        j                  j                  | d   t        d            fS N        r   r    )r   r"   r#   r$   r%   fmaxr   r   r   r   s      r   relurG   E   sg    jE**+yy~~a..yy~~adGCL1499>>!A$PS3UUUr   c                h   t        t        | t                     rFt        | dkD        }|r|n
t	        d      t
        j                  j                  | t	        d            fS t        | d   dkD        }t        | d   dkD        }|r|d   n
t	        d      |r|d   n
t	        d      f}|t        |       fS )Nr   rD   r    )	r   r"   r#   r   r   r$   r%   rE   rG   )r   r=   r   r   x_posx0_posx1_posdxs           r   drelurM   M   s    
 jE**+At'#,		q'#,0OOO1"1"d1gWS\f47'RU,W47{r   c                ~   t        t        | t                     r,t        j                  j                  | t        d            | z  S t        j                  j                  | d   t        d            t        j                  j                  | d   t        d            f}t        j                  j                  ||       S rC   )r   r"   r#   r$   r%   rE   r   r&   )r   r   r   relu_xs       r   relu_sqrP   \   s    jE**+yy~~a.22))..1ws|4diinnQqT7SV<6XYyy))&!44r   c                H   t        t        | t                     rt        |       }|| z  }d||z  z  }||fS t        |       }t        j
                  j                  ||       }t        j
                  j                  dt        j
                  j                  ||            }||fS )a  
    ReLU squared backward pass: computes gradient w.r.t. x and recomputes forward
    Given: relu_sq_out = max(x, 0) * x, and dout = grad w.r.t. relu_sq_out
    Returns: (dx, relu_sq_out) where:
    - dx = dout * 2 * x if x > 0, else 0
    - relu_sq_out = max(x, 0) * x
           @)rR   rR   )r   r"   r#   rG   r$   r%   r&   )r   r=   r   r   rO   relu_sq_outrL   s          r   drelu_sqrT   e   s     jE**+aqjD6M";aii00;YY''
DII4N4NtU[4\];r   c          
      P   t        j                  dt         j                  z        }d|z  }t        t	        | t
                     r d| dt        | ||| | z  z  z   z        z   z  z  S t        j                  j                  | |       }t        j                  j                  |||f||f      }t        j                  j                  | |      }t        |d         t        |d         f}t        j                  j                  || |       }	t        j                  j                  d|	      S )z
    gelu(x) = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3)))
            = 0.5 * x * (1 + tanh(x * (0.797885 + 0.0356774 * x * x)))
       Hm?r   r,   r   r    r   r1   sqrtpir   r"   r#   r   r$   r%   r&   r'   )
r   r   r   sqrt_2_over_pisqrt_2_over_pi_coeffx_sqx_sq_scaledztanh_zx_tanh_zs
             r   gelu_tanh_approxrb   ~   s    YYq477{+N#n4jE**+ T!~0DA0NNOPPR
 	
 yy))!Q/ii00')=>Q_@`
 II&&q+6qt*d1Q4j)99--fa;yy))*h??r   c                ^   t        j                  dt         j                  z        }d|z  }d|z  }t        t	        | t
                     rM| | z  }t        | |||z  z   z        }dd|z  z   }	| |	z  }
d||z  z
  }|||z  z   }|	| d||z  z  z  z   }||z  }||
fS t        j                  j                  | |       }t        j                  j                  |||f||f      }t        j                  j                  | |      }t        |d         t        |d         f}t        j                  j                  |dd      }	t        j                  j                  | |	      }
t        j                  j                  ||d    |d    fd      }t        j                  j                  |||f||f      }t        j                  j                  ||      }t        j                  j                  | |      }t        j                  j                  |d|	      }t        j                  j                  ||      }||
fS )	a  
    GELU tanh approximation backward pass: computes gradient w.r.t. x and recomputes forward
    Given: gelu_out = 0.5 * x * (1 + tanh(x * (c1 + c2 * x^2))), and dout = grad w.r.t. gelu_out
    Returns: (dx, gelu_out)

    Derivative uses the chain rule:
    d/dx[gelu(x)] = 0.5 * (1 + tanh(z)) + 0.5 * x * sech^2(z) * dz/dx
    where z = x * (c1 + c2 * x^2), dz/dx = c1 + 3 * c2 * x^2
    and sech^2(z) = 1 - tanh^2(z)
    rV   rW   g      @r   r    r   r   r/   rX   )r   r=   r   r   r[   r\   sqrt_2_over_pi_coeff_3r]   r`   half_tanh_z_plus_onegelu_outsech2_zdz_dxdgelurL   r^   r_   sech2_dz_dxx_sech2_dz_dxs                      r   dgelu_tanh_approxrl      s'    YYq477{+N#n4 #77jE**+1ua>,@4,GGHI"S6\1++ fvo%!7$!>>$qC7U?,C'DDE\8| yy))!Q/ii00')=>Q_@`
 II&&q+6qt*d1Q4j)#yy99&*jY99--a1EF )),,Vvayj6!9*5MzZ		**)+AB^UcDd
 ii00%@		221kB		**=*FZ[YY''e48|r   c                   t        t        | t                     r^t        | dkD        }|sLt        j
                  j                  t        t        j
                  j                  | d            dz   d      S | S t        j                  t
        j                        }t        j                  j                  | ||f      }t        j
                  j                  |d   d      t        j
                  j                  |d   d      f}t        j                  j                  |d      }t        j
                  j                  |d   d      t        j
                  j                  |d   d      f}t        j                  d      }	t        j                  j                  ||	|	f      }
t        | d   dkD        }t        | d   dkD        }|s|
d   n| d   |s|
d   fS | d   fS )	N      4@Tr-   r,   r   r    r/   rR   )r   r"   r#   r   r$   r1   logr   r2   r3   r4   r%   r&   r6   )r   r   r   
use_linearr7   x_log2ex_expx_exp_p1log_x_exp_p1ln2
softplus_xuse_linear_0use_linear_1s                r   softplusry      s    jE**+QX&
  IIMM'$))--D-"ABSHSWMX	
 	
 466")),,Q0@AwqzD9499==QR^b=;cd99--eZ@IINN8A;N6IINN8A;N6
 hhsmYY//sCjI
qtd{+qtd{+!-JqM1Q4!-JqM
 	
34Q4
 	
r   c                z    t        | dkD        }||t        j                  j                  |  d      z  z
  }|s|S |S )Nrn   Tr-   )r   r$   r1   r2   )r<   r=   r   r   rp   rL   s         r   dsoftplus_from_outputr{      sA     t$J	tyy}}cTD}99	9B2)T)r   c                    t        t        | t                     r| t        |       z  S t        j
                  j                  | t        |             S )z:
    silu(x) = x * sigmoid(x) = x * rcp(1 + exp(-x)).
    r   r"   r#   r;   r$   r%   r&   rF   s      r   silur~      s?    
 jE**+71:~yy))!WQZ88r   F)already_halvedr   r   r   c                T   t        t        | t                     r$t        |       rd| z  n| }|t        |      z  |z   S t        |       r t        j
                  j                  d|       n| }t        |d         t        |d         f}t        j
                  j                  |||      S )z
    silu(x) = x * sigmoid(x) = x * (1 + tanh(x / 2)) / 2 = (0.5 * x) * tanh(0.5 * x) + (0.5 * x)
    This compiles down to 3 SASS instructions: FMUL to get 0.5 * x, MUFU.TANH, and FFMA.
    r   r   r   r    r!   )r   r   r   r   r(   r)   s         r   	silu_tanhr     s     jE**+&>'9:qV$v-->H^I[>\++J:bcF1IVAY8yy))&+vFFr   c                   t        t        | t                     r"t        |       }| |z  }|||z  z
  |z   |z  }||fS t        |       }t        j
                  j                  | |      }t        j
                  j                  ||d    |d    f|      }t        j
                  j                  ||      }t        j
                  j                  ||      }||fS )z
    SiLU backward pass: computes d_silu(x) * dout and recomputes silu(x).

    d_silu(x) = sigmoid(x) * (1 + x * (1 - sigmoid(x))).
    r   r    )	r   r"   r#   r;   r$   r%   r&   r'   r6   )	r   r=   r   r   	sigmoid_xsilu_xd_silu_x_dout sigmoid_x_minus_silu_x_sigmoid_x,sigmoid_x_minus_silu_x_sigmoid_x_plus_silu_xs	            r   dsilur     s     jE**+AJ	Y"Vi%77&@DHf$$AJ	++Ay9+/99+E+E
VAYJ/,
( 8<yy7Q7Q,f8
4 		228$
 f$$r   c                   t        t        | t                     rZt        |       r!d| z  }t        |      }d|z  dz   }||z  |z   }nt        |       }	d|	z  dz   }| |	z  | z   }|||z  z
  |z   |z  }
|
|fS t        |       rt        j
                  j                  d|       }t        |d         t        |d         f}t        j
                  j                  |dd      }t        j
                  j                  |||      }n^t        | d         t        | d         f}	t        j
                  j                  |	dd      }t        j
                  j                  | |	|       }t        j
                  j                  ||d    |d    f|      }t        j
                  j                  ||      }t        j
                  j                  ||      }
|
|fS )zE
    SiLU backward using sigmoid(x) = 0.5 * (1 + tanh(0.5 * x)).
    r   r   r   r    )	r   r"   r#   r   r$   r%   r&   r'   r6   )r   r=   r   r   r   r(   r)   r   r   tanh_xr   r   r   s                r   
dsilu_tanhr   9  s    jE**+.()1WFv,Kk)C/Ik)F2F!WFfs*IZ!^F"Vi%77&@DHf$$.()YY//
A>Fq	?DO<K		22;
JWIYY//VLF1Q4j$qt*-F		226:zRIYY//61=F+/99+E+E
VAYJ/,
( 8<yy7Q7Q,f8
4 		228$
 f$$r   yc                    t        t        | t                     rt        |       |z  S t        j
                  j                  t        |       |      S r?   )r   r"   r#   r~   r$   r%   r&   r   r   r   r   s       r   swiglur   g  s=    jE**+Aw{yy))$q'155r   c                    t        t        | t                     rt        |       |z  S t        j
                  j                  t        |       |      S r?   )r   r"   r#   r   r$   r%   r&   r   s       r   swiglu_tanhr   o  s>    jE**+|ayy)))A,::r   c                Z   t        t        | t                     r4t        |       }| |z  }||z  }|||z  z
  |z  |z   }||z  }	|}
||z  }|	|
|fS t        |       }t        j
                  j                  | |      }t        j
                  j                  ||      }t        j
                  j                  ||d    |d    f|      }t        j
                  j                  |||      }t        j
                  j                  ||      }	|}
t        j
                  j                  ||      }|	|
|fS )a  
    SwiGLU backward pass: computes gradients w.r.t. x (gate) and y (up projection)
    Given: swiglu_out = silu(x) * y, and dout = grad w.r.t. swiglu_out
    Returns: (dx, dy, swiglu_out) where dx = dout * y * d_silu(x), dy = dout * silu(x)

    d_silu(x) = sigmoid(x) * (1 + x * (1 - sigmoid(x)))

    This has been optimized to use fewer instructions (i.e. we expand things out
    to use FFMA instead of FADD and FMUL).
    r   r    r   r"   r#   r;   r$   r%   r&   r'   )r   r   r=   r   r   r   r   silu_x_doutr   rL   dy
swiglu_outr   s                r   dswiglur   w  s2   & jE**+AJ	Ytm #Vi%774?+MQaZ
2z!! AJ	++Ay9ii00>+/99+E+E
VAYJ/,
( 		22,dK
 YY''q9YY//:
2z!!r   c                   t        t        | t                     r\t        |       rt        |       }| |z  }nt	        |       }d|z  dz   }| |z  | z   }||z  }	|||z  z
  |z  |	z   }
|
|z  }|	}||z  }|||fS t        |       r,t        |       }t
        j                  j                  | |      }n^t	        | d         t	        | d         f}t
        j                  j                  |dd      }t
        j                  j                  | ||       }t
        j                  j                  ||      }	t
        j                  j                  ||d    |d    f|      }t
        j                  j                  |||	      }
t
        j                  j                  |
|      }|	}t
        j                  j                  ||      }|||fS )zG
    SwiGLU backward using sigmoid(x) = 0.5 * (1 + tanh(0.5 * x)).
    r   r   r    r   )	r   r"   r#   r*   r   r$   r%   r&   r'   )r   r   r=   r   r   r   r   r   r   r   r   rL   r   r   r   s                  r   dswiglu_tanhr     s    jE**+.()$QI]F!WFfs*IZ!^Ftm"Vi%774?+MQaZ
2z!!.()$QIYY//9=F1Q4j$qt*-F		226:zRIYY//61=Fii00>+/99+E+E
VAYJ/,
( 		22,dK
 YY''q9YY//:
2z!!r   alphac                H   t        t        | t                     rt        || z        }| |z  }||z  |z   S t        j
                  j                  ||f|       }t        |      }t        j
                  j                  | |      }t        j
                  j                  |||      S )zThe swiglu variant used in gpt-oss, which has a scaling factor on x and bias of 1 to y.
    https://github.com/openai/gpt-oss/blob/7be9334950053a888e24887a57dac797a17d6e00/gpt_oss/torch/model.py#L249
    x * sigmoid(alpha * x) * (y + 1)
    r   )r   r   r   r   r   sigmoid_alpha_xr   alpha_xs           r   
swiglu_oair     s     jE**+!%!),_$zF"")),,eU^Q?!'*++A?yy))&!V<<r   c                   t        t        | t                     r!d| z  }|t        ||z        z  |z   }||z  |z   S t        j
                  j                  d|       }t        j
                  j                  ||f|      }t        |d         t        |d         f}t        j
                  j                  |||      }t        j
                  j                  |||      S )z8Tanh-based swiglu_oai kept for SASS/accuracy comparison.r   r   r   r    r!   )	r   r   r   r   r   r(   r   alpha_x_halftanh_alpha_x_halfs	            r   swiglu_oai_tanhr     s    
 jE**+q$uv~..7zF""++J:yy115%.&I!,q/2Da4IJ++F4EvNyy))&!V<<r   c                   t        t        | t                     r@t        || z        }| |z  }||z  }|||||z  z
  z  z   |z  }	|	|z  |	z   }
|}||z  |z   }|
||fS t        j
                  j                  ||f|       }t        |      }t        j
                  j                  | |      }t        j
                  j                  ||      }t        j
                  j                  ||d    |d    f|      }t        j
                  j                  ||f||      }t        j
                  j                  ||      }	t        j
                  j                  |	||	      }
|}t        j
                  j                  |||      }|
||fS )au  
    Swiglu OAI backward pass: computes gradients w.r.t. x and y
    Given: swiglu_oai_out = x * sigmoid(alpha * x) * (y + 1), and dout = grad w.r.t. swiglu_oai_out
    Returns: (dx, dy, swiglu_oai_out)

    Derivative of x * sigmoid(alpha * x) w.r.t. x:
    d/dx[x * sigmoid(alpha * x)] = sigmoid(alpha * x) + alpha * x * sigmoid(alpha * x) * (1 - sigmoid(alpha * x))
    r   r    r   )r   r   r=   r   r   r   r   r   r   r   rL   r   r   r   silu_x_minus_productsigmoid_plus_alpha_diffs                   r   dswiglu_oair     s    jE**+!%!),_$tm(5FVo=U4U+VVZ^^Q.aZ&(
2z!!)),,eU^Q?!'*++A?ii00>#yy99oa((?1+=*=> 
 #'))"<"<EN0/#
 		223JDQYY''q-HYY//6B
2z!!r   c                   t        t        | t                     rKd|z  | z  }ddt        |      z  z   }| |z  }||z  }	|||||z  z
  z  z   |z  }
|
|z  |
z   }|	}||z  |z   }|||fS t        j
                  j                  d|z  d|z  f|       }t        |d         t        |d         f}t        j
                  j                  |dd      }t        j
                  j                  | |      }t        j
                  j                  ||      }	t        j
                  j                  ||d    |d    f|      }t        j
                  j                  ||f||      }t        j
                  j                  ||      }
t        j
                  j                  |
||
      }|	}t        j
                  j                  |||      }|||fS )z9Tanh-based dswiglu_oai kept for SASS/accuracy comparison.r   r   r    r   r!   )r   r   r=   r   r   r   r   r   r   r   r   rL   r   r   r   r   r   s                    r   dswiglu_oai_tanhr   (  s   
 jE**+eq(d<&8 88_$tm(5FVo=U4U+VVZ^^Q.aZ&(
2z!!yy11C%K3;2PRST!,q/2Da4IJ))445F
T^_++A?ii00>#yy99oa((?1+=*=> 
 #'))"<"<EN0/#
 		223JDQYY''q-HYY//6B
2z!!r   c                    t        t        | t                     rt        |       }||z  S t        |       }t        j
                  j                  ||      S )z:GLU: Gated Linear Unit
    glu(x, y) = sigmoid(x) * y
    r}   )r   r   r   r   r   s        r   glur   J  sI    
 jE**+AJ	1}AJ	yy)))Q77r   c                z   t        t        | t                     r$t        |       }||z  }||z  }||z
  |z  }|}	||	|fS t        |       }t        j
                  j                  ||      }t        j
                  j                  ||      }t        ||      }
t        j
                  j                  |
|      }|}	||	|fS )a/  
    GLU backward pass: computes gradients w.r.t. x (gate) and y (up projection)
    Given: glu_out = sigmoid(x) * y, and dout = grad w.r.t. glu_out
    Returns: (dx, dy, glu_out) where:
    - dx = dout * y * sigmoid(x) * (1 - sigmoid(x))
    - dy = dout * sigmoid(x)
    - glu_out = sigmoid(x) * y
    )r   r"   r#   r;   r$   r%   r&   sub_packed_f32x2)r   r   r=   r   r   r   sigmoid_x_doutglu_outrL   r   y_minus_glu_outs              r   dglur   W  s     jE**+AJ	"T)a-
 'k^+2wAJ	33ItD)),,Y:*1g6YY''H2wr   c                    t        t        | t                     r,t        j                  j                  | t        d            |z  S t        |       }t        j                  j                  ||      S )zPReGLU: ReLU Gated Linear Unit
    reglu(x, y) = relu(x) * y = max(x, 0) * y
    rD   )	r   r"   r#   r$   r%   rE   r   rG   r&   )r   r   r   r   rO   s        r   reglur   z  sT    
 jE**+yy~~a.22ayy))&!44r   c                R   t        t        | t                     rXt        | dkD        }t        j
                  j                  | t        d            }|r||z  n
t        d      }||z  }||z  }	|||	fS t        | d   dkD        }
t        | d   dkD        }t        |       }t        j
                  j                  ||      }|
r|d   n
t        d      |r|d   n
t        d      f}t        j
                  j                  ||      }t        j
                  j                  ||      }	|||	fS )a!  
    ReGLU backward pass: computes gradients w.r.t. x (gate) and y (up projection)
    Given: reglu_out = relu(x) * y, and dout = grad w.r.t. reglu_out
    Returns: (dx, dy, reglu_out) where:
    - dx = dout * y if x > 0, else 0
    - dy = dout * relu(x)
    - reglu_out = relu(x) * y
    r   rD   r    )
r   r"   r#   r   r$   r%   rE   r   rG   r&   )r   r   r=   r   r   rI   rO   rL   r   	reglu_outrJ   rK   dout_ys                r   dreglur     s    jE**+A73<0 dQhgclF]QJ	2y  1"1"a++D!4"vayVq	QXY\Q]_YY''f5II..vq9	2y  r   c                    t        t        | t                     rt        |       |z  S t        j
                  j                  t        |       |      S )zhGeGLU: GELU Gated Linear Unit
    geglu(x, y) = gelu(x) * y
    Uses the tanh approximation of GELU
    )r   r"   r#   rb   r$   r%   r&   r   s       r   geglur     sC     jE**+"Q&&yy))*:1*=qAAr   c                d   t        t        | t                     r#t        | |      \  }}||z  }||z  }||z  }	|||	fS t        | |      \  }}t        j
                  j                  ||      }t        j
                  j                  ||      }t        j
                  j                  ||      }	|||	fS )a  
    GeGLU backward pass: computes gradients w.r.t. x (gate) and y (up projection)
    Given: geglu_out = gelu(x) * y, and dout = grad w.r.t. geglu_out
    Returns: (dx, dy, geglu_out) where:
    - dx = dout * y * d_gelu(x)
    - dy = dout * gelu(x)
    - geglu_out = gelu(x) * y
    )r   r"   r#   rl   r$   r%   r&   )
r   r   r=   r   r   dgelu_x_doutgelu_xrL   r   	geglu_outs
             r   dgeglur     s     jE**+0D9fAd]QJ	2y    1D9fYY''a8YY''5II..vq9	2y  r   )Nr~   z	silu-tanhrG   rP   rb   )r   zswiglu-tanhr   zswiglu_oai-tanhr   r   r   )gZd;?):r1   typingr   	functoolsr   cutlass.cuter$   cutlassr   r   r   cutlass.cutlass_dslr   r	   cutlass._mlir.dialectsr
   r   F32_or_F32x2r%   calc_packed_f32x2_opr   floatr   r*   r;   rA   rG   jitrM   rP   rT   rb   rl   ry   r{   r~   boolr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
act_fn_mapdact_fn_mapgate_fn_mapdgate_fn_mapr@   r   r   <module>r      s       0 0 . - w/00 II""
##  $(T 
EGO 
g 
 
 )-$ OL O< O O $(T N| Nl N N =Ad $g $W $w $ $
 !%$ VL V< V V 04

'

<%&
 
 
 $(T 5| 5l 5 5 04'
<%& 
 . -1d @ @| @ @2 04;;';
<%&; ;| %)d 
 
| 
 
 
6 >Bt *w *g *PW * 
 * !%$ 9L 9< 9 9 9>DT G G$ GVb G G 
 	%%
% <%&% %@ 
 !*%*%
*% 	*% <%&*% *%Z 48T 6l 6| 6l 6 6 9=$ ;< ;L ;< ; ;  	0"0"0" 0" <|340" 0"f  !."."." ."
 ." <|34." ."b 5:=CGD==$=-2== =$ 5:=CGD==$=-2== =  IN#"W[`d#"#"$#",8#"AF#"
<|34#" #"L IN"W[`d""$",8"AF"
<|34" "B 15$ 	8< 	8L 	8< 	8 	8 AE$$,8
<|34 D 37D 5\ 5l 5\ 5 5 AE$!!$!,8!
<|34! 
 !: 37D B\ Bl B\ B B AE$!!$!,8!
<|34! !D (
 ) & 'r   