
    ^j3                        d Z ddlZddlmZ ddlmZ d Zd Z G d dej                  j                        Z
d)d	Z G d
 dej                        Zd Zd Z G d dej                  j                        Zd)dZ G d dej                        Zd)defdZd Z G d dej                  j                        Zd)defdZ G d dej                        Zd Zd Z G d dej                  j                        Zd)dZ G d  d!ej                        Zd" Zd# Z G d$ d%ej                  j                        Z d)defd&Z! G d' d(ej                        Z"y)*a~   Activations (memory-efficient w/ custom autograd)

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

These activations are not compatible with jit scripting or ONNX export of the model, please use
basic versions of the activations.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalc                 J    | j                  t        j                  |             S N)multorchsigmoidxs    e/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/activations_me.py	swish_fwdr      s    55q!""    c                 N    t        j                  |       }||d| d|z
  z  z   z  z  S N   )r   r	   )r   grad_output	x_sigmoids      r   	swish_bwdr      s.    a I)q1I+>'>?@@r   c                   @    e Zd ZdZed        Zed        Zed        Zy)SwishAutoFnz optimised Swish w/ memory-efficient checkpoint
    Inspired by conversation btw Jeremy Howard & Adam Pazske
    https://twitter.com/jeremyphoward/status/1188251041835315200
    c                 H    | j                  d|| j                  d|            S )NMulSigmoid)op)gr   s     r   symboliczSwishAutoFn.symbolic   s     ttE1add9a011r   c                 :    | j                  |       t        |      S r   )save_for_backwardr   ctxr   s     r   forwardzSwishAutoFn.forward#   s    a |r   c                 8    | j                   d   }t        ||      S Nr   )saved_tensorsr   r    r   r   s      r   backwardzSwishAutoFn.backward(   s    a K((r   N)__name__
__module____qualname____doc__staticmethodr   r!   r&    r   r   r   r      sC     2 2   ) )r   r   c                 ,    t         j                  |       S r   r   applyr   inplaces     r   swish_mer2   .   s    Qr   c                   ,     e Zd Zddef fdZd Z xZS )SwishMer1   c                 "    t         |           y r   super__init__selfr1   	__class__s     r   r8   zSwishMe.__init__3       r   c                 ,    t         j                  |      S r   r.   r:   r   s     r   r!   zSwishMe.forward6   s      ##r   Fr'   r(   r)   boolr8   r!   __classcell__r;   s   @r   r4   r4   2   s     $r   r4   c                 p    | j                  t        j                  t        j                  |                   S r   )r   r   tanhFsoftplusr
   s    r   mish_fwdrH   :   s"    55AJJqM*++r   c                     t        j                  |       }t        j                  |       j	                         }|j                  || |z  d||z  z
  z  z         S r   )r   r	   rF   rG   rE   r   )r   r   r   	x_tanh_sps       r   mish_bwdrK   >   sM    a I

1""$I??9q9}I	<Q8Q'RRSSr   c                   0    e Zd ZdZed        Zed        Zy)
MishAutoFnz Mish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    A memory efficient variant of Mish
    c                 :    | j                  |       t        |      S r   )r   rH   r   s     r   r!   zMishAutoFn.forwardH   s    a {r   c                 8    | j                   d   }t        ||      S r#   )r$   rK   r%   s      r   r&   zMishAutoFn.backwardM   s    a ;''r   Nr'   r(   r)   r*   r+   r!   r&   r,   r   r   rM   rM   D   s/       ( (r   rM   c                 ,    t         j                  |       S r   rM   r/   r0   s     r   mish_merS   S   s    Ar   c                   ,     e Zd Zddef fdZd Z xZS )MishMer1   c                 "    t         |           y r   r6   r9   s     r   r8   zMishMe.__init__X   r<   r   c                 ,    t         j                  |      S r   rR   r>   s     r   r!   zMishMe.forward[   s    ""r   r?   r@   rC   s   @r   rU   rU   W   s     #r   rU   r1   c                 L    | dz   j                  dd      j                  d      S N   r      minmax      @clampdivr0   s     r   hard_sigmoid_fwdrc   _   s$    E==QA=&**2..r   c                 T    t        j                  |       | dk\  | dk  z  z  dz  }||z  S )N            @r_   )r   	ones_liker   r   ms      r   hard_sigmoid_bwdrj   c   s1    qCxAG45:A?r   c                   ,    e Zd Zed        Zed        Zy)HardSigmoidAutoFnc                 :    | j                  |       t        |      S r   )r   rc   r   s     r   r!   zHardSigmoidAutoFn.forwardi   s    a ""r   c                 8    | j                   d   }t        ||      S r#   )r$   rj   r%   s      r   r&   zHardSigmoidAutoFn.backwardn   s    a ;//r   N)r'   r(   r)   r+   r!   r&   r,   r   r   rl   rl   h   s(    # # 0 0r   rl   c                 ,    t         j                  |       S r   rl   r/   r0   s     r   hard_sigmoid_merq   t   s    ""1%%r   c                   ,     e Zd Zddef fdZd Z xZS )HardSigmoidMer1   c                 "    t         |           y r   r6   r9   s     r   r8   zHardSigmoidMe.__init__y   r<   r   c                 ,    t         j                  |      S r   rp   r>   s     r   r!   zHardSigmoidMe.forward|   s     &&q))r   r?   r@   rC   s   @r   rs   rs   x   s     *r   rs   c                 R    | | dz   j                  dd      j                  d      z  S rY   r`   r
   s    r   hard_swish_fwdrw      s)    A}}}*..r222r   c                     t        j                  |       | dk\  z  }t        j                  | dk\  | dk  z  | dz  dz   |      }||z  S )Nrf   re         ?r   rg   whererh   s      r   hard_swish_bwdr|      sJ    a2g&AQ#X!r'*QVb[!<A?r   c                   @    e Zd ZdZed        Zed        Zed        Zy)HardSwishAutoFnz'A memory efficient HardSwish activationc                 :    | j                  |       t        |      S r   )r   rw   r   s     r   r!   zHardSwishAutoFn.forward   s    a a  r   c                 8    | j                   d   }t        ||      S r#   )r$   r|   r%   s      r   r&   zHardSwishAutoFn.backward   s    a a--r   c                 <   | j                  d|| j                  dt        j                  dt        j                                    }| j                  d|| j                  dt        j                  dt        j                              | j                  dt        j                  dt        j                                    }| j                  d	|| j                  dt        j                  dt        j                                    }| j                  d
||      S )NAddConstantrZ   )dtype)value_tClipr   r[   Divr   )r   r   tensorfloat)r   r:   input	hardtanh_s       r   r   zHardSwishAutoFn.symbolic   s    UD!$$z5<<QVQ\Q\;]$"^_DDZaW\WbWbAc(dfgfjfjku  @E  @L  @L  MN  V[  Va  Va  @bfj  gc  d	DD	144
ELLQRZ_ZeZeDf4+gh	ttE4++r   N)r'   r(   r)   r*   r+   r!   r&   r   r,   r   r   r~   r~      s?    1! ! . . , ,r   r~   c                 ,    t         j                  |       S r   r~   r/   r0   s     r   hard_swish_mer      s      ##r   c                   ,     e Zd Zddef fdZd Z xZS )HardSwishMer1   c                 "    t         |           y r   r6   r9   s     r   r8   zHardSwishMe.__init__   r<   r   c                 ,    t         j                  |      S r   r   r>   s     r   r!   zHardSwishMe.forward   s    $$Q''r   r?   r@   rC   s   @r   r   r      s     (r   r   c                 :    d| z  | dz   j                  dd      z  S )Nry      r   r\   )ra   r
   s    r   hard_mish_fwdr      s"    7a!e]]qa]000r   c                     t        j                  |       | dk\  z  }t        j                  | dk\  | dk  z  | dz   |      }||z  S )Ng       g        g      ?rz   rh   s      r   hard_mish_bwdr      sF    a3h'AQ#X!r'*AFA6A?r   c                   0    e Zd ZdZed        Zed        Zy)HardMishAutoFnz A memory efficient variant of Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    c                 :    | j                  |       t        |      S r   )r   r   r   s     r   r!   zHardMishAutoFn.forward   s    a Qr   c                 8    | j                   d   }t        ||      S r#   )r$   r   r%   s      r   r&   zHardMishAutoFn.backward   s    a Q,,r   NrP   r,   r   r   r   r      s/         - -r   r   c                 ,    t         j                  |       S r   r   r/   r0   s     r   hard_mish_mer      s    ""r   c                   ,     e Zd Zddef fdZd Z xZS )
HardMishMer1   c                 "    t         |           y r   r6   r9   s     r   r8   zHardMishMe.__init__   r<   r   c                 ,    t         j                  |      S r   r   r>   s     r   r!   zHardMishMe.forward   s    ##A&&r   r?   r@   rC   s   @r   r   r      s     'r   r   r?   )#r*   r   r   torch.nnr   rF   r   r   autogradFunctionr   r2   Moduler4   rH   rK   rM   rS   rU   rA   rc   rj   rl   rq   rs   rw   r|   r~   r   r   r   r   r   r   r   r,   r   r   <module>r      s5  	   $#A
)%..)) )( $bii $,T((( (#RYY #/ /
	0// 	0& &*BII *3,enn-- ,($(")) (1-U^^,, - #T #' 'r   