
    ^j                        d Z ddlZddlmZ ddlmZ d(defdZ G d dej                        Z	d(defd	Z
 G d
 dej                        Zd(defdZ G d dej                        Zd(defdZ G d dej                        Zd(defdZ G d dej                        Zd(defdZ G d dej                        Zd(defdZ G d dej                        Z G d dej,                        Zd(dej.                  dedej.                  fdZ G d  d!ej                        Zd(dej.                  dedej.                  fd"Z G d# d$ej                        Zd(dej.                  dedej.                  fd%Z G d& d'ej                        Zy))z Activations

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalinplacec                     |r| j                  | j                               S | j                  | j                               S )z;Swish - Described in: https://arxiv.org/abs/1710.05941
    )mul_sigmoidmulxr   s     b/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/activations.pyswishr      s/     #*166!))+AquuQYY[/AA    c                   ,     e Zd Zddef fdZd Z xZS )Swishr   c                 0    t         |           || _        y Nsuper__init__r   selfr   	__class__s     r   r   zSwish.__init__       r   c                 .    t        || j                        S r   )r   r   r   r   s     r   forwardzSwish.forward   s    Q%%r   F__name__
__module____qualname__boolr   r   __classcell__r   s   @r   r   r      s     &r   r   c                 f    | j                  t        j                  |       j                               S )zMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    NOTE: I don't have a working inplace variant
    )r	   Fsoftplustanhr
   s     r   mishr)      s$     55A##%&&r   c                   0     e Zd ZdZddef fdZd Z xZS )MishziMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    r   c                 "    t         |           y r   r   r   r   s     r   r   zMish.__init__'       r   c                     t        |      S r   )r)   r   s     r   r   zMish.forward*   s    Awr   r   )r   r    r!   __doc__r"   r   r   r#   r$   s   @r   r+   r+   $   s     r   r+   c                 F    |r| j                         S | j                         S r   )sigmoid_r   r
   s     r   r   r   .   s    "1::<3		3r   c                   ,     e Zd Zddef fdZd Z xZS )Sigmoidr   c                 0    t         |           || _        y r   r   r   s     r   r   zSigmoid.__init__4   r   r   c                 Z    | j                   r|j                         S |j                         S r   )r   r2   r   r   s     r   r   zSigmoid.forward8   s    #||qzz|<<r   r   r   r$   s   @r   r4   r4   3   s     =r   r4   c                 F    |r| j                         S | j                         S r   )tanh_r(   r
   s     r   r(   r(   <   s    1779-QVVX-r   c                   ,     e Zd Zddef fdZd Z xZS )Tanhr   c                 0    t         |           || _        y r   r   r   s     r   r   zTanh.__init__B   r   r   c                 Z    | j                   r|j                         S |j                         S r   )r   r8   r(   r   s     r   r   zTanh.forwardF   s     LLqwwy6affh6r   r   r   r$   s   @r   r:   r:   A   s     7r   r:   c                     t        j                  | dz         j                  d      }|r| j                  |      S | j	                  |      S )N      @      @)r&   relu6div_r   r	   )r   r   inners      r   
hard_swishrC   J   s;    GGAFO  $E#166%=5u5r   c                   ,     e Zd Zddef fdZd Z xZS )	HardSwishr   c                 0    t         |           || _        y r   r   r   s     r   r   zHardSwish.__init__P   r   r   c                 .    t        || j                        S r   )rC   r   r   s     r   r   zHardSwish.forwardT   s    !T\\**r   r   r   r$   s   @r   rE   rE   O   s     +r   rE   c                     |r0| j                  d      j                  dd      j                  d      S t        j                  | dz         dz  S )Nr>   g        r?   )add_clamp_rA   r&   r@   r
   s     r   hard_sigmoidrK   X   sB    vvbz  R(--b11wwq2v##r   c                   ,     e Zd Zddef fdZd Z xZS )HardSigmoidr   c                 0    t         |           || _        y r   r   r   s     r   r   zHardSigmoid.__init__`   r   r   c                 .    t        || j                        S r   )rK   r   r   s     r   r   zHardSigmoid.forwardd   s    At||,,r   r   r   r$   s   @r   rM   rM   _   s     -r   rM   c                     |r(| j                  d| dz   j                  dd      z        S d| z  | dz   j                  dd      z  S )z Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    g      ?   r   )minmax)r   clampr
   s     r   	hard_mishrU   h   sL    
 vvcQUMMaQM7788Qw!a%1!444r   c                   ,     e Zd Zddef fdZd Z xZS )HardMishr   c                 0    t         |           || _        y r   r   r   s     r   r   zHardMish.__init__t   r   r   c                 .    t        || j                        S r   )rU   r   r   s     r   r   zHardMish.forwardx   s    DLL))r   r   r   r$   s   @r   rW   rW   s   s     *r   rW   c            	       n     e Zd ZdZd
dedededdf fdZdej                  dej                  fd	Z
 xZS )PReLUz)Applies PReLU (w/ dummy inplace arg)
    num_parametersinitr   returnNc                 (    t         |   ||       y )N)r\   r]   r-   )r   r\   r]   r   r   s       r   r   zPReLU.__init__   s    TBr   inputc                 B    t        j                  || j                        S r   )r&   preluweightr   r`   s     r   r   zPReLU.forward   s    wwudkk**r   )   g      ?F)r   r    r!   r0   intfloatr"   r   torchTensorr   r#   r$   s   @r   r[   r[   |   sJ    Cs Ce CT C^b C+U\\ +ell +r   r[   r   r^   c                 ,    t        j                  |       S r   r&   gelur
   s     r   rl   rl      s    66!9r   c                   b     e Zd ZdZddef fdZdej                  dej                  fdZ xZ	S )GELULApplies the Gaussian Error Linear Units function (w/ dummy inplace arg)
    r   c                 "    t         |           y r   r-   r   s     r   r   zGELU.__init__   r.   r   r`   r^   c                 ,    t        j                  |      S r   rk   rd   s     r   r   zGELU.forward   s    vve}r   r   
r   r    r!   r0   r"   r   rh   ri   r   r#   r$   s   @r   rn   rn      s/     U\\ ell r   rn   c                 0    t        j                  | d      S Nr(   )approximaterk   r
   s     r   	gelu_tanhrv      s    66!((r   c                   b     e Zd ZdZddef fdZdej                  dej                  fdZ xZ	S )GELUTanhro   r   c                 "    t         |           y r   r-   r   s     r   r   zGELUTanh.__init__   r.   r   r`   r^   c                 0    t        j                  |d      S rt   rk   rd   s     r   r   zGELUTanh.forward   s    vve00r   r   rr   r$   s   @r   rx   rx      s/     1U\\ 1ell 1r   rx   c                 8    | t        j                  d| z        z  S )NgZd;?)rh   r   r
   s     r   
quick_gelur|      s    u}}UQY'''r   c                   b     e Zd ZdZddef fdZdej                  dej                  fdZ xZ	S )	QuickGELUro   r   c                 "    t         |           y r   r-   r   s     r   r   zQuickGELU.__init__   r.   r   r`   r^   c                     t        |      S r   )r|   rd   s     r   r   zQuickGELU.forward   s    %  r   r   rr   r$   s   @r   r~   r~      s/     !U\\ !ell !r   r~   r   )r0   rh   r   torch.nnr   r&   r"   r   Moduler   r)   r+   r   r4   r(   r:   rC   rE   rK   rM   rU   rW   r[   ri   rl   rn   rv   rx   r|   r~    r   r   <module>r      s     $Bd B&BII &'T '299 4 4
=bii =.T .
7299 764 6
+		 +$T $-")) -5$ 5*ryy *+BHH +ELL 4 ELL 299 ) ) ) )1ryy 1(%,, ( (%,, (!		 !r   