Ë
    éÓ^j„  ã                   ó  — d Z ddlZddlmZ ddlmZ d(defd„Z G d„ dej                  «      Z	d(defd	„Z
 G d
„ dej                  «      Zd(defd„Z G d„ dej                  «      Zd(defd„Z G d„ dej                  «      Zd(defd„Z G d„ dej                  «      Zd(defd„Z G d„ dej                  «      Zd(defd„Z G d„ dej                  «      Z G d„ dej,                  «      Zd(dej.                  dedej.                  fd„Z G d „ d!ej                  «      Zd(dej.                  dedej.                  fd"„Z G d#„ d$ej                  «      Zd(dej.                  dedej.                  fd%„Z G d&„ d'ej                  «      Zy))zÔ Activations

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

Hacked together by / Copyright 2020 Ross Wightman
é    N)Únn)Ú
functionalÚinplacec                 ó‚   — |r| j                  | j                  «       «      S | j                  | j                  «       «      S )z;Swish - Described in: https://arxiv.org/abs/1710.05941
    )Úmul_ÚsigmoidÚmul©Úxr   s     úb/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/activations.pyÚswishr      s/   € ñ #*ˆ1�6‰6�!—)‘)“+ÓÐA¨q¯u©u°Q·Y±Y³[Ó/AÐAó    c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )ÚSwishr   c                 ó0   •— t         ‰| �  «        || _        y ©N©ÚsuperÚ__init__r   ©Úselfr   Ú	__class__s     €r   r   zSwish.__init__   ó   ø€ Ü‰ÑÔØˆ�r   c                 ó.   — t        || j                  «      S r   )r   r   ©r   r   s     r   ÚforwardzSwish.forward   s   € Ü�Q˜Ÿ™Ó%Ð%r   ©F©Ú__name__Ú
__module__Ú__qualname__Úboolr   r   Ú__classcell__©r   s   @r   r   r      s   ø„ ñ õ ö&r   r   c                 óf   — | j                  t        j                  | «      j                  «       «      S )zšMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    NOTE: I don't have a working inplace variant
    )r	   ÚFÚsoftplusÚtanhr
   s     r   Úmishr)      s$   € ð �5‰5”—‘˜A“×#Ñ#Ó%Ó&Ð&r   c                   ó0   ‡ — e Zd ZdZddefˆ fd„Zd„ Zˆ xZS )ÚMishziMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    r   c                 ó"   •— t         ‰| �  «        y r   ©r   r   r   s     €r   r   zMish.__init__'   ó   ø€ Ü‰ÑÕr   c                 ó   — t        |«      S r   )r)   r   s     r   r   zMish.forward*   s   € Ü�A‹wˆr   r   )r   r    r!   Ú__doc__r"   r   r   r#   r$   s   @r   r+   r+   $   s   ø„ ññ õ ör   r+   c                 óF   — |r| j                  «       S | j                  «       S r   )Úsigmoid_r   r
   s     r   r   r   .   s   € Ù"ˆ1�:‰:‹<Ð3¨¯	©	«Ð3r   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )ÚSigmoidr   c                 ó0   •— t         ‰| �  «        || _        y r   r   r   s     €r   r   zSigmoid.__init__4   r   r   c                 óZ   — | j                   r|j                  «       S |j                  «       S r   )r   r2   r   r   s     r   r   zSigmoid.forward8   s   € Ø#Ÿ|š|ˆq�z‰z‹|Ð<°·±³Ð<r   r   r   r$   s   @r   r4   r4   3   s   ø„ ñ õ ö=r   r4   c                 óF   — |r| j                  «       S | j                  «       S r   )Útanh_r(   r
   s     r   r(   r(   <   s   € Ùˆ1�7‰7‹9Ð- Q§V¡V£XÐ-r   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )ÚTanhr   c                 ó0   •— t         ‰| �  «        || _        y r   r   r   s     €r   r   zTanh.__init__B   r   r   c                 óZ   — | j                   r|j                  «       S |j                  «       S r   )r   r8   r(   r   s     r   r   zTanh.forwardF   s   € Ø ŸLšLˆq�w‰w‹yÐ6¨a¯f©f«hÐ6r   r   r   r$   s   @r   r:   r:   A   s   ø„ ñ õ ö7r   r:   c                 ó˜   — t        j                  | dz   «      j                  d«      }|r| j                  |«      S | j	                  |«      S )Nç      @ç      @)r&   Úrelu6Údiv_r   r	   )r   r   Úinners      r   Ú
hard_swishrC   J   s;   € Ü�G‰G�A˜‘F‹O× Ñ  Ó$€EÙ#ˆ1�6‰6�%‹=Ð5¨¯©¨u«Ð5r   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )Ú	HardSwishr   c                 ó0   •— t         ‰| �  «        || _        y r   r   r   s     €r   r   zHardSwish.__init__P   r   r   c                 ó.   — t        || j                  «      S r   )rC   r   r   s     r   r   zHardSwish.forwardT   s   € Ü˜!˜TŸ\™\Ó*Ð*r   r   r   r$   s   @r   rE   rE   O   s   ø„ ñ õ ö+r   rE   c                 óœ   — |r0| j                  d«      j                  dd«      j                  d«      S t        j                  | dz   «      dz  S )Nr>   g        r?   )Úadd_Úclamp_rA   r&   r@   r
   s     r   Úhard_sigmoidrK   X   sB   € ÙØ�v‰v�b‹z× Ñ   RÓ(×-Ñ-¨bÓ1Ð1ä�w‰w�q˜2‘v‹ Ñ#Ð#r   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )ÚHardSigmoidr   c                 ó0   •— t         ‰| �  «        || _        y r   r   r   s     €r   r   zHardSigmoid.__init__`   r   r   c                 ó.   — t        || j                  «      S r   )rK   r   r   s     r   r   zHardSigmoid.forwardd   s   € Ü˜A˜tŸ|™|Ó,Ð,r   r   r   r$   s   @r   rM   rM   _   s   ø„ ñ õ ö-r   rM   c                 óŽ   — |r(| j                  d| dz   j                  dd¬«      z  «      S d| z  | dz   j                  dd¬«      z  S )z· Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    g      à?é   r   )ÚminÚmax)r   Úclampr
   s     r   Ú	hard_mishrU   h   sL   € ñ
 Ø�v‰v�c˜Q ™UŸM™M¨a°Q˜MÓ7Ñ7Ó8Ð8à�Q‰w˜!˜a™%Ÿ™¨1°!˜Ó4Ñ4Ð4r   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )ÚHardMishr   c                 ó0   •— t         ‰| �  «        || _        y r   r   r   s     €r   r   zHardMish.__init__t   r   r   c                 ó.   — t        || j                  «      S r   )rU   r   r   s     r   r   zHardMish.forwardx   s   € Ü˜˜DŸL™LÓ)Ð)r   r   r   r$   s   @r   rW   rW   s   s   ø„ ñ õ ö*r   rW   c            	       ón   ‡ — e Zd ZdZd
dedededdfˆ fd„Zdej                  dej                  fd	„Z
ˆ xZS )ÚPReLUz)Applies PReLU (w/ dummy inplace arg)
    Únum_parametersÚinitr   ÚreturnNc                 ó(   •— t         ‰| �  ||¬«       y )N)r\   r]   r-   )r   r\   r]   r   r   s       €r   r   zPReLU.__init__   s   ø€ Ü‰Ñ¨¸TÐÕBr   Úinputc                 óB   — t        j                  || j                  «      S r   )r&   ÚpreluÚweight©r   r`   s     r   r   zPReLU.forward‚   s   € Ü�w‰w�u˜dŸk™kÓ*Ð*r   )é   g      Ð?F)r   r    r!   r0   ÚintÚfloatr"   r   ÚtorchÚTensorr   r#   r$   s   @r   r[   r[   |   sJ   ø„ ññC sð C°eð CÈTð CÐ^bõ Cð+˜UŸ\™\ð +¨e¯l©l÷ +r   r[   r   r^   c                 ó,   — t        j                  | «      S r   ©r&   Úgelur
   s     r   rl   rl   †   s   € Ü�6‰6�!‹9Ðr   c                   ób   ‡ — e Zd ZdZddefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚGELUúLApplies the Gaussian Error Linear Units function (w/ dummy inplace arg)
    r   c                 ó"   •— t         ‰| �  «        y r   r-   r   s     €r   r   zGELU.__init__�   r.   r   r`   r^   c                 ó,   — t        j                  |«      S r   rk   rd   s     r   r   zGELU.forward�   s   € Ü�v‰v�e‹}Ðr   r   ©
r   r    r!   r0   r"   r   rh   ri   r   r#   r$   s   @r   rn   rn   Š   s/   ø„ ññ õ ð˜UŸ\™\ð ¨e¯l©l÷ r   rn   c                 ó0   — t        j                  | d¬«      S ©Nr(   )Úapproximaterk   r
   s     r   Ú	gelu_tanhrv   ”   s   € Ü�6‰6�! Ô(Ð(r   c                   ób   ‡ — e Zd ZdZddefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚGELUTanhro   r   c                 ó"   •— t         ‰| �  «        y r   r-   r   s     €r   r   zGELUTanh.__init__›   r.   r   r`   r^   c                 ó0   — t        j                  |d¬«      S rt   rk   rd   s     r   r   zGELUTanh.forwardž   s   € Ü�v‰v�e¨Ô0Ð0r   r   rr   r$   s   @r   rx   rx   ˜   s/   ø„ ññ õ ð1˜UŸ\™\ð 1¨e¯l©l÷ 1r   rx   c                 ó8   — | t        j                  d| z  «      z  S )Ng¬Zd;û?)rh   r   r
   s     r   Ú
quick_gelur|   ¢   s   € ØŒu�}‰}˜U Q™YÓ'Ñ'Ð'r   c                   ób   ‡ — e Zd ZdZddefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )Ú	QuickGELUro   r   c                 ó"   •— t         ‰| �  «        y r   r-   r   s     €r   r   zQuickGELU.__init__©   r.   r   r`   r^   c                 ó   — t        |«      S r   )r|   rd   s     r   r   zQuickGELU.forward¬   s   € Ü˜%Ó Ð r   r   rr   r$   s   @r   r~   r~   ¦   s/   ø„ ññ õ ð!˜UŸ\™\ð !¨e¯l©l÷ !r   r~   r   )r0   rh   r   Útorch.nnr   r&   r"   r   ÚModuler   r)   r+   r   r4   r(   r:   rC   rE   rK   rM   rU   rW   r[   ri   rl   rn   rv   rx   r|   r~   © r   r   ú<module>r„      s�  ðñó Ý Ý $ñB�dó Bô&ˆB�I‰Iô &ñ'�Tó 'ôˆ2�9‰9ô ñ4˜ó 4ô
=ˆb�i‰iô =ñ.�Tó .ô
7ˆ2�9‰9ô 7ñ6˜4ó 6ô
+�—	‘	ô +ñ$˜Tó $ô-�"—)‘)ô -ñ5˜$ó 5ô*ˆr�y‰yô *ô+ˆB�H‰Hô +ñˆE�L‰Lð  4ð °E·L±Ló ôˆ2�9‰9ô ñ)�—‘ð )¨ð )¸¿¹ó )ô1ˆr�y‰yô 1ñ(�%—,‘,ð (¨ð (¸%¿,¹,ó (ô!�—	‘	õ !r   