Ë
    éÓ^j3  ã                   ó¨  — d Z ddlZddlmZ ddlmZ d„ Zd„ Z G d„ dej                  j                  «      Z
d)d	„Z G d
„ dej                  «      Zd„ Zd„ Z G d„ dej                  j                  «      Zd)d„Z G d„ dej                  «      Zd)defd„Zd„ Z G d„ dej                  j                  «      Zd)defd„Z G d„ dej                  «      Zd„ Zd„ Z G d„ dej                  j                  «      Zd)d„Z G d „ d!ej                  «      Zd"„ Zd#„ Z G d$„ d%ej                  j                  «      Z d)defd&„Z! G d'„ d(ej                  «      Z"y)*a~   Activations (memory-efficient w/ custom autograd)

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

These activations are not compatible with jit scripting or ONNX export of the model, please use
basic versions of the activations.

Hacked together by / Copyright 2020 Ross Wightman
é    N)Únn)Ú
functionalc                 óJ   — | j                  t        j                  | «      «      S ©N)ÚmulÚtorchÚsigmoid©Úxs    úe/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/activations_me.pyÚ	swish_fwdr      s   € Ø�5‰5”—‘˜qÓ!Ó"Ð"ó    c                 óN   — t        j                  | «      }||d| d|z
  z  z   z  z  S ©Né   )r   r	   )r   Úgrad_outputÚ	x_sigmoids      r   Ú	swish_bwdr      s.   € Ü—‘˜aÓ €IØ˜) q¨1°°I±Ñ+>Ñ'>Ñ?Ñ@Ð@r   c                   ó@   — e Zd ZdZed„ «       Zed„ «       Zed„ «       Zy)ÚSwishAutoFnz² optimised Swish w/ memory-efficient checkpoint
    Inspired by conversation btw Jeremy Howard & Adam Pazske
    https://twitter.com/jeremyphoward/status/1188251041835315200
    c                 óH   — | j                  d|| j                  d|«      «      S )NÚMulÚSigmoid)Úop)Úgr   s     r   ÚsymboliczSwishAutoFn.symbolic   s    € à�t‰t�E˜1˜aŸd™d 9¨aÓ0Ó1Ð1r   c                 ó:   — | j                  |«       t        |«      S r   )Úsave_for_backwardr   ©Úctxr   s     r   ÚforwardzSwishAutoFn.forward#   s   € à×Ñ˜aÔ Ü˜‹|Ðr   c                 ó8   — | j                   d   }t        ||«      S ©Nr   )Úsaved_tensorsr   ©r    r   r   s      r   ÚbackwardzSwishAutoFn.backward(   s   € à×Ñ˜aÑ ˆÜ˜˜KÓ(Ð(r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ústaticmethodr   r!   r&   © r   r   r   r      sC   „ ñð ñ2ó ð2ð ñó ðð ñ)ó ñ)r   r   c                 ó,   — t         j                  | «      S r   ©r   Úapply©r   Úinplaces     r   Úswish_mer2   .   s   € Ü×Ñ˜QÓÐr   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )ÚSwishMer1   c                 ó"   •— t         ‰| �  «        y r   ©ÚsuperÚ__init__©Úselfr1   Ú	__class__s     €r   r8   zSwishMe.__init__3   ó   ø€ Ü‰ÑÕr   c                 ó,   — t         j                  |«      S r   r.   ©r:   r   s     r   r!   zSwishMe.forward6   s   € Ü× Ñ  Ó#Ð#r   ©F©r'   r(   r)   Úboolr8   r!   Ú__classcell__©r;   s   @r   r4   r4   2   s   ø„ ñ õ ö$r   r4   c                 óp   — | j                  t        j                  t        j                  | «      «      «      S r   )r   r   ÚtanhÚFÚsoftplusr
   s    r   Úmish_fwdrH   :   s"   € Ø�5‰5”—‘œAŸJ™J q›MÓ*Ó+Ð+r   c                 ó²   — t        j                  | «      }t        j                  | «      j	                  «       }|j                  || |z  d||z  z
  z  z   «      S r   )r   r	   rF   rG   rE   r   )r   r   r   Ú	x_tanh_sps       r   Úmish_bwdrK   >   sM   € Ü—‘˜aÓ €IÜ—
‘
˜1“×"Ñ"Ó$€IØ�?‰?˜9 q¨9¡}¸¸IÈ	Ñ<QÑ8QÑ'RÑRÓSÐSr   c                   ó0   — e Zd ZdZed„ «       Zed„ «       Zy)Ú
MishAutoFnz‘ Mish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    A memory efficient variant of Mish
    c                 ó:   — | j                  |«       t        |«      S r   )r   rH   r   s     r   r!   zMishAutoFn.forwardH   s   € à×Ñ˜aÔ Ü˜‹{Ðr   c                 ó8   — | j                   d   }t        ||«      S r#   )r$   rK   r%   s      r   r&   zMishAutoFn.backwardM   s   € à×Ñ˜aÑ ˆÜ˜˜;Ó'Ð'r   N©r'   r(   r)   r*   r+   r!   r&   r,   r   r   rM   rM   D   s/   „ ñð ñó ðð ñ(ó ñ(r   rM   c                 ó,   — t         j                  | «      S r   ©rM   r/   r0   s     r   Úmish_merS   S   s   € Ü×Ñ˜AÓÐr   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )ÚMishMer1   c                 ó"   •— t         ‰| �  «        y r   r6   r9   s     €r   r8   zMishMe.__init__X   r<   r   c                 ó,   — t         j                  |«      S r   rR   r>   s     r   r!   zMishMe.forward[   s   € Ü×Ñ Ó"Ð"r   r?   r@   rC   s   @r   rU   rU   W   s   ø„ ñ õ ö#r   rU   r1   c                 óL   — | dz   j                  dd¬«      j                  d«      S ©Né   r   é   ©ÚminÚmaxç      @©ÚclampÚdivr0   s     r   Úhard_sigmoid_fwdrc   _   s$   € Ø�‰E�=‰=˜Q Aˆ=Ó&×*Ñ*¨2Ó.Ð.r   c                 óT   — t        j                  | «      | dk\  | dk  z  z  dz  }||z  S )Nç      Àç      @r_   )r   Ú	ones_like©r   r   Úms      r   Úhard_sigmoid_bwdrj   c   s1   € Ü�‰˜Ó˜q C™x¨A°©GÑ4Ñ5¸Ñ:€AØ˜‰?Ðr   c                   ó,   — e Zd Zed„ «       Zed„ «       Zy)ÚHardSigmoidAutoFnc                 ó:   — | j                  |«       t        |«      S r   )r   rc   r   s     r   r!   zHardSigmoidAutoFn.forwardi   s   € à×Ñ˜aÔ Ü Ó"Ð"r   c                 ó8   — | j                   d   }t        ||«      S r#   )r$   rj   r%   s      r   r&   zHardSigmoidAutoFn.backwardn   s   € à×Ñ˜aÑ ˆÜ  ;Ó/Ð/r   N)r'   r(   r)   r+   r!   r&   r,   r   r   rl   rl   h   s(   „ Øñ#ó ð#ð ñ0ó ñ0r   rl   c                 ó,   — t         j                  | «      S r   ©rl   r/   r0   s     r   Úhard_sigmoid_merq   t   s   € Ü×"Ñ" 1Ó%Ð%r   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )ÚHardSigmoidMer1   c                 ó"   •— t         ‰| �  «        y r   r6   r9   s     €r   r8   zHardSigmoidMe.__init__y   r<   r   c                 ó,   — t         j                  |«      S r   rp   r>   s     r   r!   zHardSigmoidMe.forward|   s   € Ü ×&Ñ& qÓ)Ð)r   r?   r@   rC   s   @r   rs   rs   x   s   ø„ ñ õ ö*r   rs   c                 óR   — | | dz   j                  dd¬«      j                  d«      z  S rY   r`   r
   s    r   Úhard_swish_fwdrw   €   s)   € Ø��A‘�}‰} ¨ˆ}Ó*×.Ñ.¨rÓ2Ñ2Ð2r   c                 óŽ   — t        j                  | «      | dk\  z  }t        j                  | dk\  | dk  z  | dz  dz   |«      }||z  S )Nrf   re   ç      à?©r   rg   Úwhererh   s      r   Úhard_swish_bwdr|   „   sJ   € Ü�‰˜Ó˜a 2™gÑ&€AÜ�‰�Q˜#‘X ! r¡'Ñ*¨Q°©V°b©[¸!Ó<€AØ˜‰?Ðr   c                   ó@   — e Zd ZdZed„ «       Zed„ «       Zed„ «       Zy)ÚHardSwishAutoFnz'A memory efficient HardSwish activationc                 ó:   — | j                  |«       t        |«      S r   )r   rw   r   s     r   r!   zHardSwishAutoFn.forwardŒ   s   € à×Ñ˜aÔ Ü˜aÓ Ð r   c                 ó8   — | j                   d   }t        ||«      S r#   )r$   r|   r%   s      r   r&   zHardSwishAutoFn.backward‘   s   € à×Ñ˜aÑ ˆÜ˜a Ó-Ð-r   c                 ó<  — | j                  d|| j                  dt        j                  dt        j                  ¬«      ¬«      «      }| j                  d|| j                  dt        j                  dt        j                  ¬«      ¬«      | j                  dt        j                  dt        j                  ¬«      ¬«      «      }| j                  d	|| j                  dt        j                  dt        j                  ¬«      ¬«      «      }| j                  d
||«      S )NÚAddÚConstantrZ   )Údtype)Úvalue_tÚClipr   r[   ÚDivr   )r   r   ÚtensorÚfloat)r   r:   ÚinputÚ	hardtanh_s       r   r   zHardSwishAutoFn.symbolic–   sú   € à—‘�U˜D !§$¡$ z¼5¿<¹<ÈÔQV×Q\ÑQ\Ô;] $Ó"^Ó_ˆØ—D‘D˜ ¨¯©¨ZÄÇÁÈaÔW\×WbÑWbÔAc¨Ó(dÐfg×fjÑfjÐkuô  @E÷  @Lñ  @Lð  MNô  V[÷  Vañ  Vaô  @bÐfjó  gcó  dˆ	Ø—D‘D˜ 	¨1¯4©4°
ÄEÇLÁLÐQRÔZ_×ZeÑZeÔDf¨4Ó+gÓhˆ	Ø�t‰t�E˜4 Ó+Ð+r   N)r'   r(   r)   r*   r+   r!   r&   r   r,   r   r   r~   r~   Š   s?   „ Ù1Øñ!ó ð!ð ñ.ó ð.ð ñ,ó ñ,r   r~   c                 ó,   — t         j                  | «      S r   ©r~   r/   r0   s     r   Úhard_swish_merŽ   ž   s   € Ü× Ñ  Ó#Ð#r   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )ÚHardSwishMer1   c                 ó"   •— t         ‰| �  «        y r   r6   r9   s     €r   r8   zHardSwishMe.__init__£   r<   r   c                 ó,   — t         j                  |«      S r   r�   r>   s     r   r!   zHardSwishMe.forward¦   s   € Ü×$Ñ$ QÓ'Ð'r   r?   r@   rC   s   @r   r�   r�   ¢   s   ø„ ñ õ ö(r   r�   c                 ó:   — d| z  | dz   j                  dd¬«      z  S )Nry   é   r   r\   )ra   r
   s    r   Úhard_mish_fwdr•   ª   s"   € Ø�‰7�a˜!‘e—]‘] q¨a�]Ó0Ñ0Ð0r   c                 óˆ   — t        j                  | «      | dk\  z  }t        j                  | dk\  | dk  z  | dz   |«      }||z  S )Ng       Àg        g      ð?rz   rh   s      r   Úhard_mish_bwdr—   ®   sF   € Ü�‰˜Ó˜a 3™hÑ'€AÜ�‰�Q˜#‘X ! r¡'Ñ*¨A°©F°AÓ6€AØ˜‰?Ðr   c                   ó0   — e Zd ZdZed„ «       Zed„ «       Zy)ÚHardMishAutoFnzÕ A memory efficient variant of Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    c                 ó:   — | j                  |«       t        |«      S r   )r   r•   r   s     r   r!   zHardMishAutoFn.forward¹   s   € à×Ñ˜aÔ Ü˜QÓÐr   c                 ó8   — | j                   d   }t        ||«      S r#   )r$   r—   r%   s      r   r&   zHardMishAutoFn.backward¾   s   € à×Ñ˜aÑ ˆÜ˜Q Ó,Ð,r   NrP   r,   r   r   r™   r™   ´   s/   „ ñð ñ ó ð ð ñ-ó ñ-r   r™   c                 ó,   — t         j                  | «      S r   ©r™   r/   r0   s     r   Úhard_mish_merž   Ä   s   € Ü×Ñ Ó"Ð"r   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )Ú
HardMishMer1   c                 ó"   •— t         ‰| �  «        y r   r6   r9   s     €r   r8   zHardMishMe.__init__É   r<   r   c                 ó,   — t         j                  |«      S r   r�   r>   s     r   r!   zHardMishMe.forwardÌ   s   € Ü×#Ñ# AÓ&Ð&r   r?   r@   rC   s   @r   r    r    È   s   ø„ ñ õ ö'r   r    r?   )#r*   r   r   Útorch.nnr   rF   r   r   ÚautogradÚFunctionr   r2   ÚModuler4   rH   rK   rM   rS   rU   rA   rc   rj   rl   rq   rs   rw   r|   r~   rŽ   r�   r•   r—   r™   rž   r    r,   r   r   ú<module>r§      s5  ðñ	ó Ý Ý $ò#òAô
)�%—.‘.×)Ñ)ô )ó( ô$ˆb�i‰iô $ò,òTô(�—‘×(Ñ(ô (óô#ˆR�Y‰Yô #ñ/ ó /òô
	0˜Ÿ™×/Ñ/ô 	0ñ& ó &ô*�B—I‘Iô *ò3òô,�e—n‘n×-Ñ-ô ,ó($ô(�"—)‘)ô (ò1òô-�U—^‘^×,Ñ,ô -ñ #˜Tó #ô'�—‘õ 'r   