
    kh                        d Z ddlZddlmZ ddlmZ d Zd Z G d dej                  j                        Z
d)d	Z G d
 dej                        Zd Zd Z G d dej                  j                        Zd)dZ G d dej                        Zd)defdZd Z G d dej                  j                        Zd)defdZ G d dej                        Zd Zd Z G d dej                  j                        Zd)dZ G d  d!ej                        Zd" Zd# Z G d$ d%ej                  j                        Z d)defd&Z! G d' d(ej                        Z"y)*a~   Activations (memory-efficient w/ custom autograd)

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

These activations are not compatible with jit scripting or ONNX export of the model, please use
basic versions of the activations.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalc                 J    | j                  t        j                  |             S N)multorchsigmoidxs    V/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/layers/activations_me.py	swish_fwdr      s    55q!""    c                 N    t        j                  |       }||d| d|z
  z  z   z  z  S N   )r   r	   )r   grad_output	x_sigmoids      r   	swish_bwdr      s.    a I)q1I+>'>?@@r   c                   @    e Zd ZdZed        Zed        Zed        Zy)SwishAutoFnz optimised Swish w/ memory-efficient checkpoint
    Inspired by conversation btw Jeremy Howard & Adam Pazske
    https://twitter.com/jeremyphoward/status/1188251041835315200
    c                 H    | j                  d|| j                  d|            S )NMulSigmoid)op)gr   s     r   symboliczSwishAutoFn.symbolic   s     ttE1add9a011r   c                 :    | j                  |       t        |      S r   )save_for_backwardr   ctxr   s     r   forwardzSwishAutoFn.forward#   s    a |r   c                 8    | j                   d   }t        ||      S Nr   )saved_tensorsr   r    r   r   s      r   backwardzSwishAutoFn.backward(   s    a K((r   N)__name__
__module____qualname____doc__staticmethodr   r!   r&    r   r   r   r      sC     2 2   ) )r   r   c                 ,    t         j                  |       S r   r   applyr   inplaces     r   swish_mer2   .   s    Qr   c                   ,     e Zd Zddef fdZd Z xZS )SwishMer1   c                 *    t         t        |           y r   )superr4   __init__selfr1   	__class__s     r   r7   zSwishMe.__init__3   s    gt%'r   c                 ,    t         j                  |      S r   r.   r9   r   s     r   r!   zSwishMe.forward6   s      ##r   Fr'   r(   r)   boolr7   r!   __classcell__r:   s   @r   r4   r4   2   s    ( ($r   r4   c                 p    | j                  t        j                  t        j                  |                   S r   )r   r   tanhFsoftplusr
   s    r   mish_fwdrF   :   s"    55AJJqM*++r   c                     t        j                  |       }t        j                  |       j	                         }|j                  || |z  d||z  z
  z  z         S r   )r   r	   rD   rE   rC   r   )r   r   r   	x_tanh_sps       r   mish_bwdrI   >   sM    a I

1""$I??9q9}I	<Q8Q'RRSSr   c                   0    e Zd ZdZed        Zed        Zy)
MishAutoFnz Mish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    A memory efficient variant of Mish
    c                 :    | j                  |       t        |      S r   )r   rF   r   s     r   r!   zMishAutoFn.forwardH   s    a {r   c                 8    | j                   d   }t        ||      S r#   )r$   rI   r%   s      r   r&   zMishAutoFn.backwardM   s    a ;''r   Nr'   r(   r)   r*   r+   r!   r&   r,   r   r   rK   rK   D   s/       ( (r   rK   c                 ,    t         j                  |       S r   rK   r/   r0   s     r   mish_merQ   S   s    Ar   c                   ,     e Zd Zddef fdZd Z xZS )MishMer1   c                 *    t         t        |           y r   )r6   rS   r7   r8   s     r   r7   zMishMe.__init__X   s    fd$&r   c                 ,    t         j                  |      S r   rP   r<   s     r   r!   zMishMe.forward[   s    ""r   r=   r>   rA   s   @r   rS   rS   W   s    ' '#r   rS   r1   c                 L    | dz   j                  dd      j                  d      S N   r      minmax      @clampdivr0   s     r   hard_sigmoid_fwdra   _   s$    E==QA=&**2..r   c                 T    t        j                  |       | dk\  | dk  z  z  dz  }||z  S )N            @r]   )r   	ones_liker   r   ms      r   hard_sigmoid_bwdrh   c   s1    qCxAG45:A?r   c                   ,    e Zd Zed        Zed        Zy)HardSigmoidAutoFnc                 :    | j                  |       t        |      S r   )r   ra   r   s     r   r!   zHardSigmoidAutoFn.forwardi   s    a ""r   c                 8    | j                   d   }t        ||      S r#   )r$   rh   r%   s      r   r&   zHardSigmoidAutoFn.backwardn   s    a ;//r   N)r'   r(   r)   r+   r!   r&   r,   r   r   rj   rj   h   s(    # # 0 0r   rj   c                 ,    t         j                  |       S r   rj   r/   r0   s     r   hard_sigmoid_mero   t   s    ""1%%r   c                   ,     e Zd Zddef fdZd Z xZS )HardSigmoidMer1   c                 *    t         t        |           y r   )r6   rq   r7   r8   s     r   r7   zHardSigmoidMe.__init__y   s    mT+-r   c                 ,    t         j                  |      S r   rn   r<   s     r   r!   zHardSigmoidMe.forward|   s     &&q))r   r=   r>   rA   s   @r   rq   rq   x   s    . .*r   rq   c                 R    | | dz   j                  dd      j                  d      z  S rW   r^   r
   s    r   hard_swish_fwdru      s)    A}}}*..r222r   c                     t        j                  |       | dk\  z  }t        j                  | dk\  | dk  z  | dz  dz   |      }||z  S )Nrd   rc         ?r   re   whererf   s      r   hard_swish_bwdrz      sJ    a2g&AQ#X!r'*QVb[!<A?r   c                   @    e Zd ZdZed        Zed        Zed        Zy)HardSwishAutoFnz'A memory efficient HardSwish activationc                 :    | j                  |       t        |      S r   )r   ru   r   s     r   r!   zHardSwishAutoFn.forward   s    a a  r   c                 8    | j                   d   }t        ||      S r#   )r$   rz   r%   s      r   r&   zHardSwishAutoFn.backward   s    a a--r   c                 <   | j                  d|| j                  dt        j                  dt        j                                    }| j                  d|| j                  dt        j                  dt        j                              | j                  dt        j                  dt        j                                    }| j                  d	|| j                  dt        j                  dt        j                                    }| j                  d
||      S )NAddConstantrX   )dtype)value_tClipr   rY   Divr   )r   r   tensorfloat)r   r9   input	hardtanh_s       r   r   zHardSwishAutoFn.symbolic   s    UD!$$z5<<QVQ\Q\;]$"^_DDZaW\WbWbAc(dfgfjfjku  @E  @L  @L  MN  V[  Va  Va  @bfj  gc  d	DD	144
ELLQRZ_ZeZeDf4+gh	ttE4++r   N)r'   r(   r)   r*   r+   r!   r&   r   r,   r   r   r|   r|      s?    1! ! . . , ,r   r|   c                 ,    t         j                  |       S r   r|   r/   r0   s     r   hard_swish_mer      s      ##r   c                   ,     e Zd Zddef fdZd Z xZS )HardSwishMer1   c                 *    t         t        |           y r   )r6   r   r7   r8   s     r   r7   zHardSwishMe.__init__   s    k4)+r   c                 ,    t         j                  |      S r   r   r<   s     r   r!   zHardSwishMe.forward   s    $$Q''r   r=   r>   rA   s   @r   r   r      s    , ,(r   r   c                 :    d| z  | dz   j                  dd      z  S )Nrw      r   rZ   )r_   r
   s    r   hard_mish_fwdr      s"    7a!e]]qa]000r   c                     t        j                  |       | dk\  z  }t        j                  | dk\  | dk  z  | dz   |      }||z  S )Ng       g        g      ?rx   rf   s      r   hard_mish_bwdr      sF    a3h'AQ#X!r'*AFA6A?r   c                   0    e Zd ZdZed        Zed        Zy)HardMishAutoFnz A memory efficient variant of Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    c                 :    | j                  |       t        |      S r   )r   r   r   s     r   r!   zHardMishAutoFn.forward   s    a Qr   c                 8    | j                   d   }t        ||      S r#   )r$   r   r%   s      r   r&   zHardMishAutoFn.backward   s    a Q,,r   NrN   r,   r   r   r   r      s/         - -r   r   c                 ,    t         j                  |       S r   r   r/   r0   s     r   hard_mish_mer      s    ""r   c                   ,     e Zd Zddef fdZd Z xZS )
HardMishMer1   c                 *    t         t        |           y r   )r6   r   r7   r8   s     r   r7   zHardMishMe.__init__   s    j$(*r   c                 ,    t         j                  |      S r   r   r<   s     r   r!   zHardMishMe.forward   s    ##A&&r   r=   r>   rA   s   @r   r   r      s    + +'r   r   r=   )#r*   r   r   torch.nnr   rD   r   r   autogradFunctionr   r2   Moduler4   rF   rI   rK   rQ   rS   r?   ra   rh   rj   ro   rq   ru   rz   r|   r   r   r   r   r   r   r   r,   r   r   <module>r      s5  	   $#A
)%..)) )( $bii $,T((( (#RYY #/ /
	0// 	0& &*BII *3,enn-- ,($(")) (1-U^^,, - #T #' 'r   