
    kh                        d Z ddlZddlmZ ddlmZ d(defdZ G d dej                        Z	d(defd	Z
 G d
 dej                        Zd(defdZ G d dej                        Zd(defdZ G d dej                        Zd(defdZ G d dej                        Zd(defdZ G d dej                        Zd(defdZ G d dej                        Z G d dej,                        Zd(dej.                  dedej.                  fdZ G d  d!ej                        Zd(dej.                  dedej.                  fd"Z G d# d$ej                        Zd(dej.                  dedej.                  fd%Z G d& d'ej                        Zy))z Activations

A collection of activations fn and modules with a common interface so that they can
easily be swapped. All have an `inplace` arg even if not used.

Hacked together by / Copyright 2020 Ross Wightman
    N)nn)
functionalinplacec                     |r| j                  | j                               S | j                  | j                               S )z;Swish - Described in: https://arxiv.org/abs/1710.05941
    )mul_sigmoidmulxr   s     S/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/layers/activations.pyswishr      s/     #*166!))+AquuQYY[/AA    c                   ,     e Zd Zddef fdZd Z xZS )Swishr   c                 8    t         t        |           || _        y N)superr   __init__r   selfr   	__class__s     r   r   zSwish.__init__   s    eT#%r   c                 .    t        || j                        S r   )r   r   r   r   s     r   forwardzSwish.forward   s    Q%%r   F__name__
__module____qualname__boolr   r   __classcell__r   s   @r   r   r      s     &r   r   c                 f    | j                  t        j                  |       j                               S )zMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    NOTE: I don't have a working inplace variant
    )r	   Fsoftplustanhr
   s     r   mishr'      s$     55A##%&&r   c                   0     e Zd ZdZddef fdZd Z xZS )MishziMish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681
    r   c                 *    t         t        |           y r   )r   r)   r   r   s     r   r   zMish.__init__'       dD"$r   c                     t        |      S r   )r'   r   s     r   r   zMish.forward*   s    Awr   r   )r   r   r   __doc__r    r   r   r!   r"   s   @r   r)   r)   $   s    % %r   r)   c                 F    |r| j                         S | j                         S r   )sigmoid_r   r
   s     r   r   r   .   s    "1::<3		3r   c                   ,     e Zd Zddef fdZd Z xZS )Sigmoidr   c                 8    t         t        |           || _        y r   )r   r1   r   r   r   s     r   r   zSigmoid.__init__4   s    gt%'r   c                 Z    | j                   r|j                         S |j                         S r   )r   r/   r   r   s     r   r   zSigmoid.forward8   s    #||qzz|<<r   r   r   r"   s   @r   r1   r1   3   s     =r   r1   c                 F    |r| j                         S | j                         S r   )tanh_r&   r
   s     r   r&   r&   <   s    1779-QVVX-r   c                   ,     e Zd Zddef fdZd Z xZS )Tanhr   c                 8    t         t        |           || _        y r   )r   r7   r   r   r   s     r   r   zTanh.__init__B   s    dD"$r   c                 Z    | j                   r|j                         S |j                         S r   )r   r5   r&   r   s     r   r   zTanh.forwardF   s     LLqwwy6affh6r   r   r   r"   s   @r   r7   r7   A   s     7r   r7   c                     t        j                  | dz         j                  d      }|r| j                  |      S | j	                  |      S )N      @      @)r$   relu6div_r   r	   )r   r   inners      r   
hard_swishr@   J   s;    GGAFO  $E#166%=5u5r   c                   ,     e Zd Zddef fdZd Z xZS )	HardSwishr   c                 8    t         t        |           || _        y r   )r   rB   r   r   r   s     r   r   zHardSwish.__init__P   s    i')r   c                 .    t        || j                        S r   )r@   r   r   s     r   r   zHardSwish.forwardT   s    !T\\**r   r   r   r"   s   @r   rB   rB   O   s     +r   rB   c                     |r0| j                  d      j                  dd      j                  d      S t        j                  | dz         dz  S )Nr;   g        r<   )add_clamp_r>   r$   r=   r
   s     r   hard_sigmoidrH   X   sB    vvbz  R(--b11wwq2v##r   c                   ,     e Zd Zddef fdZd Z xZS )HardSigmoidr   c                 8    t         t        |           || _        y r   )r   rJ   r   r   r   s     r   r   zHardSigmoid.__init__`   s    k4)+r   c                 .    t        || j                        S r   )rH   r   r   s     r   r   zHardSigmoid.forwardd   s    At||,,r   r   r   r"   s   @r   rJ   rJ   _   s     -r   rJ   c                     |r(| j                  d| dz   j                  dd      z        S d| z  | dz   j                  dd      z  S )z Hard Mish
    Experimental, based on notes by Mish author Diganta Misra at
      https://github.com/digantamisra98/H-Mish/blob/0da20d4bc58e696b6803f2523c58d3c8a82782d0/README.md
    g      ?   r   )minmax)r   clampr
   s     r   	hard_mishrR   h   sL    
 vvcQUMMaQM7788Qw!a%1!444r   c                   ,     e Zd Zddef fdZd Z xZS )HardMishr   c                 8    t         t        |           || _        y r   )r   rT   r   r   r   s     r   r   zHardMish.__init__t   s    h&(r   c                 .    t        || j                        S r   )rR   r   r   s     r   r   zHardMish.forwardx   s    DLL))r   r   r   r"   s   @r   rT   rT   s   s     *r   rT   c            	       n     e Zd ZdZd
dedededdf fdZdej                  dej                  fd	Z
 xZS )PReLUz)Applies PReLU (w/ dummy inplace arg)
    num_parametersinitr   returnNc                 0    t         t        |   ||       y )N)rY   rZ   )r   rX   r   )r   rY   rZ   r   r   s       r   r   zPReLU.__init__   s    eT#>#Mr   inputc                 B    t        j                  || j                        S r   )r$   preluweightr   r]   s     r   r   zPReLU.forward   s    wwudkk**r   )   g      ?F)r   r   r   r-   intfloatr    r   torchTensorr   r!   r"   s   @r   rX   rX   |   sJ    Ns Ne NT N^b N+U\\ +ell +r   rX   r   r[   c                 ,    t        j                  |       S r   r$   gelur
   s     r   ri   ri      s    66!9r   c                   b     e Zd ZdZddef fdZdej                  dej                  fdZ xZ	S )GELULApplies the Gaussian Error Linear Units function (w/ dummy inplace arg)
    r   c                 *    t         t        |           y r   )r   rk   r   r   s     r   r   zGELU.__init__   r+   r   r]   r[   c                 ,    t        j                  |      S r   rh   ra   s     r   r   zGELU.forward   s    vve}r   r   
r   r   r   r-   r    r   re   rf   r   r!   r"   s   @r   rk   rk      s/    % %U\\ ell r   rk   c                 0    t        j                  | d      S Nr&   )approximaterh   r
   s     r   	gelu_tanhrs      s    66!((r   c                   b     e Zd ZdZddef fdZdej                  dej                  fdZ xZ	S )GELUTanhrl   r   c                 *    t         t        |           y r   )r   ru   r   r   s     r   r   zGELUTanh.__init__   s    h&(r   r]   r[   c                 0    t        j                  |d      S rq   rh   ra   s     r   r   zGELUTanh.forward   s    vve00r   r   ro   r"   s   @r   ru   ru      s/    ) )1U\\ 1ell 1r   ru   c                 8    | t        j                  d| z        z  S )NgZd;?)re   r   r
   s     r   
quick_gelury      s    u}}UQY'''r   c                   b     e Zd ZdZddef fdZdej                  dej                  fdZ xZ	S )	QuickGELUrl   r   c                 *    t         t        |           y r   )r   r{   r   r   s     r   r   zQuickGELU.__init__   s    i')r   r]   r[   c                     t        |      S r   )ry   ra   s     r   r   zQuickGELU.forward   s    %  r   r   ro   r"   s   @r   r{   r{      s/    * *!U\\ !ell !r   r{   r   )r-   re   r   torch.nnr   r$   r    r   Moduler   r'   r)   r   r1   r&   r7   r@   rB   rH   rJ   rR   rT   rX   rf   ri   rk   rs   ru   ry   r{    r   r   <module>r      s     $Bd B&BII &'T '299 4 4
=bii =.T .
7299 764 6
+		 +$T $-")) -5$ 5*ryy *+BHH +ELL 4 ELL 299 ) ) ) )1ryy 1(%,, ( (%,, (!		 !r   