
    khz"                     ^   d Z ddlmZ ddlmZ ddlmZ ddlmZ  G d dej                        Z
 G d	 d
ej                        Z eeej                  d      Z G d dej                        Z G d dej                        Z G d dej                        Z G d dej                        Zy)zl MLP module w/ dropout and configurable activation layer

Hacked together by / Copyright 2020 Ross Wightman
    )partial)nn   )GlobalResponseNorm)	to_2tuplec                   L     e Zd ZdZddej
                  ddddf fd	Zd Z xZS )Mlpz MLP as used in Vision Transformer, MLP-Mixer and related networks

    NOTE: When use_conv=True, expects 2D NCHW tensors, otherwise N*C expected.
    NT        Fc	                    t         |           |xs |}|xs |}t        |      }t        |      }	|rt        t        j
                  d      nt        j                  }
 |
|||d         | _         |       | _        t	        j                  |	d         | _
        | ||      nt	        j                         | _         |
|||d         | _        t	        j                  |	d         | _        y )Nr   kernel_sizer   bias)super__init__r   r   r   Conv2dLinearfc1actDropoutdrop1Identitynormfc2drop2)selfin_featureshidden_featuresout_features	act_layer
norm_layerr   dropuse_conv
drop_probslinear_layer	__class__s              K/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/layers/mlp.pyr   zMlp.__init__   s     	#2{)8[t_
<Dwryya8"))_47K;ZZ
1.
3=3IJ/r{{}	DGLZZ
1.
    c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }|S N)r   r   r   r   r   r   r   xs     r'   forwardzMlp.forward+   sU    HHQKHHQKJJqMIIaLHHQKJJqMr(   	__name__
__module____qualname____doc__r   GELUr   r-   __classcell__r&   s   @r'   r	   r	      s-     !gg/2r(   r	   c                   T     e Zd ZdZddej
                  dddddf fd	Zd Zd Z xZ	S )	GluMlpz MLP w/ GLU style gating
    See: https://arxiv.org/abs/1612.08083, https://arxiv.org/abs/2002.05202

    NOTE: When use_conv=True, expects 2D NCHW tensors, otherwise N*C expected.
    NTr
   Fc
                 "   t         |           |xs |}|xs |}|dz  dk(  sJ t        |      }t        |      }
|rt        t        j
                  d      nt        j                  }|rdnd| _        |	| _         ||||d         | _	         |       | _
        t	        j                  |
d         | _        | ||dz        nt	        j                         | _         ||dz  ||d         | _        t	        j                  |
d         | _        y )N   r   r   r   r   )r   r   r   r   r   r   r   	chunk_dim	gate_lastr   r   r   r   r   r   r   r   )r   r   r   r   r    r!   r   r"   r#   r<   r$   r%   r&   s               r'   r   zGluMlp.__init__;   s     	#2{)8["a'''t_
<Dwryya8"))&B"_47K;ZZ
1.
8B8NJ!34TVT_T_Ta	1 4laQZZ
1.
r(   c                    | j                   j                  Zt        j                  j	                  | j                   j                  | j                   j                  j
                  d   dz  d         t        j                  j                  | j                   j                  | j                   j                  j
                  d   dz  d  d       y )Nr   r9   ư>std)r   r   r   initones_shapenormal_weightr   s    r'   init_weightszGluMlp.init_weightsX   s    88==$GGMM$((--(;(;A(>!(C(DEF
(=(=a(@A(E(FGTRr(   c                 X   | j                  |      }|j                  d| j                        \  }}| j                  r|| j	                  |      z  n| j	                  |      |z  }| j                  |      }| j                  |      }| j                  |      }| j                  |      }|S )Nr9   )dim)	r   chunkr;   r<   r   r   r   r   r   )r   r,   x1x2s       r'   r-   zGluMlp.forward^   s    HHQK/B!%B"TXXb\B5FJJqMIIaLHHQKJJqMr(   )
r/   r0   r1   r2   r   Sigmoidr   rG   r-   r4   r5   s   @r'   r7   r7   5   s6     !jj/:Sr(   r7   F)r    r<   c                   P     e Zd ZdZddej
                  dddf fd	Zd Zd Z xZ	S )SwiGLUz SwiGLU
    NOTE: GluMLP above can implement SwiGLU, but this impl has split fc1 and
    better matches some other common impl which makes mapping checkpoints simpler.
    NTr
   c                    t         	|           |xs |}|xs |}t        |      }t        |      }t        j                  |||d         | _        t        j                  |||d         | _         |       | _        t        j                  |d         | _	        | ||      nt        j                         | _        t        j                  |||d         | _        t        j                  |d         | _        y )Nr   r   r   )r   r   r   r   r   fc1_gfc1_xr   r   r   r   r   r   r   )
r   r   r   r   r    r!   r   r"   r$   r&   s
            r'   r   zSwiGLU.__init__q   s     	#2{)8[t_
YY{O$q'J
YY{O$q'J
;ZZ
1.
3=3IJ/r{{}	99_laIZZ
1.
r(   c                     | j                   j                  3t        j                  j	                  | j                   j                         t        j                  j                  | j                   j                  d       y )Nr>   r?   )rQ   r   r   rA   rB   rD   rE   rF   s    r'   rG   zSwiGLU.init_weights   sF    ::??&GGMM$**//*


))t4r(   c                     | j                  |      }| j                  |      }| j                  |      |z  }| j                  |      }| j	                  |      }| j                  |      }| j                  |      }|S r*   )rQ   rR   r   r   r   r   r   )r   r,   x_gates      r'   r-   zSwiGLU.forward   sh    AJJqMHHVq JJqMIIaLHHQKJJqMr(   )
r/   r0   r1   r2   r   SiLUr   rG   r-   r4   r5   s   @r'   rO   rO   l   s/     !gg/05r(   rO   c                   L     e Zd ZdZddej
                  ddddf fd	Zd Z xZS )GatedMlpz MLP as used in gMLP
    NTr
   c	                 (   t         
|           |xs |}|xs |}t        |      }t        |      }	t        j                  |||d         | _         |       | _        t        j                  |	d         | _        ||dz  dk(  sJ  ||      | _	        |dz  }nt        j                         | _	        | ||      nt        j                         | _        t        j                  |||d         | _        t        j                  |	d         | _        y )Nr   r   r9   r   )r   r   r   r   r   r   r   r   r   gater   r   r   r   )r   r   r   r   r    r!   
gate_layerr   r"   r$   r&   s             r'   r   zGatedMlp.__init__   s     	#2{)8[t_
99[/QH;ZZ
1.
!"Q&!+++"?3DI-2ODI3=3IJ/r{{}	99_laIZZ
1.
r(   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }| j                  |      }|S r*   )r   r   r   rZ   r   r   r   r+   s     r'   r-   zGatedMlp.forward   sb    HHQKHHQKJJqMIIaLIIaLHHQKJJqMr(   r.   r5   s   @r'   rX   rX      s-    
 !gg/<r(   rX   c                   J     e Zd ZdZddej
                  dddf fd	Zd Z xZS )ConvMlpzG MLP using 1x1 convs that keeps spatial dims (for 2D NCHW tensors)
    NTr
   c                 f   t         |           |xs |}|xs |}t        |      }t        j                  ||d|d         | _        |r ||      nt        j                         | _         |       | _        t        j                  |      | _
        t        j                  ||d|d         | _        y )Nr   r   )r   r   )r   r   r   r   r   r   r   r   r   r   r"   r   )	r   r   r   r   r    r!   r   r"   r&   s	           r'   r   zConvMlp.__init__   s     	#2{)8[99[/qtTUwW3=J/2;;=	;JJt$	99_lPTUVPWXr(   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S r*   )r   r   r   r"   r   r+   s     r'   r-   zConvMlp.forward   sH    HHQKIIaLHHQKIIaLHHQKr(   )	r/   r0   r1   r2   r   ReLUr   r-   r4   r5   s   @r'   r^   r^      s+    
 !ggY*r(   r^   c                   J     e Zd ZdZddej
                  dddf fd	Zd Z xZS )GlobalResponseNormMlpz MLP w/ Global Response Norm (see grn.py), nn.Linear or 1x1 Conv2d

    NOTE: Intended for '2D' NCHW (use_conv=True) or NHWC (use_conv=False, channels-last) tensor layouts
    NTr
   Fc                    t         
|           |xs |}|xs |}t        |      }t        |      }|rt        t        j
                  d      nt        j                  }	 |	|||d         | _         |       | _        t	        j                  |d         | _
        t        ||       | _         |	|||d         | _        t	        j                  |d         | _        y )Nr   r   r   r   )channels_last)r   r   r   r   r   r   r   r   r   r   r   r   grnr   r   )r   r   r   r   r    r   r"   r#   r$   r%   r&   s             r'   r   zGlobalResponseNormMlp.__init__   s     	#2{)8[t_
<Dwryya8"))_47K;ZZ
1.
%o\RDGLZZ
1.
r(   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }|S r*   )r   r   r   rf   r   r   r+   s     r'   r-   zGlobalResponseNormMlp.forward  sU    HHQKHHQKJJqMHHQKHHQKJJqMr(   r.   r5   s   @r'   rc   rc      s*     !gg/0r(   rc   N)r2   	functoolsr   torchr   rf   r   helpersr   Moduler	   r7   rV   SwiGLUPackedrO   rX   r^   rc    r(   r'   <module>rn      s      # %")) %P1RYY 1h vEB+RYY +\)ryy )Xbii B$BII $r(   