
    kh&6              	       "   d Z ddlmZmZ ddlZddlmZ ddlmc mZ	 ddl
mZ ddlmZ d%defdZd%defd	Zd%defd
Zd&deeee   f   defdZd'dededefdZd(dedededefdZd)dedefdZ G d dej0                        Z G d dej0                        Z G d dej0                        Z G d dej0                        Z G d de      Z G d dej0                        Z G d d e      Z G d! d"ej0                        Z  G d# d$e       Z!y)*a7   EvoNorm in PyTorch

Based on `Evolving Normalization-Activation Layers` - https://arxiv.org/abs/2004.02967
@inproceedings{NEURIPS2020,
 author = {Liu, Hanxiao and Brock, Andy and Simonyan, Karen and Le, Quoc},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
 pages = {13539--13550},
 publisher = {Curran Associates, Inc.},
 title = {Evolving Normalization-Activation Layers},
 url = {https://proceedings.neurips.cc/paper/2020/file/9d4c03631b8b0c85ae08bf05eda37d0f-Paper.pdf},
 volume = {33},
 year = {2020}
}

An attempt at getting decent performing EvoNorms running in PyTorch.
While faster than other PyTorch impl, still quite a ways off the built-in BatchNorm
in terms of memory usage and throughput on GPUs.

I'm testing these modules on TPU w/ PyTorch XLA. Promising start but
currently working around some issues with builtin torch/tensor.var/std. Unlike
GPU, similar train speeds for EvoNormS variants and BatchNorm.

Hacked together by / Copyright 2020 Ross Wightman
    )SequenceUnionN   )create_act_layer)_assertepsc                     | j                         j                  ddd      j                  |      j                         j	                  | j
                        }|j                  | j                        S )N      FTdimunbiasedkeepdim)floatvaraddsqrttodtypeexpandshapexr   stds      P/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/layers/evo_norm.pyinstance_stdr   $   sU    
'')--FUD-
A
E
Ec
J
O
O
Q
T
TUVU\U\
]C::agg    c                     t        | d      j                  |      j                         }|j                  | j                        S )Nr
   )r   )
manual_varr   r   r   r   r   s      r   instance_std_tpur!   )   s5    
QF
#
'
'
,
1
1
3C::aggr   c                    | j                         j                         j                  dd      j                  |      j	                         j                  | j                        }|j                  | j                        S )Nr
   Tr   r   )	r   squaremeanr   r   r   r   r   r   )r   r   rmss      r   instance_rmsr'   /   s\    
'')



!
!fd
!
;
?
?
D
I
I
K
N
Nqww
WC::aggr   r   diff_sqmc                     | j                  |d      }|r-| | z  j                  |d      ||z  z
  j                  d      }|S | |z
  | |z
  z  j                  |d      }|S )NTr#   r   )r%   clamp)r   r   r(   xmr   s        r   r    r    4   sq    	
C	&BA||T|2b2g>EEaH J B1r6"((S$(?Jr   groupsflattenc                 N   | j                   \  }}}}| j                  }t        ||z  dk(  d       |rb| j                  ||d      } | j	                         j                  ddd      j                  |      j                         j                  |      }	nf| j                  ||||z  ||      } | j	                         j                  ddd      j                  |      j                         j                  |      }	|	j                  | j                         j                  ||||      S )	Nr    r   FTr   r   r      )
r   r   r   reshaper   r   r   r   r   r   )
r   r,   r   r-   BCHWx_dtyper   s
             r   	group_stdr9   >   s    JAq!QggGAJ!OR IIa$ggimmE4m@DDSINNPSST[\IIafa3ggimm	E4mHLLSQVVX[[\cd::agg&&q!Q22r   c                 p   | j                   \  }}}}t        ||z  dk(  d       |r"| j                  ||d      } t        | d|      }	n&| j                  ||||z  ||      } t        | d|      }	|	j	                  |      j                         j                  | j                         j                  ||||      S )Nr   r/   r0   )r   r(   r1   )r   r   r3   r    r   r   r   )
r   r,   r   r(   r-   r4   r5   r6   r7   r   s
             r   group_std_tpur;   K   s     JAq!QAJ!OR IIa$X6IIafa3	H=773<%%agg.66q!QBBr   c                    | j                   \  }}}}t        ||z  dk(  d       | j                  }| j                  ||||z  ||      } | j	                         j                         j                  dd      j                  |      j                         j                  |      }|j                  | j                         j                  ||||      S )Nr   r/   r1   Tr#   )r   r   r   r3   r   r$   r%   r   sqrt_r   r   )	r   r,   r   r4   r5   r6   r7   r8   r&   s	            r   	group_rmsr>   Z   s    JAq!QAJ!OR ggG			!VQ&[!Q/A
'')



!
!i
!
>
B
B3
G
M
M
O
R
RSZ
[C::agg&&q!Q22r   c                   ,     e Zd Zd fd	Zd Zd Z xZS )EvoNorm2dB0c                    t         |           || _        || _        || _        t        j                  t        j                  |            | _	        t        j                  t        j                  |            | _        |r(t        j                  t        j                  |            nd | _        | j                  dt        j                  |             | j                          y Nrunning_var)super__init__	apply_actmomentumr   nn	Parametertorchonesweightzerosbiasvregister_bufferreset_parametersselfnum_featuresrF   rG   r   _	__class__s         r   rE   zEvoNorm2dB0.__init__d   s    " ll5::l#;<LL\!:;	;Dejj67$]EJJ|,DEr   c                    t         j                  j                  | j                         t         j                  j	                  | j
                         | j                  *t         j                  j                  | j                         y y NrH   initones_rL   zeros_rN   rO   rS   s    r   rQ   zEvoNorm2dB0.reset_parameterso   M    
dkk"
tyy!66GGMM$&&! r   c                    t        |j                         dk(  d       |j                  }d}| j                  f| j                  r|j                         j                  dd      }|j                         |j                  d   z  }| j                  j                  | j                  d| j                  z
  z  |j                         | j                  z  ||dz
  z  z  z          n| j                  }|j                  | j                        j                         j!                  |      j#                  |      j%                  |      }| j                  j!                  |      j#                  |      }||z  t'        || j                        z   }||j)                  |      z  }|| j*                  j!                  |      j#                  |      z  | j,                  j!                  |      j#                  |      z   S Nr2   expected 4D inputr   r0   r   r   )r   r   r   F)r   r   r   )r   r   r   rO   trainingr   r   numelr   rC   copy_rG   detachr   r   r=   r   view	expand_asr   maxrL   rN   )	rS   r   r8   v_shaper   nleftrO   rights	            r   forwardzEvoNorm2dB0.forwardu   s   112''66}}ggimm	EmBGGI
*  &&$$DMM(9:JJL4==0AQK@AB &&77488$**,//8==gFPPQRSD		'"''0AELDHH55EDHHUO#A4;;>>'*//88499<<;P;U;UV];^^^r   )T皙?MbP?__name__
__module____qualname__rE   rQ   rn   __classcell__rV   s   @r   r@   r@   c   s    	 "_r   r@   c                   ,     e Zd Zd fd	Zd Zd Z xZS )EvoNorm2dB1c                 j   t         |           || _        || _        || _        t        j                  t        j                  |            | _	        t        j                  t        j                  |            | _        | j                  dt        j                  |             | j                          y rB   rD   rE   rF   rG   r   rH   rI   rJ   rK   rL   rM   rN   rP   rQ   rR   s         r   rE   zEvoNorm2dB1.__init__   z    " ll5::l#;<LL\!:;	]EJJ|,DEr   c                     t         j                  j                  | j                         t         j                  j	                  | j
                         y rX   rH   rZ   r[   rL   r\   rN   r]   s    r   rQ   zEvoNorm2dB1.reset_parameters   *    
dkk"
tyy!r   c                    t        |j                         dk(  d       |j                  }d}| j                  rR| j                  r|j                         j                  dd      }|j                         |j                  d   z  }| j                  j                  | j                  d| j                  z
  z  |j                         j                  | j                  j                        | j                  z  ||dz
  z  z  z          n| j                  }|j                  |      j                  |      }|j                  | j                         j#                         }|dz   t%        || j                         z  }||j'                  |      z  }|| j(                  j                  |      j                  |      z  | j*                  j                  |      j                  |      z   S r`   r   r   r   rF   rc   r   r   rd   r   rC   re   rG   rf   r   rg   r   r   r=   r'   ri   rL   rN   rS   r   r8   rj   r   rk   rl   rm   s           r   rn   zEvoNorm2dB1.forward   sz   112''>>}}ggimm	EmBGGI
*  &&$$DMM(9:JJLOOD$4$4$:$:;dmmKqTUXYTY{[\] &&&&/&&w/C77488$**,DUl1dhh77EDHHUO#A4;;##G,//88499>>';R;U;UV];^^^r   Tro   h㈵>rq   rv   s   @r   rx   rx           "_r   rx   c                   ,     e Zd Zd fd	Zd Zd Z xZS )EvoNorm2dB2c                 j   t         |           || _        || _        || _        t        j                  t        j                  |            | _	        t        j                  t        j                  |            | _        | j                  dt        j                  |             | j                          y rB   rz   rR   s         r   rE   zEvoNorm2dB2.__init__   r{   r   c                     t         j                  j                  | j                         t         j                  j	                  | j
                         y rX   r}   r]   s    r   rQ   zEvoNorm2dB2.reset_parameters   r~   r   c                    t        |j                         dk(  d       |j                  }d}| j                  rO| j                  r|j                         j                  dd      }|j                         |j                  d   z  }| j                  j                  | j                  d| j                  z
  z  |j                         j                  | j                  j                        | j                  z  ||dz
  z  z  z          n| j                  }|j                  |      j                  |      }|j                  | j                         j#                         }t%        || j                         |z
  }||j'                  |      z  }|| j(                  j                  |      j                  |      z  | j*                  j                  |      j                  |      z   S r`   r   r   s           r   rn   zEvoNorm2dB2.forward   sv   112''>>}}ggimm	EmBGGI
*  &&$$DMM(9:JJLOOD$4$4$:$:;dmmKqTUXYTY{[\] &&&&/&&w/C77488$**,D DHH-1EDHHUO#A4;;##G,//88499>>';R;U;UV];^^^r   r   rq   rv   s   @r   r   r      r   r   r   c                   ,     e Zd Zd fd	Zd Zd Z xZS )EvoNorm2dS0c                    t         |           || _        |r||z  dk(  sJ ||z  | _        n|| _        || _        t        j                  t        j                  |            | _	        t        j                  t        j                  |            | _        |r(t        j                  t        j                  |            nd | _        | j                          y Nr   )rD   rE   rF   r,   r   rH   rI   rJ   rK   rL   rM   rN   rO   rQ   rS   rT   r,   
group_sizerF   r   rU   rV   s          r   rE   zEvoNorm2dS0.__init__   s    "*,111&*4DK DKll5::l#;<LL\!:;	;Dejj67$r   c                    t         j                  j                  | j                         t         j                  j	                  | j
                         | j                  *t         j                  j                  | j                         y y rX   rY   r]   s    r   rQ   zEvoNorm2dS0.reset_parameters   r^   r   c                    t        |j                         dk(  d       |j                  }d}| j                  b| j                  j	                  |      j                  |      }|||z  j                         z  t        || j                  | j                        z  }|| j                  j	                  |      j                  |      z  | j                  j	                  |      j                  |      z   S Nr2   ra   rb   )r   r   r   rO   rg   r   sigmoidr9   r,   r   rL   rN   )rS   r   r8   rj   rO   s        r   rn   zEvoNorm2dS0.forward   s    112''66G$''0AQUOO%%	!T[[$(((KKA4;;##G,//88499>>';R;U;UV];^^^r   )    NTr   rq   rv   s   @r   r   r      s     "_r   r   c                   &     e Zd Zd fd	Zd Z xZS )EvoNorm2dS0ac                 .    t         |   |||||       y )N)r,   r   rF   r   rD   rE   r   s          r   rE   zEvoNorm2dS0a.__init__   s"    J)Y\ 	 	^r   c                    t        |j                         dk(  d       |j                  }d}t        || j                  | j
                        }| j                  @| j                  j                  |      j                  |      }|||z  j                         z  }||z  }|| j                  j                  |      j                  |      z  | j                  j                  |      j                  |      z   S r   )r   r   r   r9   r,   r   rO   rg   r   r   rL   rN   )rS   r   r8   rj   drO   s         r   rn   zEvoNorm2dS0a.forward   s    112''adhh/66G$''0AQUOO%%AE4;;##G,//88499>>';R;U;UV];^^^r   )r   NTrp   rr   rs   rt   rE   rn   ru   rv   s   @r   r   r      s    ^	_r   r   c                   0     e Zd Z	 	 d fd	Zd Zd Z xZS )EvoNorm2dS1c                    t         |           |xs t        j                  }|| _        ||rt        |      | _        nt        j                         | _        |r||z  dk(  sJ ||z  | _        n|| _        || _	        d| _
        t        j                  t        j                  |            | _        t        j                  t        j                  |            | _        | j#                          y )Nr   F)rD   rE   rH   SiLUrF   r   actIdentityr,   r   pre_act_normrI   rJ   rK   rL   rM   rN   rQ   	rS   rT   r,   r   rF   	act_layerr   rU   rV   s	           r   rE   zEvoNorm2dS1.__init__  s     	(	" Y'	2DH{{}DH*,111&*4DK DK!ll5::l#;<LL\!:;	r   c                     t         j                  j                  | j                         t         j                  j	                  | j
                         y rX   r}   r]   s    r   rQ   zEvoNorm2dS1.reset_parameters  r~   r   c                    t        |j                         dk(  d       |j                  }d}| j                  r3| j	                  |      t        || j                  | j                        z  }|| j                  j                  |      j                  |      z  | j                  j                  |      j                  |      z   S r   )r   r   r   rF   r   r9   r,   r   rL   rg   r   rN   rS   r   r8   rj   s       r   rn   zEvoNorm2dS1.forward      112''>>i4;;AAA4;;##G,//88499>>';R;U;UV];^^^r   r   NTNr   rq   rv   s   @r   r   r      s    6:04 *"_r   r   c                   *     e Zd Z	 	 d fd	Zd Z xZS )EvoNorm2dS1ac                 0    t         |   ||||||       y N)r,   r   rF   r   r   r   r   s	           r   rE   zEvoNorm2dS1a.__init__$  '     	J)_hnq 	 	sr   c                 n   t        |j                         dk(  d       |j                  }d}| j                  |      t	        || j
                  | j                        z  }|| j                  j                  |      j                  |      z  | j                  j                  |      j                  |      z   S r   )r   r   r   r   r9   r,   r   rL   rg   r   rN   r   s       r   rn   zEvoNorm2dS1a.forward*      112''HHQK)At{{DHH==4;;##G,//88499>>';R;U;UV];^^^r   r   NTNrp   r   rv   s   @r   r   r   #      6:04s_r   r   c                   0     e Zd Z	 	 d fd	Zd Zd Z xZS )EvoNorm2dS2c                    t         |           |xs t        j                  }|| _        ||rt        |      | _        nt        j                         | _        |r||z  dk(  sJ ||z  | _        n|| _        || _	        t        j                  t        j                  |            | _        t        j                  t        j                  |            | _        | j!                          y r   )rD   rE   rH   r   rF   r   r   r   r,   r   rI   rJ   rK   rL   rM   rN   rQ   r   s	           r   rE   zEvoNorm2dS2.__init__3  s     	(	" Y'	2DH{{}DH*,111&*4DK DKll5::l#;<LL\!:;	r   c                     t         j                  j                  | j                         t         j                  j	                  | j
                         y rX   r}   r]   s    r   rQ   zEvoNorm2dS2.reset_parametersG  r~   r   c                    t        |j                         dk(  d       |j                  }d}| j                  r3| j	                  |      t        || j                  | j                        z  }|| j                  j                  |      j                  |      z  | j                  j                  |      j                  |      z   S r   )r   r   r   rF   r   r>   r,   r   rL   rg   r   rN   r   s       r   rn   zEvoNorm2dS2.forwardK  r   r   r   rq   rv   s   @r   r   r   2  s    6:04 ("_r   r   c                   *     e Zd Z	 	 d fd	Zd Z xZS )EvoNorm2dS2ac                 0    t         |   ||||||       y r   r   r   s	           r   rE   zEvoNorm2dS2a.__init__U  r   r   c                 n   t        |j                         dk(  d       |j                  }d}| j                  |      t	        || j
                  | j                        z  }|| j                  j                  |      j                  |      z  | j                  j                  |      j                  |      z   S r   )r   r   r   r   r>   r,   r   rL   rg   r   rN   r   s       r   rn   zEvoNorm2dS2a.forward[  r   r   r   r   rv   s   @r   r   r   T  r   r   r   )r   )F)r   r   F)r   r   FF)r   r   )"__doc__typingr   r   rJ   torch.nnrH   torch.nn.functional
functionalF
create_actr   trace_utilsr   r   r   r!   r'   intboolr    r9   r;   r>   Moduler@   rx   r   r   r   r   r   r   r    r   r   <module>r      sU  2 #     (   
U  
uS(3-/0 D 
3 
3 
3t 
3CS CE CD C[_ C3 3 3$_")) $_N _"))  _F _"))  _F_")) _>_; _" _"))  _F_; __")) _D_; _r   