
    ^j?              	       *   d Z ddlmZmZmZmZ ddlZddlmZ ddl	mc m
Z ddlmZ ddlmZ d%defdZd%defd	Zd%defd
Zd&deeee   f   defdZd'dededefdZd(dedededefdZd)dedefdZ G d dej4                        Z G d dej4                        Z G d dej4                        Z G d dej4                        Z G d de      Z G d dej4                        Z  G d d e       Z! G d! d"ej4                        Z" G d# d$e"      Z#y)*a7   EvoNorm in PyTorch

Based on `Evolving Normalization-Activation Layers` - https://arxiv.org/abs/2004.02967
@inproceedings{NEURIPS2020,
 author = {Liu, Hanxiao and Brock, Andy and Simonyan, Karen and Le, Quoc},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M. F. Balcan and H. Lin},
 pages = {13539--13550},
 publisher = {Curran Associates, Inc.},
 title = {Evolving Normalization-Activation Layers},
 url = {https://proceedings.neurips.cc/paper/2020/file/9d4c03631b8b0c85ae08bf05eda37d0f-Paper.pdf},
 volume = {33},
 year = {2020}
}

An attempt at getting decent performing EvoNorms running in PyTorch.
While faster than other PyTorch impl, still quite a ways off the built-in BatchNorm
in terms of memory usage and throughput on GPUs.

I'm testing these modules on TPU w/ PyTorch XLA. Promising start but
currently working around some issues with builtin torch/tensor.var/std. Unlike
GPU, similar train speeds for EvoNormS variants and BatchNorm.

Hacked together by / Copyright 2020 Ross Wightman
    )OptionalSequenceTypeUnionN   )create_act_layer)_assertepsc                     | j                         j                  ddd      j                  |      j                         j	                  | j
                        }|j                  | j                        S )N      FTdimunbiasedkeepdim)floatvaraddsqrttodtypeexpandshapexr
   stds      _/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/evo_norm.pyinstance_stdr   $   sU    
'')--FUD-
A
E
Ec
J
O
O
Q
T
TUVU\U\
]C::agg    c                     t        | d      j                  |      j                         }|j                  | j                        S )Nr   )r   )
manual_varr   r   r   r   r   s      r   instance_std_tpur#   )   s5    
QF
#
'
'
,
1
1
3C::aggr    c                    | j                         j                         j                  dd      j                  |      j	                         j                  | j                        }|j                  | j                        S )Nr   Tr   r   )	r   squaremeanr   r   r   r   r   r   )r   r
   rmss      r   instance_rmsr)   /   s\    
'')



!
!fd
!
;
?
?
D
I
I
K
N
Nqww
WC::aggr    r   diff_sqmc                     | j                  |d      }|r-| | z  j                  |d      ||z  z
  j                  d      }|S | |z
  | |z
  z  j                  |d      }|S )NTr%   r   )r'   clamp)r   r   r*   xmr   s        r   r"   r"   4   sq    	
C	&BA||T|2b2g>EEaH J B1r6"((S$(?Jr    groupsflattenc                 N   | j                   \  }}}}| j                  }t        ||z  dk(  d       |rb| j                  ||d      } | j	                         j                  ddd      j                  |      j                         j                  |      }	nf| j                  ||||z  ||      } | j	                         j                  ddd      j                  |      j                         j                  |      }	|	j                  | j                         j                  ||||      S )	Nr    r   FTr   r   r      )
r   r   r	   reshaper   r   r   r   r   r   )
r   r.   r
   r/   BCHWx_dtyper   s
             r   	group_stdr;   >   s    JAq!QggGAJ!OR IIa$ggimmE4m@DDSINNPSST[\IIafa3ggimm	E4mHLLSQVVX[[\cd::agg&&q!Q22r    c                 p   | j                   \  }}}}t        ||z  dk(  d       |r"| j                  ||d      } t        | d|      }	n&| j                  ||||z  ||      } t        | d|      }	|	j	                  |      j                         j                  | j                         j                  ||||      S )Nr   r1   r2   )r   r*   r3   )r   r	   r5   r"   r   r   r   )
r   r.   r
   r*   r/   r6   r7   r8   r9   r   s
             r   group_std_tpur=   K   s     JAq!QAJ!OR IIa$X6IIafa3	H=773<%%agg.66q!QBBr    c                    | j                   \  }}}}t        ||z  dk(  d       | j                  }| j                  ||||z  ||      } | j	                         j                         j                  dd      j                  |      j                         j                  |      }|j                  | j                         j                  ||||      S )Nr   r1   r3   Tr%   )r   r	   r   r5   r   r&   r'   r   sqrt_r   r   )	r   r.   r
   r6   r7   r8   r9   r:   r(   s	            r   	group_rmsr@   Z   s    JAq!QAJ!OR ggG			!VQ&[!Q/A
'')



!
!i
!
>
B
B3
G
M
M
O
R
RSZ
[C::agg&&q!Q22r    c            	       H     e Zd Z	 	 	 	 	 ddedededef fdZd Zd Z xZ	S )	EvoNorm2dB0num_features	apply_actmomentumr
   c                    ||d}t         	|           || _        || _        || _        t        j                  t        j                  |fi |      | _	        t        j                  t        j                  |fi |      | _
        |r)t        j                  t        j                  |fi |      nd | _        | j                  dt        j                  |fi |       | j                          y Ndevicer   running_var)super__init__rD   rE   r
   nn	Parametertorchemptyweightbiasvregister_bufferonesreset_parameters
selfrC   rD   rE   r
   rI   r   _dd	__class__s
            r   rL   zEvoNorm2dB0.__init__d   s     /" ll5;;|#Br#BCLL\!@R!@A	BKekk,="=>QU]EJJ|,Jr,JKr    c                    t         j                  j                  | j                         t         j                  j	                  | j
                         | j                  *t         j                  j                  | j                         y y NrM   initones_rQ   zeros_rR   rS   rX   s    r   rV   zEvoNorm2dB0.reset_parametersz   M    
dkk"
tyy!66GGMM$&&! r    c                    t        |j                         dk(  d       |j                  }d}| j                  f| j                  r|j                         j                  dd      }|j                         |j                  d   z  }| j                  j                  | j                  d| j                  z
  z  |j                         | j                  z  ||dz
  z  z  z          n| j                  }|j                  | j                        j                         j!                  |      j#                  |      j%                  |      }| j                  j!                  |      j#                  |      }||z  t'        || j                        z   }||j)                  |      z  }|| j*                  j!                  |      j#                  |      z  | j,                  j!                  |      j#                  |      z   S Nr4   expected 4D inputr   r2   r   r   )r   r   r   F)r   r   r   )r	   r   r   rS   trainingr   r   numelr   rJ   copy_rE   detachr   r
   r?   r   view	expand_asr   maxrQ   rR   )	rX   r   r:   v_shaper   nleftrS   rights	            r   forwardzEvoNorm2dB0.forward   s   112''66}}ggimm	EmBGGI
*  &&$$DMM(9:JJL4==0AQK@AB &&77488$**,//8==gFPPQRSD		'"''0AELDHH55EDHHUO#A4;;>>'*//88499<<;P;U;UV];^^^r    )T皙?MbP?NN
__name__
__module____qualname__intboolr   rL   rV   rs   __classcell__r[   s   @r   rB   rB   c   sJ     #!     	 
  ,"_r    rB   c            	       H     e Zd Z	 	 	 	 	 ddedededef fdZd Zd Z xZ	S )	EvoNorm2dB1rC   rD   rE   r
   c                 z   ||d}t         	|           || _        || _        || _        t        j                  t        j                  |fi |      | _	        t        j                  t        j                  |fi |      | _
        | j                  dt        j                  |fi |       | j                          y rG   rK   rL   rD   rE   r
   rM   rN   rO   rP   rQ   rR   rT   rU   rV   rW   s
            r   rL   zEvoNorm2dB1.__init__        /" ll5;;|#Br#BCLL\!@R!@A	]EJJ|,Jr,JKr    c                     t         j                  j                  | j                         t         j                  j	                  | j
                         y r]   rM   r_   r`   rQ   ra   rR   rb   s    r   rV   zEvoNorm2dB1.reset_parameters   *    
dkk"
tyy!r    c                    t        |j                         dk(  d       |j                  }d}| j                  rR| j                  r|j                         j                  dd      }|j                         |j                  d   z  }| j                  j                  | j                  d| j                  z
  z  |j                         j                  | j                  j                        | j                  z  ||dz
  z  z  z          n| j                  }|j                  |      j                  |      }|j                  | j                         j#                         }|dz   t%        || j                         z  }||j'                  |      z  }|| j(                  j                  |      j                  |      z  | j*                  j                  |      j                  |      z   S re   r	   r   r   rD   rh   r   r   ri   r   rJ   rj   rE   rk   r   rl   r   r
   r?   r)   rn   rQ   rR   rX   r   r:   ro   r   rp   rq   rr   s           r   rs   zEvoNorm2dB1.forward   sz   112''>>}}ggimm	EmBGGI
*  &&$$DMM(9:JJLOOD$4$4$:$:;dmmKqTUXYTY{[\] &&&&/&&w/C77488$**,DUl1dhh77EDHHUO#A4;;##G,//88499>>';R;U;UV];^^^r    Trt   h㈵>NNrv   r}   s   @r   r   r      J     #!     	 
  *"_r    r   c            	       H     e Zd Z	 	 	 	 	 ddedededef fdZd Zd Z xZ	S )	EvoNorm2dB2rC   rD   rE   r
   c                 z   ||d}t         	|           || _        || _        || _        t        j                  t        j                  |fi |      | _	        t        j                  t        j                  |fi |      | _
        | j                  dt        j                  |fi |       | j                          y rG   r   rW   s
            r   rL   zEvoNorm2dB2.__init__   r   r    c                     t         j                  j                  | j                         t         j                  j	                  | j
                         y r]   r   rb   s    r   rV   zEvoNorm2dB2.reset_parameters   r   r    c                    t        |j                         dk(  d       |j                  }d}| j                  rO| j                  r|j                         j                  dd      }|j                         |j                  d   z  }| j                  j                  | j                  d| j                  z
  z  |j                         j                  | j                  j                        | j                  z  ||dz
  z  z  z          n| j                  }|j                  |      j                  |      }|j                  | j                         j#                         }t%        || j                         |z
  }||j'                  |      z  }|| j(                  j                  |      j                  |      z  | j*                  j                  |      j                  |      z   S re   r   r   s           r   rs   zEvoNorm2dB2.forward   sv   112''>>}}ggimm	EmBGGI
*  &&$$DMM(9:JJLOOD$4$4$:$:;dmmKqTUXYTY{[\] &&&&/&&w/C77488$**,D DHH-1EDHHUO#A4;;##G,//88499>>';R;U;UV];^^^r    r   rv   r}   s   @r   r   r      r   r    r   c                   T     e Zd Z	 	 	 	 	 	 d	dededee   dedef
 fdZd Zd Z	 xZ
S )
EvoNorm2dS0rC   r.   
group_sizerD   r
   c                    ||d}	t         
|           || _        |r||z  dk(  sJ ||z  | _        n|| _        || _        t        j                  t        j                  |fi |	      | _	        t        j                  t        j                  |fi |	      | _
        |r)t        j                  t        j                  |fi |	      nd | _        | j                          y NrH   r   )rK   rL   rD   r.   r
   rM   rN   rO   rP   rQ   rR   rS   rV   )rX   rC   r.   r   rD   r
   rI   r   rY   rZ   r[   s             r   rL   zEvoNorm2dS0.__init__   s     /"*,111&*4DK DKll5;;|#Br#BCLL\!@R!@A	BKekk,="=>QUr    c                    t         j                  j                  | j                         t         j                  j	                  | j
                         | j                  *t         j                  j                  | j                         y y r]   r^   rb   s    r   rV   zEvoNorm2dS0.reset_parameters  rc   r    c                    t        |j                         dk(  d       |j                  }d}| j                  b| j                  j	                  |      j                  |      }|||z  j                         z  t        || j                  | j                        z  }|| j                  j	                  |      j                  |      z  | j                  j	                  |      j                  |      z   S Nr4   rf   rg   )r	   r   r   rS   rl   r   sigmoidr;   r.   r
   rQ   rR   )rX   r   r:   ro   rS   s        r   rs   zEvoNorm2dS0.forward  s    112''66G$''0AQUOO%%	!T[[$(((KKA4;;##G,//88499>>';R;U;UV];^^^r    )    NTr   NN)rw   rx   ry   rz   r   r{   r   rL   rV   rs   r|   r}   s   @r   r   r      s[     (,"     !	 
    4"_r    r   c                   N     e Zd Z	 	 	 	 	 	 ddededee   dedef
 fdZd Z xZ	S )	EvoNorm2dS0arC   r.   r   rD   r
   c           	      2    t         	|   |||||||       y )N)r.   r   rD   r
   rI   r   rK   rL   )
rX   rC   r.   r   rD   r
   rI   r   rY   r[   s
            r   rL   zEvoNorm2dS0a.__init__  s,     	! 	 	
r    c                    t        |j                         dk(  d       |j                  }d}t        || j                  | j
                        }| j                  @| j                  j                  |      j                  |      }|||z  j                         z  }||z  }|| j                  j                  |      j                  |      z  | j                  j                  |      j                  |      z   S r   )r	   r   r   r;   r.   r
   rS   rl   r   r   rQ   rR   )rX   r   r:   ro   drS   s         r   rs   zEvoNorm2dS0a.forward2  s    112''adhh/66G$''0AQUOO%%AE4;;##G,//88499>>';R;U;UV];^^^r    )r   NTru   NN)
rw   rx   ry   rz   r   r{   r   rL   rs   r|   r}   s   @r   r   r     sV     (,"

 
 !	

 
 
*	_r    r   c                   z     e Zd Z	 	 	 	 	 	 	 d
dededee   dedeeej                        de	f fdZ
d Zd	 Z xZS )EvoNorm2dS1rC   r.   r   rD   	act_layerr
   c	                    ||d}
t         |           |xs t        j                  }|| _        ||rt        |      | _        nt        j                         | _        |r||z  dk(  sJ ||z  | _        n|| _        || _	        d| _
        t        j                  t        j                  |fi |
      | _        t        j                  t        j                  |fi |
      | _        | j!                          y )NrH   r   F)rK   rL   rM   SiLUrD   r   actIdentityr.   r
   pre_act_normrN   rO   rP   rQ   rR   rV   rX   rC   r.   r   rD   r   r
   rI   r   rY   rZ   r[   s              r   rL   zEvoNorm2dS1.__init__?  s     /(	" Y'	2DH{{}DH*,111&*4DK DK!ll5;;|#Br#BCLL\!@R!@A	r    c                     t         j                  j                  | j                         t         j                  j	                  | j
                         y r]   r   rb   s    r   rV   zEvoNorm2dS1.reset_parameters_  r   r    c                    t        |j                         dk(  d       |j                  }d}| j                  r3| j	                  |      t        || j                  | j                        z  }|| j                  j                  |      j                  |      z  | j                  j                  |      j                  |      z   S r   )r	   r   r   rD   r   r;   r.   r
   rQ   rl   r   rR   rX   r   r:   ro   s       r   rs   zEvoNorm2dS1.forwardc      112''>>i4;;AAA4;;##G,//88499>>';R;U;UV];^^^r    r   NTNr   NNrw   rx   ry   rz   r   r{   r   rM   Moduler   rL   rV   rs   r|   r}   s   @r   r   r   >  sv     (,"37     !	 
    RYY0   @"_r    r   c                   t     e Zd Z	 	 	 	 	 	 	 d	dededee   dedeeej                        de	f fdZ
d Z xZS )
EvoNorm2dS1arC   r.   r   rD   r   r
   c	           
      4    t         
|   ||||||||       y N)r.   r   rD   r   r
   rI   r   r   rX   rC   r.   r   rD   r   r
   rI   r   rY   r[   s             r   rL   zEvoNorm2dS1a.__init__m  /     	! 	 		
r    c                 n   t        |j                         dk(  d       |j                  }d}| j                  |      t	        || j
                  | j                        z  }|| j                  j                  |      j                  |      z  | j                  j                  |      j                  |      z   S r   )r	   r   r   r   r;   r.   r
   rQ   rl   r   rR   r   s       r   rs   zEvoNorm2dS1a.forward      112''HHQK)At{{DHH==4;;##G,//88499>>';R;U;UV];^^^r    r   NTNru   NNrw   rx   ry   rz   r   r{   r   rM   r   r   rL   rs   r|   r}   s   @r   r   r   l  p     (,"37

 
 !	

 
  RYY0
 
._r    r   c                   z     e Zd Z	 	 	 	 	 	 	 d
dededee   dedeeej                        de	f fdZ
d Zd	 Z xZS )EvoNorm2dS2rC   r.   r   rD   r   r
   c	                    ||d}
t         |           |xs t        j                  }|| _        ||rt        |      | _        nt        j                         | _        |r||z  dk(  sJ ||z  | _        n|| _        || _	        t        j                  t        j                  |fi |
      | _        t        j                  t        j                  |fi |
      | _        | j                          y r   )rK   rL   rM   r   rD   r   r   r   r.   r
   rN   rO   rP   rQ   rR   rV   r   s              r   rL   zEvoNorm2dS2.__init__  s     /(	" Y'	2DH{{}DH*,111&*4DK DKll5;;|#Br#BCLL\!@R!@A	r    c                     t         j                  j                  | j                         t         j                  j	                  | j
                         y r]   r   rb   s    r   rV   zEvoNorm2dS2.reset_parameters  r   r    c                    t        |j                         dk(  d       |j                  }d}| j                  r3| j	                  |      t        || j                  | j                        z  }|| j                  j                  |      j                  |      z  | j                  j                  |      j                  |      z   S r   )r	   r   r   rD   r   r@   r.   r
   rQ   rl   r   rR   r   s       r   rs   zEvoNorm2dS2.forward  r   r    r   r   r}   s   @r   r   r     su     (,"37     !	 
    RYY0   >"_r    r   c                   t     e Zd Z	 	 	 	 	 	 	 d	dededee   dedeeej                        de	f fdZ
d Z xZS )
EvoNorm2dS2arC   r.   r   rD   r   r
   c	           
      4    t         
|   ||||||||       y r   r   r   s             r   rL   zEvoNorm2dS2a.__init__  r   r    c                 n   t        |j                         dk(  d       |j                  }d}| j                  |      t	        || j
                  | j                        z  }|| j                  j                  |      j                  |      z  | j                  j                  |      j                  |      z   S r   )r	   r   r   r   r@   r.   r
   rQ   rl   r   rR   r   s       r   rs   zEvoNorm2dS2a.forward  r   r    r   r   r}   s   @r   r   r     r   r    r   )r   )F)r   r   F)r   r   FF)r   r   )$__doc__typingr   r   r   r   rO   torch.nnrM   torch.nn.functional
functionalF
create_actr   trace_utilsr	   r   r   r#   r)   rz   r{   r"   r;   r=   r@   r   rB   r   r   r   r   r   r   r   r    r    r   <module>r      s[  2 3 2     (   
U  
uS(3-/0 D 
3 
3 
3t 
3CS CE CD C[_ C3 3 3/_")) /_d+_")) +_\+_")) +_\(_")) (_V_; _D+_")) +_\_; _@*_")) *_Z_; _r    