
    ^jM                        d Z ddlZddlmZ ddlZddlmZ ddlmc mZ	 ddl
mZmZmZmZmZmZmZmZ 	 ddlmZ  G d dej*                        Z G d	 d
ej*                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Z G d dej.                        Zdej6                  defdZdej6                  dej6                  dej6                  defdZdej6                  dej6                  dej6                  defdZ  G d dej.                        Z! G d dejD                        Z# G d  d!ejD                        Z$ G d" d#ejD                        Z% G d$ d%ejD                        Z& G d& d'ejD                        Z' G d( d)ejD                        Z( G d* d+ejD                        Z) G d, d-ejD                        Z*y# e$ r
 ddl
mZ Y w xY w).z Normalization layers and wrappers

Norm layer definitions that support fast norm and consistent channel arg order (always first arg).

Hacked together by / Copyright 2022 Ross Wightman
    N)Tuple   )is_fast_normfast_group_normfast_layer_normfast_rms_norm
rms_norm2dfast_rms_norm2dfast_simple_normsimple_norm)rms_normc            	       x     e Zd ZU ej                  j
                  e   ed<   	 	 	 ddedede	def fdZ
d Z xZS )		GroupNorm
_fast_normnum_channels
num_groupsepsaffinec                 L    t        |   ||f||d| t               | _        y )N)r   r   super__init__r   r   )selfr   r   r   r   kwargs	__class__s         [/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/norm.pyr   zGroupNorm.__init__"   s)     	\Ts6TVT&.    c                 
   | j                   r7t        || j                  | j                  | j                  | j
                        S t        j                  || j                  | j                  | j                  | j
                        S Nr   r   r   weightbiasr   F
group_normr   xs     r   forwardzGroupNorm.forward.   S    ??"1doot{{DIItxxXX<<4??DKKDHHUUr   )    gh㈵>T)__name__
__module____qualname__torchjitFinalbool__annotations__intfloatr   r'   __classcell__r   s   @r   r   r      sR    		%%
 !
)
) 
) 	
)
 
)Vr   r   c                        e Zd ZU dZej
                  j                  e   ed<   de	f fdZ
dej                  dej                  fdZ xZS )
GroupNorm1zL Group Normalization with 1 group.
    Input: tensor in shape [B, C, *]
    r   r   c                 F    t        |   d|fi | t               | _        y )Nr   r   )r   r   r   r   s      r   r   zGroupNorm1.__init__;   s     L3F3&.r   r&   returnc                 
   | j                   r7t        || j                  | j                  | j                  | j
                        S t        j                  || j                  | j                  | j                  | j
                        S r   r    r%   s     r   r'   zGroupNorm1.forward?   r(   r   )r*   r+   r,   __doc__r-   r.   r/   r0   r1   r2   r   Tensorr'   r4   r5   s   @r   r7   r7   5   sG     		%%)S )V V%,, Vr   r7   c                        e Zd ZU dZej
                  j                  e   ed<   	 	 d
de	de
def fdZdej                  dej                  fd	Z xZS )	LayerNormz# LayerNorm w/ fast norm option
    r   r   r   r   c                 J    t        |   |f||d| t               | _        y N)r   elementwise_affiner   r   r   r   r   r   r   s        r   r   zLayerNorm.__init__K   '     	T36TVT&.r   r&   r9   c                    | j                   r9t        || j                  | j                  | j                  | j
                        }|S t        j                  || j                  | j                  | j                  | j
                        }|S r   )r   r   normalized_shaper!   r"   r   r#   
layer_normr%   s     r   r'   zLayerNorm.forwardU   sf    ??4#8#8$++tyyRVRZRZ[A  Q 5 5t{{DIItxxXAr   ư>Tr*   r+   r,   r;   r-   r.   r/   r0   r1   r2   r3   r   r<   r'   r4   r5   s   @r   r>   r>   F   s_    		%%
 	)) ) 	) %,, r   r>   c                   n     e Zd ZdZ	 	 d	dededef fdZdej                  dej                  fdZ
 xZS )
LayerNormFp32z LayerNorm
    r   r   r   c                 ,    t        |   |f||d| y r@   r   r   rB   s        r   r   zLayerNormFp32.__init__a        	T36TVTr   r&   r9   c                 N   | j                   | j                   j                         nd }| j                  | j                  j                         nd }t        j                  |j                         | j
                  ||| j                        j                  |j                        }|S r   )	r!   r3   r"   r#   rF   rE   r   todtyper   r&   r!   r"   s       r   r'   zLayerNormFp32.forwardj   sw    (,(?""$T$(II$9tyy tLLD$9$964RUUVWV]V]^r   rG   r*   r+   r,   r;   r2   r3   r0   r   r-   r<   r'   r4   r5   s   @r   rK   rK   ]   sP     	UU U 	U %,, r   rK   c                        e Zd ZU dZej
                  j                  e   ed<   	 	 d
de	de
def fdZdej                  dej                  fd	Z xZS )LayerNorm2d5 LayerNorm for channels of '2D' spatial NCHW tensors r   r   r   r   c                 J    t        |   |f||d| t               | _        y r@   r   rB   s        r   r   zLayerNorm2d.__init__u   rC   r   r&   r9   c                 `   |j                  dddd      }| j                  r8t        || j                  | j                  | j
                  | j                        }nAt        j                  || j                  | j                  | j
                  | j                        }|j                  dddd      }|S Nr         r   )	permuter   r   rE   r!   r"   r   r#   rF   r%   s     r   r'   zLayerNorm2d.forward   s    IIaAq!??4#8#8$++tyyRVRZRZ[AQ 5 5t{{DIItxxXAIIaAq!r   rG   rI   r5   s   @r   rU   rU   q   s]    ?		%%
 	)) ) 	) %,, r   rU   c                   n     e Zd ZdZ	 	 d	dededef fdZdej                  dej                  fdZ
 xZS )
LayerNorm2dFp32rV   r   r   r   c                 ,    t        |   |f||d| y r@   rM   rB   s        r   r   zLayerNorm2dFp32.__init__   rN   r   r&   r9   c                    |j                  dddd      }| j                  | j                  j                         nd }| j                  | j                  j                         nd }t	        j
                  |j                         | j                  ||| j                        j                  |j                        }|j                  dddd      }|S rY   )
r\   r!   r3   r"   r#   rF   rE   r   rP   rQ   rR   s       r   r'   zLayerNorm2dFp32.forward   s    IIaAq!(,(?""$T$(II$9tyy tLLD$9$964RUUVWV]V]^IIaAq!r   rG   rS   r5   s   @r   r^   r^      sN    ?
 	UU U 	U %,, r   r^   tensorr9   c                     t         j                  j                         r| j                         S | j                  t         j                        S )N)memory_format)r-   r.   is_scriptingis_contiguouscontiguous_format)ra   s    r   _is_contiguousrg      s;    yy##%%##%2I2I#JJr   r&   r!   r"   r   c                     t        j                  | ddd      \  }}| |z
  t        j                  ||z         z  } | |d d d d f   z  |d d d d f   z   } | S )Nr   FT)dimunbiasedkeepdim)r-   var_meanrsqrt)r&   r!   r"   r   sus         r   _layer_norm_cfrp      s_    >>!UDADAq	
Q%++a#g&&A	F1dD=!!DD$$77AHr   c                    | j                  dd      }| | z  j                  dd      ||z  z
  j                  d      }| |z
  t        j                  ||z         z  } | |j	                  dddd      z  |j	                  dddd      z   } | S )Nr   T)ri   rk   r   )meanclampr-   rm   view)r&   r!   r"   r   ro   rn   s         r   _layer_norm_cf_sqmrv      s    	1d#A
a%!T	*a!e	4;;A>A	
Q%++a#g&&A	FKK2q!$$tyyB1'==AHr   c                   N     e Zd ZdZddedef fdZdej                  fdZ	 xZ
S )LayerNormExp2da_   LayerNorm for channels_first tensors with 2d spatial dimensions (ie N, C, H, W).

    Experimental implementation w/ manual norm for tensors non-contiguous tensors.

    This improves throughput in some scenarios (tested on Ampere GPU), esp w/ channels_last
    layout. However, benefits are not always clear and can perform worse on other GPUs.
    r   r   c                 (    t         |   ||       y )N)r   rM   )r   r   r   r   s      r   r   zLayerNormExp2d.__init__   s    3/r   r9   c                 B   t        |      rgt        j                  |j                  dddd      | j                  | j
                  | j                  | j                        j                  dddd      }|S t        || j
                  | j                  | j                        }|S rY   )	rg   r#   rF   r\   rE   r!   r"   r   rp   r%   s     r   r'   zLayerNormExp2d.forward   s    !		!Q1%t'<'<dkk499VZV^V^``g`ghiklnoqr`s   q$++tyy$((CAr   )rH   )r*   r+   r,   r;   r2   r3   r   r-   r<   r'   r4   r5   s   @r   rx   rx      s+    0S 0u 0ELL r   rx   c            	            e Zd ZU dZg dZeedf   ed<   eed<   e	ed<   e	ed<   	 	 	 	 dd	eded
e	ddf fdZ
ddZdej                  dej                  fdZ xZS )RmsNorm. RmsNorm w/ fast (apex) norm if available
    rE   r   rA   r   .rE   r   rA   r   Nchannelsr   r9   c                    ||d}t         |           |}t        |t        j                        r|f}t        |      | _        || _        || _        t               | _
        | j                  r9t        j                  t        j                  | j                  fi |      | _        n| j!                  dd        | j#                          y N)devicerQ   r!   r   r   
isinstancenumbersIntegraltuplerE   r   rA   r   r   nn	Parameterr-   emptyr!   register_parameterreset_parameters	r   r   r   r   r   rQ   ddrE   r   s	           r   r   zRmsNorm.__init__        /#&(8(89 02 %&6 7"(&."",,u{{43H3H'OB'OPDK##Hd3r   c                 p    | j                   r*t        j                  j                  | j                         y y r   rA   r   initones_r!   r   s    r   r   zRmsNorm.reset_parameters   $    ""GGMM$++& #r   r&   c                     | j                   r.t        || j                  | j                  | j                        }|S t        || j                  | j                  | j                        }|S r   )r   r   rE   r!   r   r   r%   s     r   r'   zRmsNorm.forward   sU     ??a!6!6TXXNA  D114;;IAr   rH   TNNr9   Nr*   r+   r,   r;   __constants__r   r2   r1   r3   r0   r   r   r-   r<   r'   r4   r5   s   @r   r|   r|          SMCHo%	J
      	  
 4' %,, r   r|   c            	            e Zd ZU dZg dZeedf   ed<   eed<   e	ed<   	 	 	 	 ddeded	e	d
df fdZ
ddZdej                  d
ej                  fdZ xZS )RmsNormFp32r}   rE   r   rA   .rE   r   rA   Nr   r   r9   c                 t   ||d}t         |           |}t        |t        j                        r|f}t        |      | _        || _        || _        | j                  r9t        j                  t        j                  | j                  fi |      | _        n| j                  dd        | j                          y r   r   r   r   r   r   r   rE   r   rA   r   r   r-   r   r!   r   r   r   s	           r   r   zRmsNormFp32.__init__       /#&(8(89 02 %&6 7"("",,u{{43H3H'OB'OPDK##Hd3r   c                 p    | j                   r*t        j                  j                  | j                         y y r   r   r   s    r   r   zRmsNormFp32.reset_parameters  r   r   r&   c                     | j                   | j                   j                         nd }t        |j                         | j                  || j                        j                  |j                        }|S r   )r!   r3   r   rE   r   rP   rQ   r   r&   r!   s      r   r'   zRmsNormFp32.forward   sS    (,(?""$TQWWY 5 5vtxxHKKAGGTr   r   r   r   r5   s   @r   r   r          EMCHo%	J
      	  
 2' %,, r   r   c            	            e Zd ZU dZg dZeedf   ed<   eed<   e	ed<   e	ed<   	 	 	 	 dd	eded
e	ddf fdZ
ddZdej                  dej                  fdZ xZS )	RmsNorm2dB   RmsNorm2D for NCHW tensors, w/ fast apex or cast norm if available

    NOTE: It's currently (2025-05-10) faster to use an eager 2d kernel that does reduction
    on dim=1 than to permute and use internal PyTorch F.rms_norm, this may change if something
    like https://github.com/pytorch/pytorch/pull/150576 lands.
    r~   .rE   r   rA   r   Nr   r   r9   c                    ||d}t         |           |}t        |t        j                        r|f}t        |      | _        || _        || _        t               | _
        | j                  r9t        j                  t        j                  | j                  fi |      | _        n| j!                  dd        | j#                          y r   r   r   s	           r   r   zRmsNorm2d.__init__3  r   r   c                 p    | j                   r*t        j                  j                  | j                         y y r   r   r   s    r   r   zRmsNorm2d.reset_parametersM  r   r   r&   c                     | j                   r.t        || j                  | j                  | j                        }|S t        || j                  | j                  | j                        }|S r   )r   r
   rE   r!   r   r	   r%   s     r   r'   zRmsNorm2d.forwardQ  sU     ??4#8#8$++txxPA  1d33T[[$((KAr   r   r   r   r5   s   @r   r   r   &  s     TMCHo%	J
      	  
 4' %,, r   r   c            	            e Zd ZU dZg dZeedf   ed<   eed<   e	ed<   	 	 	 	 ddeded	e	d
df fdZ
ddZdej                  d
ej                  fdZ xZS )RmsNorm2dFp32r   r   .rE   r   rA   Nr   r   r9   c                 t   ||d}t         |           |}t        |t        j                        r|f}t        |      | _        || _        || _        | j                  r9t        j                  t        j                  | j                  fi |      | _        n| j                  dd        | j                          y r   r   r   s	           r   r   zRmsNorm2dFp32.__init__g  r   r   c                 p    | j                   r*t        j                  j                  | j                         y y r   r   r   s    r   r   zRmsNorm2dFp32.reset_parameters  r   r   r&   c                     | j                   | j                   j                         nd }t        |j                         | j                  || j                        j                  |j                        }|S r   )r!   r3   r	   rE   r   rP   rQ   r   s      r   r'   zRmsNorm2dFp32.forward  sS    (,(?""$Tqwwy$"7"7JMMaggVr   r   r   r   r5   s   @r   r   r   [  s     FMCHo%	J
      	  
 2' %,, r   r   c            	            e Zd ZU dZg dZeedf   ed<   eed<   e	ed<   e	ed<   	 	 	 	 dd	eded
e	ddf fdZ
ddZdej                  dej                  fdZ xZS )
SimpleNorm SimpleNorm (x / std(x))
    r~   .rE   r   rA   r   Nr   r   r9   c                    ||d}t         |           |}t        |t        j                        r|f}t        |      | _        || _        || _        t               | _
        | j                  r9t        j                  t        j                  | j                  fi |      | _        n| j!                  dd        | j#                          y r   r   r   s	           r   r   zSimpleNorm.__init__  r   r   c                 p    | j                   r*t        j                  j                  | j                         y y r   r   r   s    r   r   zSimpleNorm.reset_parameters  r   r   r&   c                     | j                   r.t        || j                  | j                  | j                        }|S t        || j                  | j                  | j                        }|S r   )r   r   rE   r!   r   r   r%   s     r   r'   zSimpleNorm.forward  sS    ?? D$9$94;;QA  At44dkk488LAr   r   r   r   r5   s   @r   r   r     s    SMCHo%	J
      	  
 4' %,, r   r   c            	            e Zd ZU dZg dZeedf   ed<   eed<   e	ed<   	 	 	 	 ddeded	e	d
df fdZ
ddZdej                  d
ej                  fdZ xZS )SimpleNormFp32r   r   .rE   r   rA   Nr   r   r9   c                 t   ||d}t         |           |}t        |t        j                        r|f}t        |      | _        || _        || _        | j                  r9t        j                  t        j                  | j                  fi |      | _        n| j                  dd        | j                          y r   r   r   s	           r   r   zSimpleNormFp32.__init__  r   r   c                 p    | j                   r*t        j                  j                  | j                         y y r   r   r   s    r   r   zSimpleNormFp32.reset_parameters  r   r   r&   c                     | j                   | j                   j                         nd }t        |j                         | j                  || j                        j                  |j                        }|S r   )r!   r3   r   rE   r   rP   rQ   r   s      r   r'   zSimpleNormFp32.forward  sS    (,(?""$T	4#8#8&$((KNNqwwWr   r   r   r   r5   s   @r   r   r     r   r   r   c            	            e Zd ZU dZg dZeedf   ed<   eed<   e	ed<   e	ed<   	 	 	 	 dd	eded
e	ddf fdZ
ddZdej                  dej                  fdZ xZS )SimpleNorm2d! SimpleNorm for NCHW tensors
    r~   .rE   r   rA   r   Nr   r   r9   c                    ||d}t         |           |}t        |t        j                        r|f}t        |      | _        || _        || _        t               | _
        | j                  r9t        j                  t        j                  | j                  fi |      | _        n| j!                  dd        | j#                          y r   r   r   s	           r   r   zSimpleNorm2d.__init__  r   r   c                 p    | j                   r*t        j                  j                  | j                         y y r   r   r   s    r   r   zSimpleNorm2d.reset_parameters  r   r   r&   c                     |j                  dddd      }| j                  r-t        || j                  | j                  | j
                        }n,t        || j                  | j                  | j
                        }|j                  dddd      }|S rY   )r\   r   r   rE   r!   r   r   r%   s     r   r'   zSimpleNorm2d.forward  sr    IIaAq!?? D$9$94;;QAAt44dkk488LAIIaAq!r   r   r   r   r5   s   @r   r   r     r   r   r   c            	            e Zd ZU dZg dZeedf   ed<   eed<   e	ed<   	 	 	 	 ddeded	e	d
df fdZ
ddZdej                  d
ej                  fdZ xZS )SimpleNorm2dFp32r   r   .rE   r   rA   Nr   r   r9   c                 t   ||d}t         |           |}t        |t        j                        r|f}t        |      | _        || _        || _        | j                  r9t        j                  t        j                  | j                  fi |      | _        n| j                  dd        | j                          y r   r   r   s	           r   r   zSimpleNorm2dFp32.__init__  r   r   c                 p    | j                   r*t        j                  j                  | j                         y y r   r   r   s    r   r   z!SimpleNorm2dFp32.reset_parameters6  r   r   r&   c                 8   |j                  dddd      }| j                  | j                  j                         nd }t        |j                         | j                  || j
                        j                  |j                        }|j                  dddd      }|S rY   )r\   r!   r3   r   rE   r   rP   rQ   r   s      r   r'   zSimpleNorm2dFp32.forward:  s{    IIaAq!(,(?""$T	4#8#8&$((KNNqwwWIIaAq!r   r   r   r   r5   s   @r   r   r     s    EMCHo%	J
      	  
 2' %,, r   r   )+r;   r   typingr   r-   torch.nnr   torch.nn.functional
functionalr#   	fast_normr   r   r   r   r	   r
   r   r   r   ImportErrorr   r7   r>   rK   rU   r^   r<   r0   rg   r3   rp   rv   rx   Moduler|   r   r   r   r   r   r   r    r   r   <module>r      s        	 	 	$,
V V,V V" .BLL (",, 0bll *K5<< KD Kell ELL  SX %,,  ELL W\ R\\ *.bii .b(")) (V2		 2j,BII ,^, ,^(RYY (V.299 .b*ryy *u  $##$s   G G,+G,