
    ^jpf                     Z   d Z ddlmZmZmZmZmZmZ ddlZddlm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZmZ ddlmZmZ ddlmZ ddlmZ 	 ddl m!Z! 	 	 	 d6dedee#ef   dee$   de$de	jJ                  f
dZ&e G d de	jN                               Z(e G d de	jR                               Z*d7dZ+e G d dej                  jJ                               Z,d Z-d Z.de/de/d e/fd!Z0 G d" d#e	jb                        Z2 G d$ d%e	jb                        Z3 G d& d'e	jh                        Z5 G d( d)e	jh                        Z6 G d* d+e	jh                        Z7 G d, d-e	jh                        Z8 G d. d/e      Z9 G d0 d1e      Z: G d2 d3e      Z; G d4 d5e      Z<y# e"$ r
 ddlm!Z! Y Iw xY w)8aw   Normalization + Activation Layers

Provides Norm+Act fns for standard PyTorch norm layers such as
* BatchNorm
* GroupNorm
* LayerNorm

This allows swapping with alternative layers that are natively both norm + act such as
* EvoNorm (evo_norm.py)
* FilterResponseNorm (filter_response_norm.py)
* InplaceABN (inplace_abn.py)

Hacked together by / Copyright 2022 Ross Wightman
    )AnyDictListOptionalTypeUnionN)nn)
functional)FrozenBatchNorm2d   )register_notrace_module)create_act_layer)is_fast_normfast_group_normfast_layer_normfast_rms_norm
rms_norm2dfast_rms_norm2d)RmsNorm	RmsNorm2d)_assert)	LayerType)rms_norm	act_layer
act_kwargsinplace	apply_actreturnc                     |xs i }|j                  d|       d }|rt        | fi |}|t        j                         S |S )Nr   )
setdefaultr   r	   Identity)r   r   r   r   acts        _/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/norm_act.py_create_actr$   *   sJ     !rJ)W-
Cy7J7K2;;=0S0    c                        e Zd ZdZdddddej
                  dddddfdededed	ed
edede	de
eef   dedeeej                        f fdZd Z xZS )BatchNormAct2dzBatchNorm + Activation

    This module performs BatchNorm + Activation in a manner that will remain backwards
    compatible with weights trained with separate bn, act. This is why we inherit from BN
    instead of composing it as a .bn member.
    h㈵>g?TNnum_featuresepsmomentumaffinetrack_running_statsr   r   r   r   
drop_layerc                     	 ||d}t        |   |f||||d| |
 |
       nt        j                         | _        t        |||	|      | _        y # t        $ r t         |   |||||       Y Vw xY w)Ndevicedtype)r*   r+   r,   r-   r   r   r   )super__init__	TypeErrorr	   r!   dropr$   r"   )selfr)   r*   r+   r,   r-   r   r   r   r   r.   r1   r2   factory_kwargs	__class__s                 r#   r5   zBatchNormAct2d.__init__@   s    	(.?NG!$7 !" %/$:JL	yZ\ef  	G!$7  	s   A A43A4c           
         t        |j                  dk(  d|j                   d       | j                  d}n| j                  }| j                  rd| j                  rX| j
                  L| j
                  j                  d       | j                  dt        | j
                        z  }n| j                  }	 | j                  rd}n| j                  d u xr | j                  d u }	 t        j                  || j                  r| j                  r| j                  nd | j                  r| j                  r| j                  nd | j                  | j                  ||| j                        }| j                  |      }| j!                  |      }|S )N   zexpected 4D input (got zD input)g        r   g      ?T)r   ndimr+   trainingr-   num_batches_trackedadd_floatrunning_meanrunning_varF
batch_normweightbiasr*   r7   r"   )r8   xexponential_average_factorbn_trainings       r#   forwardzBatchNormAct2d.forwarde   sR   !6qvvhhGH
 == ),&)-&==T55''3((--a0==(14uT=U=U7V1V.15.	 ==K,,4T4;K;Kt;SK	
 LL%)]]d6N6NDTX$(MMT5M5MDSWKKII&HH

 IIaLHHQKr%   )__name__
__module____qualname____doc__r	   ReLUintrA   boolr   r   strr   r   r   Moduler5   rK   __classcell__r:   s   @r#   r'   r'   8   s     !(,"#%77)- 48#g#g #g 	#g
 #g "&#g #g !#g S#X#g #g !bii1#gJ0r%   r'   c                   P     e Zd Zdej                  dej                  f fdZ xZS )SyncBatchNormActrH   r   c                     t         |   |      }t        | d      r| j                  |      }t        | d      r| j	                  |      }|S )Nr7   r"   )r4   rK   hasattrr7   r"   )r8   rH   r:   s     r#   rK   zSyncBatchNormAct.forward   sD    GOA4 		!A4Ar%   )rL   rM   rN   torchTensorrK   rU   rV   s   @r#   rX   rX      s#     %,,  r%   rX   c                    | }t        | t        j                  j                  j                  j
                        rxt        | t              rft        | j                  | j                  | j                  | j                  | j                  |      }| j                  |_        | j                  |_        nVt        j                  j                  | j                  | j                  | j                  | j                  | j                  |      }| j                  r?t        j                          5  | j"                  |_        | j$                  |_        d d d        | j&                  |_        | j(                  |_        | j*                  |_        | j,                  |_        t/        | d      r| j0                  |_        | j3                         D ]!  \  }}|j5                  |t7        ||             # ~ |S # 1 sw Y   xY w)N)process_groupqconfig)
isinstancer[   r	   modules	batchnorm
_BatchNormr'   rX   r)   r*   r+   r,   r-   r"   r7   SyncBatchNormno_gradrF   rG   rB   rC   r?   r>   rZ   r_   named_children
add_moduleconvert_sync_batchnorm)moduler^   module_outputnamechilds        r#   rh   rh      s   M&%((**44??@fn-,##

**+M !'

M!'M "HH22##

**M == 1'-}}$%+[["1 &,%8%8"$*$6$6!,2,F,F)!'69%$*NNM!,,. Ue  '=e]'STU1 1s   (#G++G4c                       e Zd ZdZddej
                  dddddfdededede	d	e
eef   d
edeeej                        f fdZdededededee   dee   dee   f fdZdej*                  dej*                  fdZdefdZ xZS )FrozenBatchNormAct2da$  
    BatchNormAct2d where the batch statistics and the affine parameters are fixed

    Args:
        num_features (int): Number of features ``C`` from an expected input of size ``(N, C, H, W)``
        eps (float): a value added to the denominator for numerical stability. Default: 1e-5
    r(   TNr)   r*   r   r   r   r   r.   c
                    ||	d}
t         |           || _        | j                  dt	        j
                  |fi |
       | j                  dt	        j                  |fi |
       | j                  dt	        j                  |fi |
       | j                  dt	        j
                  |fi |
       | |       nt        j                         | _	        t        ||||      | _        y )Nr0   rF   rG   rB   rC   r3   )r4   r5   r*   register_bufferr[   oneszerosr	   r!   r7   r$   r"   )r8   r)   r*   r   r   r   r   r.   r1   r2   ddr:   s              r#   r5   zFrozenBatchNormAct2d.__init__   s     /Xuzz,'E"'EFVU[[%D%DE^U[[-L-LM]EJJ|,Jr,JK$.$:JL	yZ\efr%   
state_dictprefixlocal_metadatastrictmissing_keysunexpected_keys
error_msgsc           	      H    |dz   }||v r||= t         	|   |||||||       y )Nr?   )r4   _load_from_state_dict)
r8   rt   ru   rv   rw   rx   ry   rz   num_batches_tracked_keyr:   s
            r#   r|   z*FrozenBatchNormAct2d._load_from_state_dict   s?     #)+@"@"j023%oWa	
r%   rH   r   c                    | j                   j                  dddd      }| j                  j                  dddd      }| j                  j                  dddd      }| j                  j                  dddd      }||| j
                  z   j                         z  }|||z  z
  }||z  |z   }| j                  | j                  |            }|S )Nr   )	rF   reshaperG   rC   rB   r*   rsqrtr"   r7   )r8   rH   wbrvrmscalerG   s           r#   rK   zFrozenBatchNormAct2d.forward  s     KK2q!,IIaQ*%%aQ2&&q"a3R$((]))++2:~IHHTYYq\"r%   c                     | j                   j                   d| j                  j                  d    d| j                   d| j
                   dS )N(r   z, eps=z, act=))r:   rL   rF   shaper*   r"   )r8   s    r#   __repr__zFrozenBatchNormAct2d.__repr__  sJ    ..))*!DKK,=,=a,@+AzQWX\X`X`Waabccr%   )rL   rM   rN   rO   r	   rP   rQ   rA   rR   r   r   rS   r   r   r   rT   r5   dictr   r|   r[   r\   rK   r   rU   rV   s   @r#   rn   rn      s
    "#%77)- 48gg g 	g
 !g S#Xg g !bii1g.

 
 	

 
 3i
 c
 I
$ %,, d# dr%   rn   c                 F   | }t        | t        t        f      rDt        | j                        }| j                  |_        | j
                  |_        | j
                  r| j                  j                  j                         j                         |j                  _        | j                  j                  j                         j                         |j                  _        | j                  j                  |j                  _        | j                  j                  |j                  _        | j                  |_        | j                  |_        | j                  |_        |S t        | t         j"                  j$                  j&                  j(                  t         j"                  j$                  j&                  j*                  f      r"t-        | j                        }| j                  |_        | j
                  |_        | j
                  r| j                  j                  j                         j                         |j                  _        | j                  j                  j                         j                         |j                  _        | j                  j                  |j                  _        | j                  j                  |j                  _        | j                  |_        |S | j/                         D ]'  \  }}t1        |      }||us|j3                  ||       ) |S )a  
    Converts all `BatchNorm2d` and `SyncBatchNorm` or `BatchNormAct2d` and `SyncBatchNormAct2d` layers
    of provided module into `FrozenBatchNorm2d` or `FrozenBatchNormAct2d` respectively.

    Args:
        module (torch.nn.Module): Any PyTorch module.

    Returns:
        torch.nn.Module: Resulting module

    Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
    )r`   r'   rX   rn   r)   r,   rF   dataclonedetachrG   rB   rC   r*   r7   r"   r[   r	   ra   rb   BatchNorm2drd   r   rf   freeze_batch_norm_2drg   ri   resrk   rl   	new_childs        r#   r   r     s5    C&>+;<="6#6#67!..]]
==$mm00668??ACJJO"KK,,224;;=CHHM & 3 3 8 8%1166**;;**  J 
FUXX--77CCUXXEUEUE_E_EmEmn	o 3 34!..]]
==$mm00668??ACJJO"KK,,224;;=CHHM & 3 3 8 8%1166** J	 "002 	0KD%,U3I%tY/	0 Jr%   c                 .   | }t        | t              r"t        | j                        }| j                  r| j
                  j                  j                         j                         |j
                  _        | j                  j                  j                         j                         |j                  _        | j                  j                  |j                  _        | j                  j                  |j                  _        | j                  |_        | j                  |_        | j                  |_        |S t        | t              rt         j"                  j%                  | j                        }| j                  r| j
                  j                  j                         j                         |j
                  _        | j                  j                  j                         j                         |j                  _        | j                  j                  |j                  _        | j                  j                  |j                  _        | j                  |_        |S | j'                         D ]'  \  }}t)        |      }||us|j+                  ||       ) |S )a  
    Converts all `FrozenBatchNorm2d` layers of provided module into `BatchNorm2d`. If `module` is itself and instance
    of `FrozenBatchNorm2d`, it is converted into `BatchNorm2d` and returned. Otherwise, the module is walked
    recursively and submodules are converted in place.

    Args:
        module (torch.nn.Module): Any PyTorch module.

    Returns:
        torch.nn.Module: Resulting module

    Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
    )r`   rn   r'   r)   r,   rF   r   r   r   rG   rB   rC   r*   r7   r"   r   r[   r	   r   rf   unfreeze_batch_norm_2drg   r   s        r#   r   r   B  s    C&./V001==$mm00668??ACJJO"KK,,224;;=CHHM & 3 3 8 8%1166**;;** J 
F-	.hh""6#6#67==$mm00668??ACJJO"KK,,224;;=CHHM & 3 3 8 8%1166** J	 "002 	0KD%.u5I%tY/	0 Jr%   num_channels
num_groups
group_sizec                 (    |r| |z  dk(  sJ | |z  S |S )Nr    )r   r   r   s      r#   _num_groupsr   k  s)    j(A---z))r%   c                        e Zd ZU ej                  j
                  e   ed<   dddddej                  dddddfde
de
ded	ed
ee
   dededeeef   dedeeej$                        f fdZd Z xZS )GroupNormAct
_fast_norm    r(   TNr   r   r*   r,   r   r   r   r   r   r.   c                     t         |   t        |||      |||||       |
 |
       nt        j                         | _        t        |||	|      | _        t               | _	        y )Nr*   r,   r1   r2   r3   )
r4   r5   r   r	   r!   r7   r$   r"   r   r   )r8   r   r   r*   r,   r   r   r   r   r   r.   r1   r2   r:   s                r#   r5   zGroupNormAct.__init__v  sf     	j*= 	 	
 %/$:JL	yZ\ef&.r%   c                 T   | j                   r8t        || j                  | j                  | j                  | j
                        }nAt        j                  || j                  | j                  | j                  | j
                        }| j                  |      }| j                  |      }|S N
r   r   r   rF   rG   r*   rD   
group_normr7   r"   r8   rH   s     r#   rK   zGroupNormAct.forward  p    ??4??DKKDHHUAQdiiRAIIaLHHQKr%   )rL   rM   rN   r[   jitFinalrR   __annotations__r	   rP   rQ   rA   r   r   r   rS   r   r   rT   r5   rK   rU   rV   s   @r#   r   r   r  s    		%% !(,"#%77)- 48)) ) 	)
 ) !) ) !) S#X) ) !bii1)8r%   r   c                        e Zd ZU ej                  j
                  e   ed<   dddej                  dddddf	de
dededed	ed
eeef   dedeeej$                        f fdZd Z xZS )GroupNorm1Actr   r(   TNr   r*   r,   r   r   r   r   r.   c                     t         |   d||||	|
       | |       nt        j                         | _        t        ||||      | _        t               | _        y )Nr   r   r3   	r4   r5   r	   r!   r7   r$   r"   r   r   )r8   r   r*   r,   r   r   r   r   r.   r1   r2   r:   s              r#   r5   zGroupNorm1Act.__init__  sR     	Lc&W\]$.$:JL	yZ\ef&.r%   c                 T   | j                   r8t        || j                  | j                  | j                  | j
                        }nAt        j                  || j                  | j                  | j                  | j
                        }| j                  |      }| j                  |      }|S r   r   r   s     r#   rK   zGroupNorm1Act.forward  r   r%   rL   rM   rN   r[   r   r   rR   r   r	   rP   rQ   rA   r   r   rS   r   r   r   rT   r5   rK   rU   rV   s   @r#   r   r     s    		%%
 "#%77)- 48)) ) 	)
 ) !) S#X) ) !bii1)&r%   r   c                        e Zd ZU ej                  j
                  e   ed<   dddej                  dddfde
eee   ej                  f   dededed	ed
eeef   dedeeej*                        f fdZd Z xZS )LayerNormActr   r(   TNnormalization_shaper*   r,   r   r   r   r   r.   c	                     t        
|   |f||d|	 | |       nt        j                         | _        t        ||||      | _        t               | _        y N)r*   elementwise_affiner3   r   r8   r   r*   r,   r   r   r   r   r.   kwargsr:   s             r#   r5   zLayerNormAct.__init__  sR     	,[#&[TZ[$.$:JL	yZ\ef&.r%   c                 T   | j                   r8t        || j                  | j                  | j                  | j
                        }nAt        j                  || j                  | j                  | j                  | j
                        }| j                  |      }| j                  |      }|S r   )
r   r   normalized_shaperF   rG   r*   rD   
layer_normr7   r"   r   s     r#   rK   zLayerNormAct.forward  sw    ??4#8#8$++tyyRVRZRZ[AQ 5 5t{{DIItxxXAIIaLHHQKr%   )rL   rM   rN   r[   r   r   rR   r   r	   rP   r   rQ   r   SizerA   r   r   rS   r   r   r   rT   r5   rK   rU   rV   s   @r#   r   r     s    		%%
 "#%77)- 48)!&sDIuzz'A!B) ) 	)
 ) !) S#X) ) !bii1)$r%   r   c                        e Zd Zdddej                  dddfdeeee   ej                  f   de
dededed	eeef   d
edeeej$                        f fdZd Z xZS )LayerNormActFp32r(   TNr   r*   r,   r   r   r   r   r.   c	                     t        
|   |f||d|	 | |       nt        j                         | _        t        ||||      | _        y r   r4   r5   r	   r!   r7   r$   r"   r   s             r#   r5   zLayerNormActFp32.__init__  sI     	,[#&[TZ[$.$:JL	yZ\efr%   c                    | j                   | j                   j                         nd }| j                  | j                  j                         nd }t        j                  |j                         | j
                  ||| j                        j                  |j                        }| j                  |      }| j                  |      }|S r   )rF   rA   rG   rD   r   r   r*   tor2   r7   r"   r8   rH   rF   rG   s       r#   rK   zLayerNormActFp32.forward  s    (,(?""$T$(II$9tyy tLLD$9$964RUUVWV]V]^IIaLHHQKr%   )rL   rM   rN   r	   rP   r   rQ   r   r[   r   rA   rR   r   r   rS   r   r   r   rT   r5   rK   rU   rV   s   @r#   r   r     s    
 "#%77)- 48g!&sDIuzz'A!Bg g 	g
 g !g S#Xg g !bii1g r%   r   c                        e Zd ZU ej                  j
                  e   ed<   dddej                  dddfde
dededed	ed
eeef   dedeeej$                        f fdZd Z xZS )LayerNormAct2dr   r(   TNr   r*   r,   r   r   r   r   r.   c	                     t        
|   |f||d|	 | |       nt        j                         | _        t        ||||      | _        t               | _        y r   r   r8   r   r*   r,   r   r   r   r   r.   r   r:   s             r#   r5   zLayerNormAct2d.__init__  sP     	T36TVT$.$:JL	yZ\ef&.r%   c                    |j                  dddd      }| j                  r8t        || j                  | j                  | j
                  | j                        }nAt        j                  || j                  | j                  | j
                  | j                        }|j                  dddd      }| j                  |      }| j                  |      }|S Nr         r   )permuter   r   r   rF   rG   r*   rD   r   r7   r"   r   s     r#   rK   zLayerNormAct2d.forward
  s    IIaAq!??4#8#8$++tyyRVRZRZ[AQ 5 5t{{DIItxxXAIIaAq!IIaLHHQKr%   r   rV   s   @r#   r   r     s    		%%
 "#%77)- 48)) ) 	)
 ) !) S#X) ) !bii1)"	r%   r   c                        e Zd Zdddej                  dddfdededededed	e	e
ef   d
edeeej                        f fdZd Z xZS )LayerNormAct2dFp32r(   TNr   r*   r,   r   r   r   r   r.   c	                     t        
|   |f||d|	 | |       nt        j                         | _        t        ||||      | _        y r   r   r   s             r#   r5   zLayerNormAct2dFp32.__init__  sG     	T36TVT$.$:JL	yZ\efr%   c                    |j                  dddd      }| j                  | j                  j                         nd }| j                  | j                  j                         nd }t	        j
                  |j                         | j                  ||| j                        j                  |j                        }|j                  dddd      }| j                  |      }| j                  |      }|S r   )r   rF   rA   rG   rD   r   r   r*   r   r2   r7   r"   r   s       r#   rK   zLayerNormAct2dFp32.forward(  s    IIaAq!(,(?""$T$(II$9tyy tLLD$9$964RUUVWV]V]^IIaAq!IIaLHHQKr%   )rL   rM   rN   r	   rP   rQ   rA   rR   r   r   rS   r   r   r   rT   r5   rK   rU   rV   s   @r#   r   r     s    
 "#%77)- 48gg g 	g
 g !g S#Xg g !bii1g r%   r   c                        e Zd ZdZdddej
                  dddfdedededed	e	d
e
eef   dedeeej                        f fdZdej$                  dej$                  fdZ xZS )
RmsNormAct*   RMSNorm + Activation for '2D' NCHW tensors

    NOTE: It's currently (2025-05-10) faster to use an eager 2d kernel that does reduction
    on dim=1 than to permute and use internal PyTorch F.rms_norm, this may change if something
    like https://github.com/pytorch/pytorch/pull/150576 lands.
    ư>TNr   r*   r,   r   r   r   r   r.   c	                     t        
|   d|||d|	 | |       nt        j                         | _        t        ||||      | _        t               | _        y N)channelsr*   r,   r3   r   r   r   s             r#   r5   zRmsNormAct.__init__:  P     	Q,CQ&Q$.$:JL	yZ\ef&.r%   rH   r   c                    | j                   r-t        || j                  | j                  | j                        }n,t        || j                  | j                  | j                        }| j                  |      }| j                  |      }|S r   )r   r   r   rF   r*   r   r7   r"   r   s     r#   rK   zRmsNormAct.forwardK  sd    ??a!6!6TXXNAD114;;IAIIaLHHQKr%   rL   rM   rN   rO   r	   rP   rQ   rA   rR   r   r   rS   r   r   r   rT   r5   r[   r\   rK   rU   rV   s   @r#   r   r   3       "#%77)- 48)) ) 	)
 ) !) S#X) ) !bii1)" %,, r%   r   c                        e Zd ZdZdddej
                  dddfdedededed	e	d
e
eef   dedeeej                        f fdZdej$                  dej$                  fdZ xZS )RmsNormActFp32r   r   TNr   r*   r,   r   r   r   r   r.   c	                     t        
|   d|||d|	 | |       nt        j                         | _        t        ||||      | _        y r   r   r   s             r#   r5   zRmsNormActFp32.__init__\  G     	Q,CQ&Q$.$:JL	yZ\efr%   rH   r   c                 ,   | j                   | j                   j                         nd }t        |j                         | j                  || j                        j                  |j                        }| j                  |      }| j                  |      }|S r   )	rF   rA   r   r   r*   r   r2   r7   r"   r8   rH   rF   s      r#   rK   zRmsNormActFp32.forwardl  sm    (,(?""$TQWWY 5 5vtxxHKKAGGTIIaLHHQKr%   r   rV   s   @r#   r   r   U       "#%77)- 48gg g 	g
 g !g S#Xg g !bii1g  %,, r%   r   c                        e Zd ZdZdddej
                  dddfdedededed	e	d
e
eef   dedeeej                        f fdZdej$                  dej$                  fdZ xZS )RmsNormAct2dr   r   TNr   r*   r,   r   r   r   r   r.   c	                     t        
|   d|||d|	 | |       nt        j                         | _        t        ||||      | _        t               | _        y r   r   r   s             r#   r5   zRmsNormAct2d.__init__{  r   r%   rH   r   c                    | j                   r-t        || j                  | j                  | j                        }n,t        || j                  | j                  | j                        }| j                  |      }| j                  |      }|S r   )r   r   r   rF   r*   r   r7   r"   r   s     r#   rK   zRmsNormAct2d.forward  sd    ??4#8#8$++txxPA1d33T[[$((KAIIaLHHQKr%   r   rV   s   @r#   r   r   t  r   r%   r   c                        e Zd ZdZdddej
                  dddfdedededed	e	d
e
eef   dedeeej                        f fdZdej$                  dej$                  fdZ xZS )RmsNormAct2dFp32r   r   TNr   r*   r,   r   r   r   r   r.   c	                     t        
|   d|||d|	 | |       nt        j                         | _        t        ||||      | _        y r   r   r   s             r#   r5   zRmsNormAct2dFp32.__init__  r   r%   rH   r   c                 ,   | j                   | j                   j                         nd }t        |j                         | j                  || j                        j                  |j                        }| j                  |      }| j                  |      }|S r   )	rF   rA   r   r   r*   r   r2   r7   r"   r   s      r#   rK   zRmsNormAct2dFp32.forward  sm    (,(?""$Tqwwy$"7"7JMMaggVIIaLHHQKr%   r   rV   s   @r#   r   r     r   r%   r   )NFTr   )=rO   typingr   r   r   r   r   r   r[   r	   torch.nnr
   rD   torchvision.ops.miscr   _fxr   
create_actr   	fast_normr   r   r   r   r   r   normr   r   trace_utilsr   r   torch.nn.functionalr   ImportErrorrS   rR   rT   r$   r   r'   rd   rX   rh   rn   r   r   rQ   r   	GroupNormr   r   	LayerNormr   r   r   r   r   r   r   r   r   r%   r#   <module>r      s   : 9   $ 2 ( (  %   $, &*"'	11cN1 $1 	1
 YY1 \R^^ \ \~ r''  (V @d588?? @d @dF)X&Rc s  '2<< 'TBLL @2<< >r|| 6R\\ @ : DW >9 Dy a  $##$s   F F*)F*