Ë
    éÓ^jpf  ã                   óZ  — d Z ddlmZmZmZmZmZmZ ddlZddlm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZmZ ddlmZmZ ddlmZ ddlmZ 	 ddl m!Z! 	 	 	 d6dedee#ef   dee$   de$de	jJ                  f
d„Z&e G d„ de	jN                  «      «       Z(e G d„ de	jR                  «      «       Z*d7d„Z+e G d„ dej                  jJ                  «      «       Z,d„ Z-d„ Z.de/de/d e/fd!„Z0 G d"„ d#e	jb                  «      Z2 G d$„ d%e	jb                  «      Z3 G d&„ d'e	jh                  «      Z5 G d(„ d)e	jh                  «      Z6 G d*„ d+e	jh                  «      Z7 G d,„ d-e	jh                  «      Z8 G d.„ d/e«      Z9 G d0„ d1e«      Z: G d2„ d3e«      Z; G d4„ d5e«      Z<y# e"$ r
 ddlm!Z! Y �ŒIw xY w)8aw   Normalization + Activation Layers

Provides Norm+Act fns for standard PyTorch norm layers such as
* BatchNorm
* GroupNorm
* LayerNorm

This allows swapping with alternative layers that are natively both norm + act such as
* EvoNorm (evo_norm.py)
* FilterResponseNorm (filter_response_norm.py)
* InplaceABN (inplace_abn.py)

Hacked together by / Copyright 2022 Ross Wightman
é    )ÚAnyÚDictÚListÚOptionalÚTypeÚUnionN)Únn)Ú
functional)ÚFrozenBatchNorm2dé   )Úregister_notrace_module)Úcreate_act_layer)Úis_fast_normÚfast_group_normÚfast_layer_normÚfast_rms_normÚ
rms_norm2dÚfast_rms_norm2d)ÚRmsNormÚ	RmsNorm2d)Ú_assert)Ú	LayerType)Úrms_normÚ	act_layerÚ
act_kwargsÚinplaceÚ	apply_actÚreturnc                 ó‚   — |xs i }|j                  d|«       d }|rt        | fi |¤Ž}|€t        j                  «       S |S )Nr   )Ú
setdefaultr   r	   ÚIdentity)r   r   r   r   Úacts        ú_/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/norm_act.pyÚ_create_actr$   *   sJ   € ð Ò!˜r€JØ×Ñ˜) WÔ-Ø
€CÙÜ˜yÑ7¨JÑ7ˆØ˜KŒ2�;‰;‹=Ð0¨SÐ0ó    c                   ó¨   ‡ — e Zd ZdZdddddej
                  dddddfdededed	ed
edede	de
eef   dedeeej                        fˆ fd„Zd„ Zˆ xZS )ÚBatchNormAct2dzúBatchNorm + Activation

    This module performs BatchNorm + Activation in a manner that will remain backwards
    compatible with weights trained with separate bn, act. This is why we inherit from BN
    instead of composing it as a .bn member.
    çñhãˆµøä>gš™™™™™¹?TNÚnum_featuresÚepsÚmomentumÚaffineÚtrack_running_statsr   r   r   r   Ú
drop_layerc                 óî   •— 	 ||dœ}t        ‰| �  |f||||dœ|¤Ž |
� |
«       nt        j                  «       | _        t        |||	|¬«      | _        y # t        $ r t         ‰| �  |||||¬«       Y ŒVw xY w)N©ÚdeviceÚdtype)r*   r+   r,   r-   ©r   r   r   )ÚsuperÚ__init__Ú	TypeErrorr	   r!   Údropr$   r"   )Úselfr)   r*   r+   r,   r-   r   r   r   r   r.   r1   r2   Úfactory_kwargsÚ	__class__s                 €r#   r5   zBatchNormAct2d.__init__@   sš   ø€ ð	Ø(.¸Ñ?ˆNÜ‰GÑØðàØ!ØØ$7ñð !òð" %/Ð$:‘J”LÄÇÁÃˆŒ	Ü˜y°ZÈÐ\eÔfˆ�øô ò 	ä‰GÑØØØ!ØØ$7ð ö ð	ús   ƒA ÁA4Á3A4c           
      ó  — t        |j                  dk(  d|j                  › d�«       | j                  €d}n| j                  }| j                  rd| j                  rX| j
                  �L| j
                  j                  d«       | j                  €dt        | j
                  «      z  }n| j                  }	 | j                  rd}n| j                  d u xr | j                  d u }	 t        j                  || j                  r| j                  r| j                  nd | j                  r| j                  r| j                  nd | j                  | j                  ||| j                  «      }| j                  |«      }| j!                  |«      }|S )Né   zexpected 4D input (got zD input)g        r   g      ð?T)r   Úndimr+   Útrainingr-   Únum_batches_trackedÚadd_ÚfloatÚrunning_meanÚrunning_varÚFÚ
batch_normÚweightÚbiasr*   r7   r"   )r8   ÚxÚexponential_average_factorÚbn_trainings       r#   ÚforwardzBatchNormAct2d.forwarde   sR  € ä�—‘˜!‘Ð6°q·v±v°h¸hÐGÔHð
 �=‰=Ð Ø),Ñ&à)-¯©Ð&à�=Š=˜T×5Ò5à×'Ñ'Ð3Ø×(Ñ(×-Ñ-¨aÔ0Ø—=‘=Ð(Ø14´u¸T×=UÑ=UÓ7VÑ1VÑ.à15·±Ð.ð	ð �=Š=Ø‰Kà×,Ñ,°Ð4ÒT¸4×;KÑ;KÈtÐ;SˆKð	ô
 �L‰LØà%)§]¢]°d×6NÒ6NˆD×ÒÐTXØ$(§M¢M°T×5MÒ5MˆD×ÒÐSWØ�K‰KØ�I‰IØØ&Ø�H‰Hó

ˆð �I‰I�a‹LˆØ�H‰H�Q‹KˆØˆr%   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r	   ÚReLUÚintrA   Úboolr   r   Ústrr   r   r   ÚModuler5   rK   Ú__classcell__©r:   s   @r#   r'   r'   8   s¼   ø„ ñð Ø!ØØ(,Ø"Ø#%§7¡7Ø)-Ø Ø48ØØñ#gàð#gð ð#gð ð	#gð
 ð#gð "&ð#gð ð#gð !ð#gð ˜S #˜X™ð#gð ð#gð !  b§i¡i¡Ñ1õ#göJ0r%   r'   c                   óP   ‡ — e Zd Zdej                  dej                  fˆ fd„Zˆ xZS )ÚSyncBatchNormActrH   r   c                 óš   •— t         ‰| �  |«      }t        | d«      r| j                  |«      }t        | d«      r| j	                  |«      }|S )Nr7   r"   )r4   rK   Úhasattrr7   r"   )r8   rH   r:   s     €r#   rK   zSyncBatchNormAct.forwardž   sD   ø€ Ü‰G‰O˜AÓˆÜ�4˜Ô Ø—	‘	˜!“ˆAÜ�4˜ÔØ—‘˜“ˆAØˆr%   )rL   rM   rN   ÚtorchÚTensorrK   rU   rV   s   @r#   rX   rX   ˜   s#   ø„ ð˜Ÿ™ð ¨%¯,©,÷ ñ r%   rX   c                 óî  — | }t        | t        j                  j                  j                  j
                  «      �rxt        | t        «      rft        | j                  | j                  | j                  | j                  | j                  |¬«      }| j                  |_        | j                  |_        nVt        j                  j                  | j                  | j                  | j                  | j                  | j                  |«      }| j                  r?t        j                   «       5  | j"                  |_        | j$                  |_        d d d «       | j&                  |_        | j(                  |_        | j*                  |_        | j,                  |_        t/        | d«      r| j0                  |_        | j3                  «       D ]!  \  }}|j5                  |t7        ||«      «       Œ# ~ |S # 1 sw Y   Œ¡xY w)N)Úprocess_groupÚqconfig)Ú
isinstancer[   r	   ÚmodulesÚ	batchnormÚ
_BatchNormr'   rX   r)   r*   r+   r,   r-   r"   r7   ÚSyncBatchNormÚno_gradrF   rG   rB   rC   r?   r>   rZ   r_   Únamed_childrenÚ
add_moduleÚconvert_sync_batchnorm)Úmoduler^   Úmodule_outputÚnameÚchilds        r#   rh   rh   §   s‹  € à€MÜ�&œ%Ÿ(™(×*Ñ*×4Ñ4×?Ñ?Õ@Ü�fœnÔ-ä,Ø×#Ñ#Ø—
‘
Ø—‘Ø—‘Ø×*Ñ*Ø+ôˆMð !'§
¡
ˆMÔØ!'§¡ˆMÕô "ŸH™H×2Ñ2Ø×#Ñ#Ø—
‘
Ø—‘Ø—‘Ø×*Ñ*ØóˆMð �=Š=Ü—‘“ñ 1Ø'-§}¡}�Ô$Ø%+§[¡[�Ô"÷1ð &,×%8Ñ%8ˆÔ"Ø$*×$6Ñ$6ˆÔ!Ø,2×,FÑ,FˆÔ)Ø!'§¡ˆÔÜ�6˜9Ô%Ø$*§N¡NˆMÔ!Ø×,Ñ,Ó.ò U‰ˆˆeØ× Ñ  Ô'=¸eÀ]Ó'SÕTðUàØÐ÷1ð 1ús   Ä(#G+Ç+G4c                   ó  ‡ — e Zd ZdZddej
                  dddddfdededede	d	e
eef   d
edeeej                        fˆ fd„Zdededededee   dee   dee   fˆ fd„Zdej*                  dej*                  fd„Zdefd„Zˆ xZS )ÚFrozenBatchNormAct2da$  
    BatchNormAct2d where the batch statistics and the affine parameters are fixed

    Args:
        num_features (int): Number of features ``C`` from an expected input of size ``(N, C, H, W)``
        eps (float): a value added to the denominator for numerical stability. Default: 1e-5
    r(   TNr)   r*   r   r   r   r   r.   c
                 óÖ  •— ||	dœ}
t         ‰| �  «        || _        | j                  dt	        j
                  |fi |
¤Ž«       | j                  dt	        j                  |fi |
¤Ž«       | j                  dt	        j                  |fi |
¤Ž«       | j                  dt	        j
                  |fi |
¤Ž«       |� |«       nt        j                  «       | _	        t        ||||¬«      | _        y )Nr0   rF   rG   rB   rC   r3   )r4   r5   r*   Úregister_bufferr[   ÚonesÚzerosr	   r!   r7   r$   r"   )r8   r)   r*   r   r   r   r   r.   r1   r2   Úddr:   s              €r#   r5   zFrozenBatchNormAct2d.__init__Ü   sÂ   ø€ ð ¨Ñ/ˆÜ‰ÑÔØˆŒØ×Ñ˜X¤u§z¡z°,Ñ'EÀ"Ñ'EÔFØ×Ñ˜V¤U§[¡[°Ñ%DÀÑ%DÔEØ×Ñ˜^¬U¯[©[¸Ñ-LÈÑ-LÔMØ×Ñ˜]¬E¯J©J°|Ñ,JÀrÑ,JÔKà$.Ð$:‘J”LÄÇÁÃˆŒ	Ü˜y°ZÈÐ\eÔfˆ�r%   Ú
state_dictÚprefixÚlocal_metadataÚstrictÚmissing_keysÚunexpected_keysÚ
error_msgsc           	      óH   •— |dz   }||v r||= t         ‰	| �  |||||||«       y )Nr?   )r4   Ú_load_from_state_dict)
r8   rt   ru   rv   rw   rx   ry   rz   Únum_batches_tracked_keyr:   s
            €r#   r|   z*FrozenBatchNormAct2d._load_from_state_dictó   s?   ø€ ð #)Ð+@Ñ"@ÐØ" jÑ0ØÐ2Ð3ä‰Ñ%Ø˜ °¸ÀoÐWaõ	
r%   rH   r   c                 ó–  — | j                   j                  dddd«      }| j                  j                  dddd«      }| j                  j                  dddd«      }| j                  j                  dddd«      }||| j
                  z   j                  «       z  }|||z  z
  }||z  |z   }| j                  | j                  |«      «      }|S )Nr   éÿÿÿÿ)	rF   ÚreshaperG   rC   rB   r*   Úrsqrtr"   r7   )r8   rH   ÚwÚbÚrvÚrmÚscalerG   s           r#   rK   zFrozenBatchNormAct2d.forward  sÁ   € ð �K‰K×Ñ  2 q¨!Ó,ˆØ�I‰I×Ñ˜a  Q¨Ó*ˆØ×Ñ×%Ñ% a¨¨Q°Ó2ˆØ×Ñ×&Ñ& q¨"¨a°Ó3ˆØ�R˜$Ÿ(™(‘]×)Ñ)Ó+Ñ+ˆØ�2˜‘:‰~ˆØ�‰I˜ÑˆØ�H‰H�T—Y‘Y˜q“\Ó"ˆØˆr%   c                 óœ   — | j                   j                  › d| j                  j                  d   › d| j                  › d| j
                  › d�S )Nú(r   z, eps=z, act=ú))r:   rL   rF   Úshaper*   r"   )r8   s    r#   Ú__repr__zFrozenBatchNormAct2d.__repr__  sJ   € Ø—.‘.×)Ñ)Ð*¨!¨D¯K©K×,=Ñ,=¸aÑ,@Ð+AÀÈÏÉÀzÐQWÐX\×X`ÑX`ÐWaÐabÐcÐcr%   )rL   rM   rN   rO   r	   rP   rQ   rA   rR   r   r   rS   r   r   r   rT   r5   Údictr   r|   r[   r\   rK   r‹   rU   rV   s   @r#   rn   rn   Ò   s
  ø„ ñð Ø"Ø#%§7¡7Ø)-Ø Ø48ØØñgàðgð ðgð ð	gð
 !ðgð ˜S #˜X™ðgð ðgð !  b§i¡i¡Ñ1õgð.
àð
ð ð
ð ð	
ð
 ð
ð ˜3‘ið
ð ˜c™ð
ð ˜‘Iõ
ð$˜Ÿ™ð ¨%¯,©,ó ðd˜#÷ dr%   rn   c                 óF  — | }t        | t        t        f«      �rDt        | j                  «      }| j                  |_        | j
                  |_        | j
                  r‚| j                  j                  j                  «       j                  «       |j                  _        | j                  j                  j                  «       j                  «       |j                  _        | j                  j                  |j                  _        | j                  j                  |j                  _        | j                  |_        | j                  |_        | j                  |_        |S t        | t         j"                  j$                  j&                  j(                  t         j"                  j$                  j&                  j*                  f«      �r"t-        | j                  «      }| j                  |_        | j
                  |_        | j
                  r‚| j                  j                  j                  «       j                  «       |j                  _        | j                  j                  j                  «       j                  «       |j                  _        | j                  j                  |j                  _        | j                  j                  |j                  _        | j                  |_        |S | j/                  «       D ]'  \  }}t1        |«      }||usŒ|j3                  ||«       Œ) |S )aÂ  
    Converts all `BatchNorm2d` and `SyncBatchNorm` or `BatchNormAct2d` and `SyncBatchNormAct2d` layers
    of provided module into `FrozenBatchNorm2d` or `FrozenBatchNormAct2d` respectively.

    Args:
        module (torch.nn.Module): Any PyTorch module.

    Returns:
        torch.nn.Module: Resulting module

    Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
    )r`   r'   rX   rn   r)   r,   rF   ÚdataÚcloneÚdetachrG   rB   rC   r*   r7   r"   r[   r	   ra   rb   ÚBatchNorm2drd   r   rf   Úfreeze_batch_norm_2drg   ©ri   Úresrk   rl   Ú	new_childs        r#   r’   r’     s5  € ð €CÜ�&œ>Ô+;Ð<Õ=Ü" 6×#6Ñ#6Ó7ˆØ!×.Ñ.ˆÔØ—]‘]ˆŒ
Ø�=Š=Ø$Ÿm™m×0Ñ0×6Ñ6Ó8×?Ñ?ÓAˆC�J‰JŒOØ"ŸK™K×,Ñ,×2Ñ2Ó4×;Ñ;Ó=ˆC�H‰HŒMØ &× 3Ñ 3× 8Ñ 8ˆ×ÑÔØ%×1Ñ1×6Ñ6ˆ�‰ÔØ—*‘*ˆŒØ—;‘;ˆŒØ—*‘*ˆŒð  €Jô 
�FœUŸX™X×-Ñ-×7Ñ7×CÑCÄUÇXÁX×EUÑEU×E_ÑE_×EmÑEmÐnÕ	oÜ × 3Ñ 3Ó4ˆØ!×.Ñ.ˆÔØ—]‘]ˆŒ
Ø�=Š=Ø$Ÿm™m×0Ñ0×6Ñ6Ó8×?Ñ?ÓAˆC�J‰JŒOØ"ŸK™K×,Ñ,×2Ñ2Ó4×;Ñ;Ó=ˆC�H‰HŒMØ &× 3Ñ 3× 8Ñ 8ˆ×ÑÔØ%×1Ñ1×6Ñ6ˆ�‰ÔØ—*‘*ˆŒð €Jð	 "×0Ñ0Ó2ò 	0‰KˆD�%Ü,¨UÓ3ˆIØ Ò%Ø—‘˜t YÕ/ð	0ð €Jr%   c                 ó.  — | }t        | t        «      �r"t        | j                  «      }| j                  r‚| j
                  j                  j                  «       j                  «       |j
                  _        | j                  j                  j                  «       j                  «       |j                  _        | j                  j                  |j                  _        | j                  j                  |j                  _        | j                  |_        | j                  |_        | j                  |_        |S t        | t        «      �rt         j"                  j%                  | j                  «      }| j                  r‚| j
                  j                  j                  «       j                  «       |j
                  _        | j                  j                  j                  «       j                  «       |j                  _        | j                  j                  |j                  _        | j                  j                  |j                  _        | j                  |_        |S | j'                  «       D ]'  \  }}t)        |«      }||usŒ|j+                  ||«       Œ) |S )a  
    Converts all `FrozenBatchNorm2d` layers of provided module into `BatchNorm2d`. If `module` is itself and instance
    of `FrozenBatchNorm2d`, it is converted into `BatchNorm2d` and returned. Otherwise, the module is walked
    recursively and submodules are converted in place.

    Args:
        module (torch.nn.Module): Any PyTorch module.

    Returns:
        torch.nn.Module: Resulting module

    Inspired by https://github.com/pytorch/pytorch/blob/a5895f85be0f10212791145bfedc0261d364f103/torch/nn/modules/batchnorm.py#L762
    )r`   rn   r'   r)   r,   rF   rŽ   r�   r�   rG   rB   rC   r*   r7   r"   r   r[   r	   r‘   rf   Úunfreeze_batch_norm_2drg   r“   s        r#   r—   r—   B  sØ  € ð €CÜ�&Ô.Õ/Ü˜V×0Ñ0Ó1ˆØ�=Š=Ø$Ÿm™m×0Ñ0×6Ñ6Ó8×?Ñ?ÓAˆC�J‰JŒOØ"ŸK™K×,Ñ,×2Ñ2Ó4×;Ñ;Ó=ˆC�H‰HŒMØ &× 3Ñ 3× 8Ñ 8ˆ×ÑÔØ%×1Ñ1×6Ñ6ˆ�‰ÔØ—*‘*ˆŒØ—;‘;ˆŒØ—*‘*ˆŒð €Jô 
�FÔ-Õ	.Ü�h‰h×"Ñ" 6×#6Ñ#6Ó7ˆØ�=Š=Ø$Ÿm™m×0Ñ0×6Ñ6Ó8×?Ñ?ÓAˆC�J‰JŒOØ"ŸK™K×,Ñ,×2Ñ2Ó4×;Ñ;Ó=ˆC�H‰HŒMØ &× 3Ñ 3× 8Ñ 8ˆ×ÑÔØ%×1Ñ1×6Ñ6ˆ�‰ÔØ—*‘*ˆŒð €Jð	 "×0Ñ0Ó2ò 	0‰KˆD�%Ü.¨uÓ5ˆIØ Ò%Ø—‘˜t YÕ/ð	0ð €Jr%   Únum_channelsÚ
num_groupsÚ
group_sizec                 ó(   — |r| |z  dk(  sJ ‚| |z  S |S )Nr   © )r˜   r™   rš   s      r#   Ú_num_groupsr�   k  s)   € ÙØ˜jÑ(¨AÒ-Ð-Ð-Ø˜zÑ)Ð)ØÐr%   c                   óä   ‡ — e Zd ZU ej                  j
                  e   ed<   dddddej                  dddddfde
de
ded	ed
ee
   dededeeef   dedeeej$                        fˆ fd„Zd„ Zˆ xZS )ÚGroupNormActÚ
_fast_normé    r(   TNr˜   r™   r*   r,   rš   r   r   r   r   r.   c                 óÐ   •— t         ‰| �  t        |||«      |||||¬«       |
� |
«       nt        j                  «       | _        t        |||	|¬«      | _        t        «       | _	        y )N©r*   r,   r1   r2   r3   )
r4   r5   r�   r	   r!   r7   r$   r"   r   r    )r8   r˜   r™   r*   r,   rš   r   r   r   r   r.   r1   r2   r:   s                €r#   r5   zGroupNormAct.__init__v  sf   ø€ ô 	‰ÑÜ˜ j°*Ó=ØØØØØð 	ô 	
ð %/Ð$:‘J”LÄÇÁÃˆŒ	Ü˜y°ZÈÐ\eÔfˆŒä&›.ˆ�r%   c                 óT  — | j                   r8t        || j                  | j                  | j                  | j
                  «      }nAt        j                  || j                  | j                  | j                  | j
                  «      }| j                  |«      }| j                  |«      }|S ©N©
r    r   r™   rF   rG   r*   rD   Ú
group_normr7   r"   ©r8   rH   s     r#   rK   zGroupNormAct.forward’  óp   € Ø�?Š?Ü  4§?¡?°D·K±KÀÇÁÈDÏHÉHÓU‰Aä—‘˜Q §¡°·±¸d¿i¹iÈÏÉÓRˆAØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØˆr%   )rL   rM   rN   r[   ÚjitÚFinalrR   Ú__annotations__r	   rP   rQ   rA   r   r   r   rS   r   r   rT   r5   rK   rU   rV   s   @r#   rŸ   rŸ   r  sÂ   ø… Ø—	‘	—‘ Ñ%Ó%ð !ØØØ(,Ø"Ø#%§7¡7Ø)-Ø Ø48ØØñ)àð)ð ð)ð ð	)ð
 ð)ð ! ™ð)ð ð)ð !ð)ð ˜S #˜X™ð)ð ð)ð !  b§i¡i¡Ñ1õ)ö8r%   rŸ   c                   óÒ   ‡ — e Zd ZU ej                  j
                  e   ed<   dddej                  dddddf	de
dededed	ed
eeef   dedeeej$                        fˆ fd„Zd„ Zˆ xZS )ÚGroupNorm1Actr    r(   TNr˜   r*   r,   r   r   r   r   r.   c                 óº   •— t         ‰| �  d||||	|
¬«       |� |«       nt        j                  «       | _        t        ||||¬«      | _        t        «       | _        y )Nr   r£   r3   ©	r4   r5   r	   r!   r7   r$   r"   r   r    )r8   r˜   r*   r,   r   r   r   r   r.   r1   r2   r:   s              €r#   r5   zGroupNorm1Act.__init__Ÿ  sR   ø€ ô 	‰Ñ˜˜L¨c¸&ÈÐW\ÐÔ]Ø$.Ð$:‘J”LÄÇÁÃˆŒ	Ü˜y°ZÈÐ\eÔfˆŒä&›.ˆ�r%   c                 óT  — | j                   r8t        || j                  | j                  | j                  | j
                  «      }nAt        j                  || j                  | j                  | j                  | j
                  «      }| j                  |«      }| j                  |«      }|S r¥   r¦   r¨   s     r#   rK   zGroupNorm1Act.forward²  r©   r%   ©rL   rM   rN   r[   rª   r«   rR   r¬   r	   rP   rQ   rA   r   r   rS   r   r   r   rT   r5   rK   rU   rV   s   @r#   r®   r®   œ  s¤   ø… Ø—	‘	—‘ Ñ%Ó%ð
 ØØ"Ø#%§7¡7Ø)-Ø Ø48ØØñ)àð)ð ð)ð ð	)ð
 ð)ð !ð)ð ˜S #˜X™ð)ð ð)ð !  b§i¡i¡Ñ1õ)ö&r%   r®   c                   óô   ‡ — e Zd ZU ej                  j
                  e   ed<   dddej                  dddfde
eee   ej                  f   dededed	ed
eeef   dedeeej*                        fˆ fd„Zd„ Zˆ xZS )ÚLayerNormActr    r(   TNÚnormalization_shaper*   r,   r   r   r   r   r.   c	                 ó¶   •— t        ‰
| �  |f||dœ|	¤Ž |� |«       nt        j                  «       | _        t        ||||¬«      | _        t        «       | _        y ©N)r*   Úelementwise_affiner3   r°   ©r8   rµ   r*   r,   r   r   r   r   r.   Úkwargsr:   s             €r#   r5   zLayerNormAct.__init__¿  sR   ø€ ô 	‰ÑÐ,Ð[°#È&Ñ[ÐTZÒ[Ø$.Ð$:‘J”LÄÇÁÃˆŒ	Ü˜y°ZÈÐ\eÔfˆŒä&›.ˆ�r%   c                 óT  — | j                   r8t        || j                  | j                  | j                  | j
                  «      }nAt        j                  || j                  | j                  | j                  | j
                  «      }| j                  |«      }| j                  |«      }|S r¥   )
r    r   Únormalized_shaperF   rG   r*   rD   Ú
layer_normr7   r"   r¨   s     r#   rK   zLayerNormAct.forwardÑ  sw   € Ø�?Š?Ü  4×#8Ñ#8¸$¿+¹+ÀtÇyÁyÐRV×RZÑRZÓ[‰Aä—‘˜Q × 5Ñ 5°t·{±{ÀDÇIÁIÈtÏxÉxÓXˆAØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØˆr%   )rL   rM   rN   r[   rª   r«   rR   r¬   r	   rP   r   rQ   r   ÚSizerA   r   r   rS   r   r   r   rT   r5   rK   rU   rV   s   @r#   r´   r´   ¼  s²   ø… Ø—	‘	—‘ Ñ%Ó%ð
 ØØ"Ø#%§7¡7Ø)-Ø Ø48ñ)à!& s¨D°©I°u·z±zÐ'AÑ!Bð)ð ð)ð ð	)ð
 ð)ð !ð)ð ˜S #˜X™ð)ð ð)ð !  b§i¡i¡Ñ1õ)ö$r%   r´   c                   óº   ‡ — e Zd Zdddej                  dddfdeeee   ej                  f   de
dededed	eeef   d
edeeej$                        fˆ fd„Zd„ Zˆ xZS )ÚLayerNormActFp32r(   TNrµ   r*   r,   r   r   r   r   r.   c	                 ó˜   •— t        ‰
| �  |f||dœ|	¤Ž |� |«       nt        j                  «       | _        t        ||||¬«      | _        y r·   ©r4   r5   r	   r!   r7   r$   r"   r¹   s             €r#   r5   zLayerNormActFp32.__init__Ý  sI   ø€ ô 	‰ÑÐ,Ð[°#È&Ñ[ÐTZÒ[Ø$.Ð$:‘J”LÄÇÁÃˆŒ	Ü˜y°ZÈÐ\eÔfˆ�r%   c                 ó’  — | j                   �| j                   j                  «       nd }| j                  �| j                  j                  «       nd }t        j                  |j                  «       | j
                  ||| j                  «      j                  |j                  «      }| j                  |«      }| j                  |«      }|S r¥   )rF   rA   rG   rD   r½   r¼   r*   Útor2   r7   r"   ©r8   rH   rF   rG   s       r#   rK   zLayerNormActFp32.forwardí  s‘   € Ø(,¯©Ð(?�—‘×"Ñ"Ô$ÀTˆØ$(§I¡IÐ$9ˆt�y‰y�‰Ô ¸tˆÜ�L‰L˜Ÿ™› D×$9Ñ$9¸6À4ÈÏÉÓR×UÑUÐVW×V]ÑV]Ó^ˆØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØˆr%   )rL   rM   rN   r	   rP   r   rQ   r   r[   r¾   rA   rR   r   r   rS   r   r   r   rT   r5   rK   rU   rV   s   @r#   rÀ   rÀ   Û  s¨   ø„ ð
 ØØ"Ø#%§7¡7Ø)-Ø Ø48ñgà!& s¨D°©I°u·z±zÐ'AÑ!Bðgð ðgð ð	gð
 ðgð !ðgð ˜S #˜X™ðgð ðgð !  b§i¡i¡Ñ1õgö r%   rÀ   c                   óÎ   ‡ — e Zd ZU ej                  j
                  e   ed<   dddej                  dddfde
dededed	ed
eeef   dedeeej$                        fˆ fd„Zd„ Zˆ xZS )ÚLayerNormAct2dr    r(   TNr˜   r*   r,   r   r   r   r   r.   c	                 ó¶   •— t        ‰
| �  |f||dœ|	¤Ž |� |«       nt        j                  «       | _        t        ||||¬«      | _        t        «       | _        y r·   r°   ©r8   r˜   r*   r,   r   r   r   r   r.   rº   r:   s             €r#   r5   zLayerNormAct2d.__init__ù  sP   ø€ ô 	‰Ñ˜ÐT¨3À6ÑTÈVÒTØ$.Ð$:‘J”LÄÇÁÃˆŒ	Ü˜y°ZÈÐ\eÔfˆŒÜ&›.ˆ�r%   c                 ó¤  — |j                  dddd«      }| j                  r8t        || j                  | j                  | j
                  | j                  «      }nAt        j                  || j                  | j                  | j
                  | j                  «      }|j                  dddd«      }| j                  |«      }| j                  |«      }|S ©Nr   é   é   r   )Úpermuter    r   r¼   rF   rG   r*   rD   r½   r7   r"   r¨   s     r#   rK   zLayerNormAct2d.forward
  sŸ   € Ø�I‰I�a˜˜A˜qÓ!ˆØ�?Š?Ü  4×#8Ñ#8¸$¿+¹+ÀtÇyÁyÐRV×RZÑRZÓ[‰Aä—‘˜Q × 5Ñ 5°t·{±{ÀDÇIÁIÈtÏxÉxÓXˆAØ�I‰I�a˜˜A˜qÓ!ˆØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØˆr%   r²   rV   s   @r#   rÇ   rÇ   ö  sž   ø… Ø—	‘	—‘ Ñ%Ó%ð
 ØØ"Ø#%§7¡7Ø)-Ø Ø48ñ)àð)ð ð)ð ð	)ð
 ð)ð !ð)ð ˜S #˜X™ð)ð ð)ð !  b§i¡i¡Ñ1õ)ö"	r%   rÇ   c                   ó”   ‡ — e Zd Zdddej                  dddfdededededed	e	e
ef   d
edeeej                        fˆ fd„Zd„ Zˆ xZS )ÚLayerNormAct2dFp32r(   TNr˜   r*   r,   r   r   r   r   r.   c	                 ó˜   •— t        ‰
| �  |f||dœ|	¤Ž |� |«       nt        j                  «       | _        t        ||||¬«      | _        y r·   rÂ   rÉ   s             €r#   r5   zLayerNormAct2dFp32.__init__  sG   ø€ ô 	‰Ñ˜ÐT¨3À6ÑTÈVÒTØ$.Ð$:‘J”LÄÇÁÃˆŒ	Ü˜y°ZÈÐ\eÔfˆ�r%   c                 óâ  — |j                  dddd«      }| j                  �| j                  j                  «       nd }| j                  �| j                  j                  «       nd }t	        j
                  |j                  «       | j                  ||| j                  «      j                  |j                  «      }|j                  dddd«      }| j                  |«      }| j                  |«      }|S rË   )rÎ   rF   rA   rG   rD   r½   r¼   r*   rÄ   r2   r7   r"   rÅ   s       r#   rK   zLayerNormAct2dFp32.forward(  s¹   € Ø�I‰I�a˜˜A˜qÓ!ˆØ(,¯©Ð(?�—‘×"Ñ"Ô$ÀTˆØ$(§I¡IÐ$9ˆt�y‰y�‰Ô ¸tˆÜ�L‰L˜Ÿ™› D×$9Ñ$9¸6À4ÈÏÉÓR×UÑUÐVW×V]ÑV]Ó^ˆØ�I‰I�a˜˜A˜qÓ!ˆØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØˆr%   )rL   rM   rN   r	   rP   rQ   rA   rR   r   r   rS   r   r   r   rT   r5   rK   rU   rV   s   @r#   rÐ   rÐ     s”   ø„ ð
 ØØ"Ø#%§7¡7Ø)-Ø Ø48ñgàðgð ðgð ð	gð
 ðgð !ðgð ˜S #˜X™ðgð ðgð !  b§i¡i¡Ñ1õgö r%   rÐ   c                   óÊ   ‡ — e Zd ZdZdddej
                  dddfdedededed	e	d
e
eef   dedeeej                        fˆ fd„Zdej$                  dej$                  fd„Zˆ xZS )Ú
RmsNormActá*   RMSNorm + Activation for '2D' NCHW tensors

    NOTE: It's currently (2025-05-10) faster to use an eager 2d kernel that does reduction
    on dim=1 than to permute and use internal PyTorch F.rms_norm, this may change if something
    like https://github.com/pytorch/pytorch/pull/150576 lands.
    ç�íµ ÷Æ°>TNr˜   r*   r,   r   r   r   r   r.   c	                 ó¶   •— t        ‰
| �  d|||dœ|	¤Ž |� |«       nt        j                  «       | _        t        ||||¬«      | _        t        «       | _        y ©N)Úchannelsr*   r,   r3   rœ   r°   rÉ   s             €r#   r5   zRmsNormAct.__init__:  óP   ø€ ô 	‰ÑÐQ ,°CÀÑQÈ&ÒQØ$.Ð$:‘J”LÄÇÁÃˆŒ	Ü˜y°ZÈÐ\eÔfˆŒÜ&›.ˆ�r%   rH   r   c                 ó  — | j                   r-t        || j                  | j                  | j                  «      }n,t        || j                  | j                  | j                  «      }| j                  |«      }| j                  |«      }|S r¥   )r    r   r¼   rF   r*   r   r7   r"   r¨   s     r#   rK   zRmsNormAct.forwardK  sd   € Ø�?Š?Ü˜a ×!6Ñ!6¸¿¹ÀTÇXÁXÓN‰Aä˜˜D×1Ñ1°4·;±;ÀÇÁÓIˆAØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØˆr%   ©rL   rM   rN   rO   r	   rP   rQ   rA   rR   r   r   rS   r   r   r   rT   r5   r[   r\   rK   rU   rV   s   @r#   rÔ   rÔ   3  ó¦   ø„ ñð ØØ"Ø#%§7¡7Ø)-Ø Ø48ñ)àð)ð ð)ð ð	)ð
 ð)ð !ð)ð ˜S #˜X™ð)ð ð)ð !  b§i¡i¡Ñ1õ)ð"˜Ÿ™ð ¨%¯,©,÷ r%   rÔ   c                   óÊ   ‡ — e Zd ZdZdddej
                  dddfdedededed	e	d
e
eef   dedeeej                        fˆ fd„Zdej$                  dej$                  fd„Zˆ xZS )ÚRmsNormActFp32rÕ   rÖ   TNr˜   r*   r,   r   r   r   r   r.   c	                 ó˜   •— t        ‰
| �  d|||dœ|	¤Ž |� |«       nt        j                  «       | _        t        ||||¬«      | _        y rØ   rÂ   rÉ   s             €r#   r5   zRmsNormActFp32.__init__\  óG   ø€ ô 	‰ÑÐQ ,°CÀÑQÈ&ÒQØ$.Ð$:‘J”LÄÇÁÃˆŒ	Ü˜y°ZÈÐ\eÔfˆ�r%   rH   r   c                 ó,  — | j                   �| j                   j                  «       nd }t        |j                  «       | j                  || j                  «      j                  |j                  «      }| j                  |«      }| j                  |«      }|S r¥   )	rF   rA   r   r¼   r*   rÄ   r2   r7   r"   ©r8   rH   rF   s      r#   rK   zRmsNormActFp32.forwardl  sm   € Ø(,¯©Ð(?�—‘×"Ñ"Ô$ÀTˆÜ�Q—W‘W“Y × 5Ñ 5°v¸t¿x¹xÓH×KÑKÈAÏGÉGÓTˆØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØˆr%   rÜ   rV   s   @r#   rß   rß   U  ó¯   ø„ ñð ØØ"Ø#%§7¡7Ø)-Ø Ø48ñgàðgð ðgð ð	gð
 ðgð !ðgð ˜S #˜X™ðgð ðgð !  b§i¡i¡Ñ1õgð ˜Ÿ™ð ¨%¯,©,÷ r%   rß   c                   óÊ   ‡ — e Zd ZdZdddej
                  dddfdedededed	e	d
e
eef   dedeeej                        fˆ fd„Zdej$                  dej$                  fd„Zˆ xZS )ÚRmsNormAct2drÕ   rÖ   TNr˜   r*   r,   r   r   r   r   r.   c	                 ó¶   •— t        ‰
| �  d|||dœ|	¤Ž |� |«       nt        j                  «       | _        t        ||||¬«      | _        t        «       | _        y rØ   r°   rÉ   s             €r#   r5   zRmsNormAct2d.__init__{  rÚ   r%   rH   r   c                 ó  — | j                   r-t        || j                  | j                  | j                  «      }n,t        || j                  | j                  | j                  «      }| j                  |«      }| j                  |«      }|S r¥   )r    r   r¼   rF   r*   r   r7   r"   r¨   s     r#   rK   zRmsNormAct2d.forwardŒ  sd   € Ø�?Š?Ü  4×#8Ñ#8¸$¿+¹+ÀtÇxÁxÓP‰Aä˜1˜d×3Ñ3°T·[±[À$Ç(Á(ÓKˆAØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØˆr%   rÜ   rV   s   @r#   ræ   ræ   t  rÝ   r%   ræ   c                   óÊ   ‡ — e Zd ZdZdddej
                  dddfdedededed	e	d
e
eef   dedeeej                        fˆ fd„Zdej$                  dej$                  fd„Zˆ xZS )ÚRmsNormAct2dFp32rÕ   rÖ   TNr˜   r*   r,   r   r   r   r   r.   c	                 ó˜   •— t        ‰
| �  d|||dœ|	¤Ž |� |«       nt        j                  «       | _        t        ||||¬«      | _        y rØ   rÂ   rÉ   s             €r#   r5   zRmsNormAct2dFp32.__init__�  rá   r%   rH   r   c                 ó,  — | j                   �| j                   j                  «       nd }t        |j                  «       | j                  || j                  «      j                  |j                  «      }| j                  |«      }| j                  |«      }|S r¥   )	rF   rA   r   r¼   r*   rÄ   r2   r7   r"   rã   s      r#   rK   zRmsNormAct2dFp32.forward­  sm   € Ø(,¯©Ð(?�—‘×"Ñ"Ô$ÀTˆÜ�q—w‘w“y $×"7Ñ"7¸ÀÇÁÓJ×MÑMÈaÏgÉgÓVˆØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØˆr%   rÜ   rV   s   @r#   rê   rê   –  rä   r%   rê   )NFTr¥   )=rO   Útypingr   r   r   r   r   r   r[   r	   Útorch.nnr
   rD   Útorchvision.ops.miscr   Ú_fxr   Ú
create_actr   Ú	fast_normr   r   r   r   r   r   Únormr   r   Útrace_utilsr   r   Útorch.nn.functionalr   ÚImportErrorrS   rR   rT   r$   r‘   r'   rd   rX   rh   rn   r’   r—   rQ   r�   Ú	GroupNormrŸ   r®   Ú	LayerNormr´   rÀ   rÇ   rÐ   rÔ   rß   ræ   rê   rœ   r%   r#   ú<module>rù      sÖ  ðñ÷ :× 9ã Ý Ý $Ý 2å (Ý (÷÷ ÷ %Ý  Ý ð$Ý,ð &*Ø"'Øñ	1Øð1à˜˜c˜‘Nð1ð ˜$‘ð1ð ð	1ð
 ‡Y�Yó1ð ô\�R—^‘^ó \ó ð\ð~ ô�r×'Ñ'ó ó ðó(ðV ô@d˜5Ÿ8™8Ÿ?™?ó @dó ð@dòF)òX&ðR˜cð ¨sð Àó ô'�2—<‘<ô 'ôT�B—L‘Lô ô@�2—<‘<ô ô>�r—|‘|ô ô6�R—\‘\ô ô@˜Ÿ™ô ô:�ô ôD�Wô ô>�9ô ôD�yõ øða ò $ß#Ð#ð$ús   ÁF ÆF*Æ)F*