Ë
    éÓ^j¹²  ã                   óV  — d Z ddlmZ ddlmZmZ ddlmZ ddlm	Z	m
Z
mZmZmZ ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZ d	d
lm Z  d	dl!m"Z" d	dl#m$Z$ d	dl%m&Z&m'Z' ddgZ(e G d„ d«      «       Z) G d„ dejT                  «      Z+�dde,de-de
fd„Z. G d„ dejT                  «      Z/e" G d„ dejT                  «      «       Z0	 	 	 	 	 	 �dde1de1de,d ee
   d!ee
   d"e2deejf                  e1ee,e	f   f   fd#„Z4 e5dd$d%d&d'd(d)d*d+d,d-d.d/d0d1¬2«      Z6 G d3„ dejT                  «      Z7	 	 	 	 	 �dd4ee1d5f   d6ee1d5f   d7ee1   d!e,d8ee,   d9eee,e	f      de)fd:„Z8�dd4ee1d5f   d6ee1d5f   de)fd;„Z9	 	 	 	 	 	 	 �dd4ee1d5f   d6ee1d5f   d7e1d=e-d>e-d!e,d8e,d9eee,e	f      de)fd?„Z:	 	 	 �d	d4ee1d5f   d6ee1d5f   d!e,d@e2de)f
dA„Z; e5�d
i dB e;dC¬D«      “dE e;dF¬D«      “dG e;dH¬D«      “dI e;dJ¬D«      “dK e;dL¬D«      “dM e;dN¬D«      “dO e;dP¬D«      “dQ e:dC¬D«      “dR e:dF¬D«      “dS e:dH¬D«      “dT e:dJ¬D«      “dU e:dL¬D«      “dV e:dN¬D«      “dW e:dP¬D«      “dX e:dY¬D«      “dZ e:dCd[d\d] e5d]d^¬_«      d`¬a«      “db e:dCd[d\d]dc e5«       d`¬d«      “de e:dFdfd\d]dc e5«       d`¬d«      “dg e:dHdfd\d]dc e5«       d`¬d«      “dh e:dJdfd\d]dc e5«       d`¬d«      “di e9dj¬D«      “dk e9dl¬D«      “dm e9dndo¬p«      “dq e9drds¬p«      “dt e9dudv¬p«      “dw e9dxdy¬p«      “dz e8d{¬D«      “d| e8d}¬D«      “d~ e8d¬D«      “d€ e8d{d< e5d�¬‚«      ¬ƒ«      “d„ e8d}d< e5d�¬‚«      ¬ƒ«      “d… e8dd< e5d�¬‚«      ¬ƒ«      “d† e8d{dc e5«       ¬ƒ«      “d‡ e8d}dc e5«       ¬ƒ«      “dˆ e8ddc e5«       ¬ƒ«      “d‰ e:dŠd‹d[d^d] e5d]d^¬_«      d`¬Œ«      “ŽZ<�dd�e,dŽe2d�e	de7fd�„Z=�dd‘e,d�e	dee,e	f   fd’„Z> e&i d“ e>d”d•d–d—d˜d™dš¬›«      “dœ e>d”d�dždŸd d¡dš¬›«      “d¢ e>d”d£d¤d˜d¥d¦dš¬›«      “d§ e>d”d¨d©d dªd«dš¬›«      “d¬ e>d”d­d®d¯d°d±dš¬›«      “d² e>d”d³d´dªdµd¶dš¬›«      “d· e>d”d¸d¹dºd»d¼dš¬›«      “dQ e>dd–d—d˜¬½«      “dR e>ddždŸd ¬½«      “dS e>dd¤d˜d¥¬½«      “dT e>dd©d dª¬½«      “dU e>dd®d¯d°¬½«      “dV e>dd´dªdµ¬½«      “dW e>dd¹dºd»¬½«      “dX e>dd¾d¿dÀ¬½«      “dÁ e>d”dÂdždŸdÃd¬Ä«      “dÅ e>d”dÆdždŸdÃd¬Ä«      “i dÇ e>d”dÈd¤d˜d d¬Ä«      “dÉ e>d”dÊd©d d¯d¬Ä«      “dh e>ddËd¥dºd¬Ì«      “di e>dd–d—d˜dÍ¬Î«      “dÏ e>d”dÐd¤d˜dÃdÍ¬Ñ«      “dm e>dd¤dÒdÓdÍ¬Î«      “dq e>ddÔdÃd dÍ¬Î«      “dt e>dd©d d¯dÍ¬Î«      “dw e>dd®d¯dÕdÍ¬Î«      “dz e>ddÍ¬Ö«      “d× e>d”dØd¤d˜dÃd«dÍ¬Ù«      “d~ e>ddÍ¬Ö«      “d€ e>ddÍ¬Ö«      “d„ e>ddÍ¬Ö«      “d… e>ddÍ¬Ö«      “d† e>ddÍ¬Ö«      “d‡ e>ddÍ¬Ö«      “¥ e>ddÍ¬Ö«       e>d”dÚdÚdÛdÜdÝ¬Þ«      dßœ¥«      Z?e'�ddŽe2d�e	de7fdà„«       Z@e'�ddŽe2d�e	de7fdá„«       ZAe'�ddŽe2d�e	de7fdâ„«       ZBe'�ddŽe2d�e	de7fdã„«       ZCe'�ddŽe2d�e	de7fdä„«       ZDe'�ddŽe2d�e	de7fdå„«       ZEe'�ddŽe2d�e	de7fdæ„«       ZFe'�ddŽe2d�e	de7fdç„«       ZGe'�ddŽe2d�e	de7fdè„«       ZHe'�ddŽe2d�e	de7fdé„«       ZIe'�ddŽe2d�e	de7fdê„«       ZJe'�ddŽe2d�e	de7fdë„«       ZKe'�ddŽe2d�e	de7fdì„«       ZLe'�ddŽe2d�e	de7fdí„«       ZMe'�ddŽe2d�e	de7fdî„«       ZNe'�ddŽe2d�e	de7fdï„«       ZOe'�ddŽe2d�e	de7fdð„«       ZPe'�ddŽe2d�e	de7fdñ„«       ZQe'�ddŽe2d�e	de7fdò„«       ZRe'�ddŽe2d�e	de7fdó„«       ZSe'�ddŽe2d�e	de7fdô„«       ZTe'�ddŽe2d�e	de7fdõ„«       ZUe'�ddŽe2d�e	de7fdö„«       ZVe'�ddŽe2d�e	de7fd÷„«       ZWe'�ddŽe2d�e	de7fdø„«       ZXe'�ddŽe2d�e	de7fdù„«       ZYe'�ddŽe2d�e	de7fdú„«       ZZe'�ddŽe2d�e	de7fdû„«       Z[e'�ddŽe2d�e	de7fdü„«       Z\e'�ddŽe2d�e	de7fdý„«       Z]e'�ddŽe2d�e	de7fdþ„«       Z^e'�ddŽe2d�e	de7fdÿ„«       Z_e'�ddŽe2d�e	de7f�d „«       Z`e'�ddŽe2d�e	de7f�d„«       Zae'�ddŽe2d�e	de7f�d„«       Zbe'�ddŽe2d�e	de7f�d„«       Zcy(  aæ   Normalization Free Nets. NFNet, NF-RegNet, NF-ResNet (pre-activation) Models

Paper: `Characterizing signal propagation to close the performance gap in unnormalized ResNets`
    - https://arxiv.org/abs/2101.08692

Paper: `High-Performance Large-Scale Image Recognition Without Normalization`
    - https://arxiv.org/abs/2102.06171

Official Deepmind JAX code: https://github.com/deepmind/deepmind-research/tree/master/nfnets

Status:
* These models are a work in progress, experiments ongoing.
* Pretrained weights for two models so far, more to come.
* Model details updated to closer match official JAX code now that it's released
* NF-ResNet, NF-RegNet-B, and NFNet-F models supported

Hacked together by / copyright Ross Wightman, 2021.
é    )ÚOrderedDict)Ú	dataclassÚreplace)Úpartial)ÚAnyÚCallableÚDictÚOptionalÚTupleN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)
ÚClassifierHeadÚDropPathÚcalculate_drop_path_ratesÚAvgPool2dSameÚScaledStdConv2dÚScaledStdConv2dSameÚget_act_layerÚ
get_act_fnÚget_attnÚmake_divisibleé   )Úbuild_model_with_cfg)Úregister_notrace_module)Úcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelÚNormFreeNetÚNfCfgc                   ón  — e Zd ZU dZeeeeef   ed<   eeeeef   ed<   dZeed<   dZ	e
ed<   dZee   ed	<   dZee   ed
<   dZee
   ed<   dZeee
ef      ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   d Ze
ed!<   y)"r    z.Configuration for Normalization-Free Networks.ÚdepthsÚchannelsgš™™™™™É?ÚalphaÚ3x3Ú	stem_typeNÚstem_chsÚ
group_sizeÚ
attn_layerÚattn_kwargsç       @Ú	attn_gainç      ð?Úwidth_factorç      à?Úbottle_ratior   Únum_featuresé   Úch_divFÚregÚ
extra_convÚgamma_in_actÚsame_paddinggñhãˆµøä>Ústd_conv_epsÚskipinitÚzero_init_fcÚsiluÚ	act_layer) Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚintÚ__annotations__r$   Úfloatr&   Ústrr'   r
   r(   r)   r*   r	   r   r,   r.   r0   r1   r3   r4   Úboolr5   r6   r7   r8   r9   r:   r<   © ó    ú\/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/nfnet.pyr    r    &   s  … á8Ø�#�s˜C Ð$Ñ%Ó%Ø�C˜˜c 3Ð&Ñ'Ó'Ø€Eˆ5ÓØ€IˆsÓØ"€Hˆh�s‰mÓ"Ø $€J�˜‘Ó$Ø $€J�˜‘Ó$Ø,0€K�˜$˜s C˜x™.Ñ)Ó0Ø€IˆuÓØ€L�%ÓØ€L�%ÓØ€L�#ÓØ€FˆCƒOØ€CˆÓØ€J�ÓØ€L�$ÓØ€L�$ÓØ€L�%ÓØ€HˆdÓØ€L�$ÓØ€IˆsÔrG   c                   ój   ‡ — e Zd ZdZd	dededefˆ fd„Zdej                  dej                  fd„Z
ˆ xZS )
ÚGammaActz.Activation function with gamma scaling factor.Úact_typeÚgammaÚinplacec                 ó^   •— t         ‰| �  «        t        |«      | _        || _        || _        y)zÕInitialize GammaAct.

        Args:
            act_type: Type of activation function.
            gamma: Scaling factor for activation output.
            inplace: Whether to perform activation in-place.
        N)ÚsuperÚ__init__r   Úact_fnrL   rM   )ÚselfrK   rL   rM   Ú	__class__s       €rH   rP   zGammaAct.__init__C   s*   ø€ ô 	‰ÑÔÜ  Ó*ˆŒØˆŒ
Øˆ�rG   ÚxÚreturnc                 ón   — | j                  || j                  ¬«      j                  | j                  «      S )zzForward pass.

        Args:
            x: Input tensor.

        Returns:
            Scaled activation output.
        ©rM   )rQ   rM   Úmul_rL   ©rR   rT   s     rH   ÚforwardzGammaAct.forwardP   s*   € ð �{‰{˜1 d§l¡lˆ{Ó3×8Ñ8¸¿¹ÓDÐDrG   )Úrelur-   F)r=   r>   r?   r@   rD   rC   rE   rP   ÚtorchÚTensorrZ   Ú__classcell__©rS   s   @rH   rJ   rJ   @   s>   ø„ Ù8ñ ð °eð ÈDõ ð	E˜Ÿ™ð 	E¨%¯,©,÷ 	ErG   rJ   r-   rK   rL   rU   c                 ó2   ‡ ‡— ddt         dt        fˆ ˆfd„}|S )zÝCreate activation function factory with gamma scaling.

    Args:
        act_type: Type of activation function.
        gamma: Scaling factor for activation output.

    Returns:
        Activation function factory.
    rM   rU   c                 ó    •— t        ‰‰| ¬«      S )N)rL   rM   )rJ   )rM   rK   rL   s    €€rH   Ú_createzact_with_gamma.<locals>._createf   s   ø€ Ü˜¨°wÔ?Ð?rG   ©F)rE   rJ   )rK   rL   rb   s   `` rH   Úact_with_gammard   \   s   ù€ ñ@œð @¬(ö @à€NrG   c                   óˆ   ‡ — e Zd ZdZdddeddfdededededee   d	efˆ fd
„Zde	j                  de	j                  fd„Zˆ xZS )ÚDownsampleAvgzEAvgPool downsampling as in 'D' ResNet variants with dilation support.r   NÚin_chsÚout_chsÚstrideÚdilationÚfirst_dilationÚ
conv_layerc	                 ó  •— t         ‰| �  «        |dk(  r|nd}	|dkD  s|dkD  r2|	dk(  r|dkD  rt        nt        j                  }
 |
d|	dd¬«      | _        nt        j                  «       | _         |||dd||¬«      | _        y)a7  Initialize DownsampleAvg.

        Args:
            in_chs: Input channels.
            out_chs: Output channels.
            stride: Stride for downsampling.
            dilation: Dilation rate.
            first_dilation: First dilation rate (unused).
            conv_layer: Convolution layer type.
        r   é   TF)Ú	ceil_modeÚcount_include_pad)ri   ÚdeviceÚdtypeN)rO   rP   r   ÚnnÚ	AvgPool2dÚpoolÚIdentityÚconv)rR   rg   rh   ri   rj   rk   rl   rq   rr   Ú
avg_strideÚavg_pool_fnrS   s              €rH   rP   zDownsampleAvg.__init__n   s{   ø€ ô* 	‰ÑÔØ'¨1š}‘V°!ˆ
Ø�AŠ:˜ AšØ+5¸ª?¸xÈ!º|�-ÔQS×Q]ÑQ]ˆKÙ# A z¸TÐUZÔ[ˆD�IäŸ™›ˆDŒIÙ˜v w°¸!ÀFÐRWÔXˆ�	rG   rT   rU   c                 óB   — | j                  | j                  |«      «      S )ztForward pass.

        Args:
            x: Input tensor.

        Returns:
            Downsampled tensor.
        )rw   ru   rY   s     rH   rZ   zDownsampleAvg.forwardŒ   s   € ð �y‰y˜Ÿ™ 1›Ó&Ð&rG   )r=   r>   r?   r@   r   rA   r
   r   rP   r\   r]   rZ   r^   r_   s   @rH   rf   rf   k   s   ø„ ÙOð ØØ,0Ø#2ØØñYàðYð ðYð ð	Yð
 ðYð % S™MðYð !õYð<	'˜Ÿ™ð 	'¨%¯,©,÷ 	'rG   rf   c            %       óê   ‡ — e Zd ZdZddddddddddddddded	ddfd
edee   dededee   dedededee   dededededee	   dedee	   de	def$ˆ fd„Z
dej                  dej                  fd„Zˆ xZS ) ÚNormFreeBlockz-Normalization-Free pre-activation block.
    Nr   r-   ç      Ð?TFr+   ç        rg   rh   ri   rj   rk   r$   Úbetar0   r(   r3   r4   r5   r9   r)   r,   r<   rl   Údrop_path_ratec                 ót  •— ||dœ}t         ‰| �  «        |xs |}|xs |}t        |r||z  n||z  |
«      }|	sdn||	z  }|	r|	|
z  dk(  r|	|z  }|| _        || _        || _        ||k7  s
|dk7  s||k7  rt        ||f||||dœ|¤Ž| _        nd| _         |«       | _         |||dfi |¤Ž| _	         |d¬«      | _
         |||df|||d	œ|¤Ž| _        |r# |d¬«      | _         |||dfd||d	œ|¤Ž| _        nd| _        d| _        |r|� ||fi |¤Ž| _        nd| _         |«       | _         |||dfd
|rdndi|¤Ž| _        |s|� ||fi |¤Ž| _        nd| _        |dkD  rt%        |«      nt'        j(                  «       | _        |r.t'        j,                  t/        j0                  di |¤Ž«      | _        yd| _        y)a„  Initialize NormFreeBlock.

        Args:
            in_chs: Input channels.
            out_chs: Output channels.
            stride: Stride for convolution.
            dilation: Dilation rate.
            first_dilation: First dilation rate.
            alpha: Alpha scaling factor for residual.
            beta: Beta scaling factor for pre-activation.
            bottle_ratio: Bottleneck ratio.
            group_size: Group convolution size.
            ch_div: Channel divisor for rounding.
            reg: Use RegNet-style configuration.
            extra_conv: Add extra 3x3 convolution.
            skipinit: Use skipinit initialization.
            attn_layer: Attention layer type.
            attn_gain: Attention gain factor.
            act_layer: Activation layer type.
            conv_layer: Convolution layer type.
            drop_path_rate: Stochastic depth drop rate.
        ©rq   rr   r   r   )ri   rj   rk   rl   NTrW   é   )ri   rj   ÚgroupsÚ	gain_initr-   r~   )r~   )rO   rP   r   r$   r   r,   rf   Ú
downsampleÚact1Úconv1Úact2Úconv2Úact2bÚconv2bÚattnÚact3Úconv3Ú	attn_lastr   rs   rv   Ú	drop_pathÚ	Parameterr\   ÚtensorÚskipinit_gain)rR   rg   rh   ri   rj   rk   r$   r   r0   r(   r3   r4   r5   r9   r)   r,   r<   rl   r€   rq   rr   ÚddÚmid_chsr„   rS   s                           €rH   rP   zNormFreeBlock.__init__�   s  ø€ ðZ ¨Ñ/ˆÜ‰ÑÔØ'Ò3¨8ˆØÒ#˜Vˆä ¹# ¨,Ò!6À7È\ÑCYÐ[aÓbˆÙ$‘¨'°ZÑ*?ˆÙ˜* vÑ-°Ò2Ø  6Ñ)ˆGØˆŒ
ØˆŒ	Ø"ˆŒà�WÒ ¨!¢¨x¸>Ò/IÜ+ØØðð Ø!Ø-Ø%ñð ñˆD�Oð #ˆDŒOá“KˆŒ	Ù ¨°Ñ9°bÑ9ˆŒ
Ù dÔ+ˆŒ	Ù ¨°!Ðq¸FÈ^ÐdjÑqÐnpÑqˆŒ
ÙÙ"¨4Ô0ˆDŒJÙ$ W¨g°qÐkÀÈXÐ^dÑkÐhjÑkˆD�KàˆDŒJØˆDŒKÙ�:Ð)Ù" 7Ñ1¨bÑ1ˆD�IàˆDŒIÙ“KˆŒ	Ù ¨°!Ñ\ÁX¹rÐSUÐ\ÐY[Ñ\ˆŒ
Ù�zÐ-Ù'¨Ñ6°2Ñ6ˆD�Nà!ˆDŒNØ5CÀaÒ5Gœ .Ô1ÌRÏ[É[Ë]ˆŒÙEMœRŸ\™\¬%¯,©,Ñ*@¸RÑ*@ÓAˆÕÐSWˆÕrG   rT   rU   c                 ó¶  — | j                  |«      | j                  z  }|}| j                  �| j                  |«      }| j                  |«      }| j	                  | j                  |«      «      }| j                  � | j                  | j                  |«      «      }| j                  �| j                  | j                  |«      z  }| j                  | j                  |«      «      }| j                  �| j                  | j                  |«      z  }| j                  |«      }| j                  �|j                  | j                  «       || j                   z  |z   }|S )zoForward pass.

        Args:
            x: Input tensor.

        Returns:
            Output tensor.
        )r‡   r   r†   rˆ   rŠ   r‰   rŒ   r‹   r�   r,   r�   rŽ   r�   r‘   r”   rX   r$   )rR   rT   ÚoutÚshortcuts       rH   rZ   zNormFreeBlock.forwardû   s  € ð �i‰i˜‹l˜TŸY™YÑ&ˆð ˆØ�?‰?Ð&Ø—‘ sÓ+ˆHð �j‰j˜‹oˆØ�j‰j˜Ÿ™ 3›Ó(ˆØ�;‰;Ð"Ø—+‘+˜dŸj™j¨›oÓ.ˆCØ�9‰9Ð Ø—.‘. 4§9¡9¨S£>Ñ1ˆCØ�j‰j˜Ÿ™ 3›Ó(ˆØ�>‰>Ð%Ø—.‘. 4§>¡>°#Ó#6Ñ6ˆCØ�n‰n˜SÓ!ˆà×ÑÐ)Ø�H‰H�T×'Ñ'Ô(Ø�D—J‘JÑ Ñ)ˆØˆ
rG   )r=   r>   r?   r@   r   rA   r
   rC   rE   r   rP   r\   r]   rZ   r^   r_   s   @rH   r|   r|   ˜   sR  ø„ ñð &*ØØØ,0ØØØ"&Ø(,ØØØ$Ø"Ø-1Ø"Ø,0Ø#2Ø$&ØØñ+\Xàð\Xð ˜c‘]ð\Xð ð	\Xð
 ð\Xð % S™Mð\Xð ð\Xð ð\Xð  ð\Xð ! ™ð\Xð ð\Xð ð\Xð ð\Xð ð\Xð ! Ñ*ð\Xð  ð!\Xð"   Ñ)ð#\Xð$ !ð%\Xð& "õ'\Xð|˜Ÿ™ð ¨%¯,©,÷ rG   r|   Ú rg   rh   r&   rl   r<   Úpreact_featurec                 óŒ  — ||dœ}d}	t        |dd¬«      }
t        «       }|dv sJ ‚d|v r¾d|v r+d|vsJ ‚|d	z  |d
z  |dz  |f}d}d
}	t        |dz  dd¬«      }
n1d|v rd|z  d	z  |dz  |f}n|dz  |dz  |f}d}t        |dz  dd¬«      }
t        |«      dz
  }t        t	        ||«      «      D ]7  \  }\  }} || |fd|dœ|¤Ž|d|dz   › �<   ||k7  r |d¬«      |d|dz   › �<   |} Œ9 n%d|v r || |fdddœ|¤Ž|d<   n || |fdddœ|¤Ž|d<   d|v rt        j                  ddd¬«      |d<   d
}	t        j                  |«      |	|
fS )a¬  Create stem module for NFNet models.

    Args:
        in_chs: Input channels.
        out_chs: Output channels.
        stem_type: Type of stem ('', 'deep', 'deep_tiered', 'deep_quad', '3x3', '7x7', etc.).
        conv_layer: Convolution layer type.
        act_layer: Activation layer type.
        preact_feature: Use pre-activation feature.

    Returns:
        Tuple of (stem_module, stem_stride, stem_feature_info).
    r‚   rn   ú	stem.conv©Únum_chsÚ	reductionÚmodule)	rš   ÚdeepÚdeep_tieredÚ	deep_quadr%   Ú7x7Ú	deep_poolÚ3x3_poolÚ7x7_poolr¢   Úquadru   r2   é   )rn   r   r   rn   z
stem.conv3Útieredrƒ   )rn   r   r   z
stem.conv2r   )Úkernel_sizeri   rw   TrW   Úactr%   é   )ri   Úpadding)Údictr   ÚlenÚ	enumerateÚziprs   Ú	MaxPool2dÚ
Sequential)rg   rh   r&   rl   r<   r›   rq   rr   r•   Ústem_strideÚstem_featureÚstemr'   ÚstridesÚlast_idxÚiÚcÚss                     rH   Úcreate_stemr¾     sÒ  € ð.  UÑ	+€BØ€KÜ °1¸[ÔI€LÜ‹=€DØÐsÑsÐsÐsØ�ÑØ�YÑà Ñ*Ð*Ð*Ø 1™ g°¡l°G¸q±LÀ'ÐJˆHØ"ˆGØˆKÜ¨°1©ÀÈ,ÔW‰Là˜9Ñ$Ø ™K¨1Ñ,¨g¸©l¸GÐD‘à# q™L¨'°Q©,¸Ð@�ØˆGÜ¨°1©ÀÈ,ÔWˆLÜ�x“= 1Ñ$ˆÜ"¤3 x°Ó#9Ó:ò 	‰IˆA‰v��1Ù#-¨f°aÐ#WÀQÈqÑ#WÐTVÑ#WˆD�4˜˜A™�w�Ñ Ø�HŠ}Ù&/¸Ô&=��s˜1˜q™5˜'�]Ñ#Ø‰Fñ		ð
 
�)Ñ	á! &¨'ÐQ¸qÈÑQÈbÑQˆˆVŠñ " &¨'ÐQ¸qÈÑQÈbÑQˆˆV‰à�ÑÜ—|‘| A¨a¸Ô;ˆˆV‰Øˆä�=‰=˜Ó ¨\Ð9Ð9rG   g   `·Uô?g   ÀyXô?g    \9û?g   `aKû?g   À¦µþ?g   €Ú ð?g    `lû?g   `iû?g   à|ð?g     7@g   €-œü?g    øµ@g   `®¹þ?g    ©€ù?)ÚidentityÚceluÚeluÚgeluÚ
leaky_reluÚlog_sigmoidÚlog_softmaxr[   Úrelu6ÚseluÚsigmoidr;   ÚsoftsignÚsoftplusÚtanhc                   ó  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 ddedededededed	ed
efˆ fd„Z	e
j                  j                  ddedeeef   fd„«       Ze
j                  j                  ddeddfd„«       Ze
j                  j                  dej$                  fd„«       Zddedee   ddfd„Zde
j,                  de
j,                  fd„Zdde
j,                  dede
j,                  fd„Zde
j,                  de
j,                  fd„Zˆ xZS )r   a*   Normalization-Free Network

    As described in :
    `Characterizing signal propagation to close the performance gap in unnormalized ResNets`
        - https://arxiv.org/abs/2101.08692
    and
    `High-Performance Large-Scale Image Recognition Without Normalization` - https://arxiv.org/abs/2102.06171

    This model aims to cover both the NFRegNet-Bx models as detailed in the paper's code snippets and
    the (preact) ResNet models described earlier in the paper.

    There are a few differences:
        * channels are rounded to be divisible by 8 by default (keep tensor core kernels happy),
            this changes channel dim and param counts slightly from the paper models
        * activation correcting gamma constants are moved into the ScaledStdConv as it has less performance
            impact in PyTorch when done with the weight scaling there. This likely wasn't a concern in the JAX impl.
        * a config option `gamma_in_act` can be enabled to not apply gamma in StdConv as described above, but
            apply it in each activation. This is slightly slower, numerically different, but matches official impl.
        * skipinit is disabled by default, it seems to have a rather drastic impact on GPU memory use and throughput
            for what it is/does. Approx 8-10% throughput loss.
    NÚcfgÚnum_classesÚin_chansÚglobal_poolÚoutput_strideÚ	drop_rater€   Úkwargsc
                 ó´  •— t         ‰"| �  «        ||	dœ}|| _        || _        || _        d| _        t        |fi |
¤Ž}|j                  t        v sJ d|j                  › d�«       ‚|j                  rt        nt        }|j                  r@t        |j                  t        |j                     ¬«      }t        ||j                  ¬«      }n>t!        |j                  «      }t        |t        |j                     |j                  ¬«      }|j"                  r)t        t%        |j"                  «      fi |j&                  ¤Žnd}t)        |j*                  xs |j,                  d	   |j.                  z  |j0                  «      }t3        |||j4                  f||d
œ|¤Ž\  | _        }}|g| _        t;        ||j<                  d¬«      }|}|}d}d}g }t?        |j<                  «      D �]Ÿ  \  }}|d	k(  r|dkD  rdnd}||k\  r|dkD  r||z  }d}||z  }|dv rdnd}g }tA        |j<                  |   «      D �]  }|d	k(  xr |d	k(  }t)        |j,                  |   |j.                  z  |j0                  «      }|tC        d0i d|“d|“d|jD                  “dd|dz  z  “d|d	k(  r|nd“d|“d|“d|jF                  “d|jH                  r|rdn|jJ                  “d|j0                  “d|jH                  “d|jL                  “d|jN                  “d|“d |jP                  “d!|“d"|“d#||   |   “|¤Žgz  }|d	k(  rd}||jD                  dz  z  }|}|}�Œ | xj8                  tS        ||d$|› �¬%«      gz  c_        |tU        jV                  |Ž gz  }�Œ¢ tU        jV                  |Ž | _,        |jZ                  rrt)        |j.                  |jZ                  z  |j0                  «      | _-         ||| jZ                  dfi |¤Ž| _.        tS        | jZ                  |d&¬%«      | j8                  d'<   n || _-        tU        j^                  «       | _.         ||jZ                  d	kD  ¬(«      | _0        | jZ                  | _1        te        | jZ                  |f|| j                  d)œ|¤Ž| _3        | ji                  «       D �]:  \  } }!d*| v r²tk        |!tT        jl                  «      r˜|jn                  r*tT        jp                  js                  |!jt                  «       n+tT        jp                  jw                  |!jt                  d+d,«       |!jx                  €Œ“tT        jp                  js                  |!jx                  «       Œ½tk        |!tT        jz                  «      sŒØtT        jp                  j}                  |!jt                  d-d.¬/«       |!jx                  €�ŒtT        jp                  js                  |!jx                  «       �Œ= y)1aÁ  
        Args:
            cfg: Model architecture configuration.
            num_classes: Number of classifier classes.
            in_chans: Number of input channels.
            global_pool: Global pooling type.
            output_stride: Output stride of network, one of (8, 16, 32).
            drop_rate: Dropout rate.
            drop_path_rate: Stochastic depth drop-path rate.
            **kwargs: Extra kwargs overlayed onto cfg.
        r‚   Fz3Please add non-linearity constants for activation (z).)rL   )Úeps)rL   rÕ   Nr   )rl   r<   T)Ú	stagewiser   r-   rn   )r   rn   rg   rh   r$   r   r/   ri   rj   rk   r(   r0   r3   r4   r5   r9   r)   r,   r<   rl   r€   zstages.rž   Ú
final_convéÿÿÿÿrW   )Ú	pool_typerÒ   Úfcr~   g{®Gáz„?Úfan_inÚlinear)ÚmodeÚnonlinearityrF   )?rO   rP   rÎ   rÏ   rÒ   Úgrad_checkpointingr   r<   Ú_nonlin_gammar7   r   r   r6   rd   r   r8   r   r)   r   r*   r   r'   r#   r.   r3   r¾   r&   r¸   Úfeature_infor   r"   r²   Úranger|   r$   r(   r4   r0   r5   r9   r,   r°   rs   rµ   Ústagesr1   r×   rv   Ú	final_actÚhead_hidden_sizer   ÚheadÚnamed_modulesÚ
isinstanceÚLinearr:   ÚinitÚzeros_ÚweightÚnormal_ÚbiasÚConv2dÚkaiming_normal_)#rR   rÍ   rÎ   rÏ   rÐ   rÑ   rÒ   r€   rq   rr   rÓ   r•   rl   r<   r)   r'   r¶   Ú	stem_featÚdrop_path_ratesÚprev_chsÚ
net_striderj   Úexpected_varrã   Ú	stage_idxÚstage_depthri   rk   ÚblocksÚ	block_idxÚfirst_blockrh   ÚnÚmrS   s#                                     €rH   rP   zNormFreeNet.__init__†  s˜  ø€ ô0 	‰ÑÔØ¨Ñ/ˆØ&ˆÔØ ˆŒØ"ˆŒØ"'ˆÔä�cÑ$˜VÑ$ˆØ�}‰}¤Ñ-ÐvÐ1dÐeh×erÑerÐdsÐsuÐ/vÓvÐ-Ø,/×,<Ò,<Õ(Ä/ˆ
Ø×ÒÜ& s§}¡}¼MÈ#Ï-É-Ñ<XÔYˆIÜ  °×1AÑ1AÔB‰Jä% c§m¡mÓ4ˆIÜ  ´=ÀÇÁÑ3OÐUX×UeÑUeÔfˆJØMPÏ^Ê^”WœX c§n¡nÓ5ÑI¸¿¹ÒIÐaeˆ
ä! 3§<¡<Ò#B°3·<±<À±?Àc×FVÑFVÑ"VÐX[×XbÑXbÓcˆÜ,7ØØØ�M‰Mð-
ð "Øñ-
ð ñ-
Ñ)ˆŒ	�; 	ð '˜KˆÔÜ3°NÀCÇJÁJÐZ^Ô_ˆØˆØ ˆ
ØˆØˆØˆÜ&/°·
±
Ó&;ó &	/Ñ"ˆI�{Ø# qš.¨[¸1ª_‘QÀ!ˆFØ˜]Ò*¨v¸ªzØ˜FÑ"�Ø�Ø˜&Ñ ˆJØ"*¨fÑ"4™Q¸!ˆNàˆFÜ" 3§:¡:¨iÑ#8Ó9ó #�	Ø'¨1™nÒ?°¸a±�Ü(¨¯©°iÑ)@À3×CSÑCSÑ)SÐUX×U_ÑU_Ó`�Øœ=ò Ù#ðÙ-4ðàŸ)š)ðð ˜l¨cÑ1Ò1ðð &/°!¢^™6¸øð	ñ
 &ðñ $2ðð  #Ÿ~š~ðð (+§w¢w±;¡ÀC×DTÑDTøðð Ÿ:š:ðð Ÿšðð  #Ÿ~š~ðð !Ÿ\š\ðñ  *ðð "Ÿmšmðñ (ðñ   *ð!ð" $3°9Ñ#=¸iÒ#HØñ%ð ñ �ð(  ’>Ø#%�LØ §	¡	¨Q¡Ñ.�Ø!)�Ø"’ð7#ð8 ×Ò¤$¨xÀ:ÐX_Ð`iÐ_jÐVkÔ"lÐ!mÑmÕØ”r—}‘} fÐ-Ð.Ñ.ŠFðM&	/ôN —m‘m VÐ,ˆŒà×Òä .¨s×/?Ñ/?À#×BRÑBRÑ/RÐTW×T^ÑT^Ó _ˆDÔÙ(¨°4×3DÑ3DÀaÑNÈ2ÑNˆDŒOÜ$(°×1BÑ1BÈjÐcmÔ$oˆD×Ñ˜bÒ!à (ˆDÔÜ Ÿk™k›mˆDŒOÙ"¨3×+;Ñ+;¸aÑ+?Ô@ˆŒà $× 1Ñ 1ˆÔÜ"Ø×ÑØð
ð "Ø—n‘nñ	
ð
 ñ
ˆŒ	ð ×&Ñ&Ó(ó 	+‰DˆAˆqØ�q‰yœZ¨¬2¯9©9Ô5Ø×#Ò#Ü—G‘G—N‘N 1§8¡8Õ,ä—G‘G—O‘O A§H¡H¨b°#Ô6Ø—6‘6Ñ%Ü—G‘G—N‘N 1§6¡6Õ*Ü˜AœrŸy™yÕ)Ü—‘×'Ñ'¨¯©°xÈhÐ'ÔWØ—6‘6Ò%Ü—G‘G—N‘N 1§6¡6Ö*ñ	+rG   ÚcoarserU   c                 ó0   — t        d|rdnddfdg¬«      }|S )z"Group parameters for optimization.z^stemz^stages\.(\d+)z^stages\.(\d+)\.(\d+)N)z^final_conv)iŸ† )r¸   rø   )r°   )rR   rý   Úmatchers      rH   Úgroup_matcherzNormFreeNet.group_matcher
  s.   € ô Øá&,Ñ"Ð2JÈDÐQØ*ðô
ˆð ˆrG   Úenablec                 ó   — || _         y)z)Enable or disable gradient checkpointing.N)rß   )rR   r  s     rH   Úset_grad_checkpointingz"NormFreeNet.set_grad_checkpointing  s   € ð #)ˆÕrG   c                 ó.   — | j                   j                  S )zGet the classifier head.)ræ   rÚ   )rR   s    rH   Úget_classifierzNormFreeNet.get_classifier  s   € ð �y‰y�|‰|ÐrG   c                 óJ   — || _         | j                  j                  ||«       y)zŸReset the classifier head.

        Args:
            num_classes: Number of classes for new classifier.
            global_pool: Global pooling type.
        N)rÎ   ræ   Úreset)rR   rÎ   rÐ   s      rH   Úreset_classifierzNormFreeNet.reset_classifier   s   € ð 'ˆÔØ�	‰	�‰˜ [Õ1rG   rT   c                 ó  — | j                  |«      }| j                  r5t        j                  j	                  «       st        | j                  |«      }n| j                  |«      }| j                  |«      }| j                  |«      }|S )z’Forward pass through feature extraction layers.

        Args:
            x: Input tensor.

        Returns:
            Feature tensor.
        )	r¸   rß   r\   ÚjitÚis_scriptingr   rã   r×   rä   rY   s     rH   Úforward_featureszNormFreeNet.forward_features*  sg   € ð �I‰I�a‹LˆØ×"Ò"¬5¯9©9×+AÑ+AÔ+CÜ˜tŸ{™{¨AÓ.‰Aà—‘˜A“ˆAØ�O‰O˜AÓˆØ�N‰N˜1ÓˆØˆrG   Ú
pre_logitsc                 óN   — |r| j                  ||¬«      S | j                  |«      S )zàForward pass through classifier head.

        Args:
            x: Input features.
            pre_logits: Return features before final linear layer.

        Returns:
            Classification logits or features.
        )r  )ræ   )rR   rT   r  s      rH   Úforward_headzNormFreeNet.forward_head<  s(   € ñ 7Aˆt�y‰y˜ zˆyÓ2ÐRÀdÇiÁiÐPQÃlÐRrG   c                 óJ   — | j                  |«      }| j                  |«      }|S )zoForward pass.

        Args:
            x: Input tensor.

        Returns:
            Output logits.
        )r  r  rY   s     rH   rZ   zNormFreeNet.forwardH  s)   € ð ×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆrG   )éè  rƒ   Úavgé    r~   r~   NNrc   )T)N)r=   r>   r?   r@   r    rA   rD   rC   r   rP   r\   r
  ÚignorerE   r	   r   r  rs   ÚModuler  r
   r  r]   r  r  rZ   r^   r_   s   @rH   r   r   p  sƒ  ø„ ñð0  $ØØ$Ø!#Ø!Ø$&ØØñB+àðB+ð ðB+ð ð	B+ð
 ðB+ð ðB+ð ðB+ð "ðB+ð õB+ðH ‡Y�Y×Ññ	 Dð 	°T¸#¸s¸(±^ò 	ó ð	ð ‡Y�Y×Ññ)¨Tð )¸Tò )ó ð)ð ‡Y�Y×Ñð §	¡	ò ó ðñ2¨Cð 2¸hÀs¹mð 2ÐW[ó 2ð %§,¡,ð °5·<±<ó ñ$
S˜eŸl™lð 
S¸ð 
SÈÏÉó 
Sð˜Ÿ™ð ¨%¯,©,÷ rG   r"   .r#   r(   r)   r*   c                 ó:   — |xs i }t        | |ddd||||¬«	      }|S )ar  Create NFNet ResNet configuration.

    Args:
        depths: Number of blocks in each stage.
        channels: Channel dimensions for each stage.
        group_size: Group convolution size.
        act_layer: Activation layer type.
        attn_layer: Attention layer type.
        attn_kwargs: Attention layer arguments.

    Returns:
        NFNet configuration.
    r¨   é@   r}   )	r"   r#   r&   r'   r0   r(   r<   r)   r*   )r    )r"   r#   r(   r<   r)   r*   rÍ   s          rH   Ú
_nfres_cfgr  V  s:   € ð* Ò# €KÜ
ØØØØØØØØØô
€Cð €JrG   c                 ó^   — d|d   z  dz  }t        d¬«      }t        | |dddd	|d
d|¬«
      }|S )zÂCreate NFNet RegNet configuration.

    Args:
        depths: Number of blocks in each stage.
        channels: Channel dimensions for each stage.

    Returns:
        NFNet configuration.
    i   rØ   é¸  r/   ©Úrd_ratior%   r2   g      è?g      @TÚse)
r"   r#   r&   r(   r.   r0   r1   r4   r)   r*   )r°   r    )r"   r#   r1   r*   rÍ   s        rH   Ú
_nfreg_cfgr  z  sP   € ð ˜( 2™,Ñ&¨#Ñ-€LÜ Ô$€KÜ
ØØØØØØØ!ØØØô€Cð €JrG   r  r0   Ú	feat_multc                 ót   — t        |d   |z  «      }|�|nt        d¬«      }t        | |dd||d||||¬«      }	|	S )aÊ  Create NFNet configuration.

    Args:
        depths: Number of blocks in each stage.
        channels: Channel dimensions for each stage.
        group_size: Group convolution size.
        bottle_ratio: Bottleneck ratio.
        feat_mult: Feature multiplier for final layer.
        act_layer: Activation layer type.
        attn_layer: Attention layer type.
        attn_kwargs: Attention layer arguments.

    Returns:
        NFNet configuration.
    rØ   r/   r  r¤   é€   T)r"   r#   r&   r'   r(   r0   r5   r1   r<   r)   r*   )rA   r°   r    )
r"   r#   r(   r0   r  r<   r)   r*   r1   rÍ   s
             rH   Ú
_nfnet_cfgr"  •  sZ   € ô2 �x ‘| iÑ/Ó0€LØ!,Ð!8‘+¼dÈCÔ>P€KÜ
ØØØØØØ!ØØ!ØØØô€Cð €JrG   r9   c                 ój   — t        | |ddddddd|t        |d   dz  «      |dt        d¬«      ¬	«      }|S )
a  Create DeepMind NFNet configuration.

    Args:
        depths: Number of blocks in each stage.
        channels: Channel dimensions for each stage.
        act_layer: Activation layer type.
        skipinit: Use skipinit initialization.

    Returns:
        NFNet configuration.
    r¤   r!  r/   TrØ   r+   r  r  )r"   r#   r&   r'   r(   r0   r5   r6   r7   r9   r1   r<   r)   r*   )r    rA   r°   )r"   r#   r<   r9   rÍ   s        rH   Ú_dm_nfnet_cfgr$  À  sR   € ô" ØØØØØØØØØØÜ˜ "™¨Ñ+Ó,ØØÜ #Ô&ô€Cð  €JrG   Údm_nfnet_f0)r   rn   é   rƒ   )r"   Údm_nfnet_f1)rn   rª   é   r&  Údm_nfnet_f2)rƒ   r&  é   é	   Údm_nfnet_f3)rª   r2   é   r(  Údm_nfnet_f4)é   é
   é   é   Údm_nfnet_f5)r&  r(  é$   r*  Údm_nfnet_f6)r®   é   é*   é   Únfnet_f0Únfnet_f1Únfnet_f2Únfnet_f3Únfnet_f4Únfnet_f5Únfnet_f6Únfnet_f7)r2   é   é0   r-  Únfnet_l0g      ø?r  r}   r2   )r  Ú
rd_divisorr;   )r"   r  r(   r0   r*   r<   Úeca_nfnet_l0Úeca)r"   r  r(   r0   r)   r*   r<   Úeca_nfnet_l1rn   Úeca_nfnet_l2Úeca_nfnet_l3Únf_regnet_b0)r   rƒ   r&  r&  Únf_regnet_b1)rn   rª   r®   r®   Únf_regnet_b2)rn   rª   r2   r2   )é8   ép   éè   iè  )r"   r#   Únf_regnet_b3)rn   r/  r+  r+  )rM  r!  éø   i  Únf_regnet_b4)rn   r&  é   rS  )r  é�   é   ih  Únf_regnet_b5)rƒ   r®   r6  r6  )éP   é¨   iP  iÀ  Únf_resnet26)rn   rn   rn   rn   Únf_resnet50)rƒ   rª   r&  rƒ   Únf_resnet101)rƒ   rª   é   rƒ   Únf_seresnet26g      °?r  )r"   r)   r*   Únf_seresnet50Únf_seresnet101Únf_ecaresnet26Únf_ecaresnet50Únf_ecaresnet101Ú
test_nfnet)r   r   r   r   )r  r  é`   r!  )r"   r#   r  r(   r0   r*   r<   ÚvariantÚ
pretrainedrÓ   c                 óV   — t         |    }t        d¬«      }t        t        | |f||dœ|¤ŽS )zÝCreate a NormFreeNet model.

    Args:
        variant: Model variant name.
        pretrained: Load pretrained weights.
        **kwargs: Additional model arguments.

    Returns:
        NormFreeNet model instance.
    T)Úflatten_sequential)Ú	model_cfgÚfeature_cfg)Ú
model_cfgsr°   r   r   )re  rf  rÓ   ri  rj  s        rH   Ú_create_normfreenetrl  %  sE   € ô ˜7Ñ#€IÜ¨$Ô/€KÜÜØØðð Øñð ñð rG   Úurlc                 ó2   — | dddddt         t        dddd	œ|¥S )
z»Create default configuration dictionary.

    Args:
        url: Model weight URL.
        **kwargs: Additional configuration options.

    Returns:
        Configuration dictionary.
    r  ©rƒ   éà   rp  ©r®   r®   çÍÌÌÌÌÌì?Úbicubicz
stem.conv1zhead.fcz
apache-2.0)rm  rÎ   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚmeanÚstdÚ
first_convÚ
classifierÚlicenser   )rm  rÓ   s     rH   Ú_dcfgr}  <  s4   € ð Ø¨=ÀvØ¨)Ü%Ô.BØ"°)Èñð ðð rG   zdm_nfnet_f0.dm_in1kztimm/zmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f0-604f9c3a.pth)r&  r&  )rƒ   éÀ   r~  )rƒ   é   r  rr  Úsquash)Ú	hf_hub_idrm  ru  rt  Útest_input_sizerv  Ú	crop_modezdm_nfnet_f1.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f1-fc540f82.pthrq  ro  )rƒ   é@  r„  g…ëQ¸í?zdm_nfnet_f2.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f2-89875923.pth)r2   r2   )rƒ   é`  r…  gq=
×£pí?zdm_nfnet_f3.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f3-d74ab3aa.pth)r0  r0  )rƒ   é   r†  g®Gázî?zdm_nfnet_f4.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f4-0ac5b10b.pth)r(  r(  )rƒ   é€  r‡  )rƒ   é   rˆ  g;ßO�—nî?zdm_nfnet_f5.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f5-ecb20ab1.pth)é   r‰  )rƒ   é   rŠ  gºI+‡î?zdm_nfnet_f6.dm_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-dnf-weights/dm_nfnet_f6-e0f12116.pth)r6  r6  )rƒ   éÀ  r‹  )rƒ   é@  rŒ  gd;ßO�—î?)rm  ru  rt  r‚  )r2  r2  )rƒ   éà  r�  )rƒ   é`  rŽ  znfnet_l0.ra2_in1kzjhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/nfnet_l0_ra2-45c6688d.pth)rƒ   rU  rU  )r�  rm  ru  rt  r‚  Útest_crop_pctzeca_nfnet_l0.ra2_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/ecanfnet_l0_ra2-e3e9ac50.pthzeca_nfnet_l1.ra2_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/ecanfnet_l1_ra2-7dce93cd.pthzeca_nfnet_l2.ra3_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/ecanfnet_l2_ra3-da781a61.pth)rS  rS  )rm  ru  rt  r‚  r�  r�   )rm  ru  rt  r‚  rz  znf_regnet_b1.ra2_in1kzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/nf_regnet_b1_256_ra2-ad85cfef.pth)r�  rm  ru  rt  r‚  rz  )rƒ   éð   r�  )rƒ   é  r‘  )r+  r+  )rƒ   éÈ  r’  )rm  rz  znf_resnet50.ra2_in1kzmhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/nf_resnet50_ra2-9f236009.pth)r�  rm  ru  rt  r‚  rv  rz  )r/   r/   r/   gffffffî?)rƒ   é    r“  )r/  r/  )r�  rx  ry  rv  rt  ru  )rb  ztest_nfnet.r160_in1kc                 ó   — t        dd| i|¤ŽS )z&NFNet-F0 (DeepMind weight compatible).rf  )r%  ©rl  ©rf  rÓ   s     rH   r%  r%  ¸  ó   € ô ÑN¸ÐNÀvÑNÐNrG   c                 ó   — t        dd| i|¤ŽS )z&NFNet-F1 (DeepMind weight compatible).rf  )r'  r•  r–  s     rH   r'  r'  ¾  r—  rG   c                 ó   — t        dd| i|¤ŽS )z&NFNet-F2 (DeepMind weight compatible).rf  )r)  r•  r–  s     rH   r)  r)  Ä  r—  rG   c                 ó   — t        dd| i|¤ŽS )z&NFNet-F3 (DeepMind weight compatible).rf  )r,  r•  r–  s     rH   r,  r,  Ê  r—  rG   c                 ó   — t        dd| i|¤ŽS )z&NFNet-F4 (DeepMind weight compatible).rf  )r.  r•  r–  s     rH   r.  r.  Ð  r—  rG   c                 ó   — t        dd| i|¤ŽS )z&NFNet-F5 (DeepMind weight compatible).rf  )r3  r•  r–  s     rH   r3  r3  Ö  r—  rG   c                 ó   — t        dd| i|¤ŽS )z&NFNet-F6 (DeepMind weight compatible).rf  )r5  r•  r–  s     rH   r5  r5  Ü  r—  rG   c                 ó   — t        dd| i|¤ŽS )z	NFNet-F0.rf  )r9  r•  r–  s     rH   r9  r9  â  ó   € ô ÑK°jÐKÀFÑKÐKrG   c                 ó   — t        dd| i|¤ŽS )z	NFNet-F1.rf  )r:  r•  r–  s     rH   r:  r:  è  rŸ  rG   c                 ó   — t        dd| i|¤ŽS )z	NFNet-F2.rf  )r;  r•  r–  s     rH   r;  r;  î  rŸ  rG   c                 ó   — t        dd| i|¤ŽS )z	NFNet-F3.rf  )r<  r•  r–  s     rH   r<  r<  ô  rŸ  rG   c                 ó   — t        dd| i|¤ŽS )z	NFNet-F4.rf  )r=  r•  r–  s     rH   r=  r=  ú  rŸ  rG   c                 ó   — t        dd| i|¤ŽS )z	NFNet-F5.rf  )r>  r•  r–  s     rH   r>  r>     rŸ  rG   c                 ó   — t        dd| i|¤ŽS )z	NFNet-F6.rf  )r?  r•  r–  s     rH   r?  r?    rŸ  rG   c                 ó   — t        dd| i|¤ŽS )z	NFNet-F7.rf  )r@  r•  r–  s     rH   r@  r@    rŸ  rG   c                 ó   — t        dd| i|¤ŽS )zˆNFNet-L0b w/ SiLU.

    My experimental 'light' model w/ F0 repeats, 1.5x final_conv mult, 64 group_size, .25 bottleneck & SE ratio
    rf  )rC  r•  r–  s     rH   rC  rC    s   € ô ÑK°jÐKÀFÑKÐKrG   c                 ó   — t        dd| i|¤ŽS )z‹ECA-NFNet-L0 w/ SiLU.

    My experimental 'light' model w/ F0 repeats, 1.5x final_conv mult, 64 group_size, .25 bottleneck & ECA attn
    rf  )rE  r•  r–  s     rH   rE  rE    ó   € ô ÑO¸*ÐOÈÑOÐOrG   c                 ó   — t        dd| i|¤ŽS )z‹ECA-NFNet-L1 w/ SiLU.

    My experimental 'light' model w/ F1 repeats, 2.0x final_conv mult, 64 group_size, .25 bottleneck & ECA attn
    rf  )rG  r•  r–  s     rH   rG  rG  $  r©  rG   c                 ó   — t        dd| i|¤ŽS )z‹ECA-NFNet-L2 w/ SiLU.

    My experimental 'light' model w/ F2 repeats, 2.0x final_conv mult, 64 group_size, .25 bottleneck & ECA attn
    rf  )rH  r•  r–  s     rH   rH  rH  -  r©  rG   c                 ó   — t        dd| i|¤ŽS )z‹ECA-NFNet-L3 w/ SiLU.

    My experimental 'light' model w/ F3 repeats, 2.0x final_conv mult, 64 group_size, .25 bottleneck & ECA attn
    rf  )rI  r•  r–  s     rH   rI  rI  6  r©  rG   c                 ó   — t        dd| i|¤ŽS )z"Normalization-Free RegNet-B0.
    rf  )rJ  r•  r–  s     rH   rJ  rJ  ?  ó   € ô ÑO¸*ÐOÈÑOÐOrG   c                 ó   — t        dd| i|¤ŽS )z"Normalization-Free RegNet-B1.
    rf  )rK  r•  r–  s     rH   rK  rK  F  r®  rG   c                 ó   — t        dd| i|¤ŽS )z"Normalization-Free RegNet-B2.
    rf  )rL  r•  r–  s     rH   rL  rL  M  r®  rG   c                 ó   — t        dd| i|¤ŽS )z"Normalization-Free RegNet-B3.
    rf  )rP  r•  r–  s     rH   rP  rP  T  r®  rG   c                 ó   — t        dd| i|¤ŽS )z"Normalization-Free RegNet-B4.
    rf  )rR  r•  r–  s     rH   rR  rR  [  r®  rG   c                 ó   — t        dd| i|¤ŽS )z"Normalization-Free RegNet-B5.
    rf  )rV  r•  r–  s     rH   rV  rV  b  r®  rG   c                 ó   — t        dd| i|¤ŽS )z"Normalization-Free ResNet-26.
    rf  )rY  r•  r–  s     rH   rY  rY  i  ó   € ô ÑN¸ÐNÀvÑNÐNrG   c                 ó   — t        dd| i|¤ŽS )z"Normalization-Free ResNet-50.
    rf  )rZ  r•  r–  s     rH   rZ  rZ  p  rµ  rG   c                 ó   — t        dd| i|¤ŽS )z#Normalization-Free ResNet-101.
    rf  )r[  r•  r–  s     rH   r[  r[  w  r®  rG   c                 ó   — t        dd| i|¤ŽS )zNormalization-Free SE-ResNet26.rf  )r]  r•  r–  s     rH   r]  r]  ~  ó   € ô ÑP¸:ÐPÈÑPÐPrG   c                 ó   — t        dd| i|¤ŽS )zNormalization-Free SE-ResNet50.rf  )r^  r•  r–  s     rH   r^  r^  „  r¹  rG   c                 ó   — t        dd| i|¤ŽS )z Normalization-Free SE-ResNet101.rf  )r_  r•  r–  s     rH   r_  r_  Š  ó   € ô ÑQ¸JÐQÈ&ÑQÐQrG   c                 ó   — t        dd| i|¤ŽS )z Normalization-Free ECA-ResNet26.rf  )r`  r•  r–  s     rH   r`  r`  �  r¼  rG   c                 ó   — t        dd| i|¤ŽS )z Normalization-Free ECA-ResNet50.rf  )ra  r•  r–  s     rH   ra  ra  –  r¼  rG   c                 ó   — t        dd| i|¤ŽS )z!Normalization-Free ECA-ResNet101.rf  )rb  r•  r–  s     rH   rb  rb  œ  s   € ô ÑR¸ZÐRÈ6ÑRÐRrG   c                 ó   — t        dd| i|¤ŽS )z%Test NFNet model for experimentation.rf  )rc  r•  r–  s     rH   rc  rc  ¢  s   € ô ÑM¸
ÐMÀfÑMÐMrG   )r-   )rš   NNTNN))r  rˆ  i   i   Nr[   NN))rB  éh   éÐ   r  )©r  rˆ  é   rÄ  r!  r/   r+   rÂ   r  N)rÃ  rÂ   TrF   rc   )rš   )dr@   Úcollectionsr   Údataclassesr   r   Ú	functoolsr   Útypingr   r   r	   r
   r   r\   Útorch.nnrs   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   Ú_builderr   Ú_features_fxr   Ú_manipulater   Ú	_registryr   r   Ú__all__r    r  rJ   rD   rC   rd   rf   r|   rA   rE   rµ   r¾   r°   rà   r   r  r  r"  r$  rk  rl  r}  Údefault_cfgsr%  r'  r)  r,  r.  r3  r5  r9  r:  r;  r<  r=  r>  r?  r@  rC  rE  rG  rH  rI  rJ  rK  rL  rP  rR  rV  rY  rZ  r[  r]  r^  r_  r`  ra  rb  rc  rF   rG   rH   ú<module>rÒ     sT  ðñõ$ $ß *Ý ß 7Õ 7ã Ý ç A÷8÷ 8÷ 8å *Ý 1Ý 'ß <à˜'Ð
"€ð ÷ð ó ðô2Eˆr�y‰yô Eò8˜Sð ¨ð ¸ó ô*'�B—I‘Iô *'ðZ ôA�B—I‘Ió Aó ðAðN Ø)-Ø(,Ø#ØØò<:Øð<:àð<:ð ð<:ð ˜XÑ&ð	<:ð
 ˜HÑ%ð<:ð ð<:ð ˆ2�=‰=˜#˜t C¨ H™~Ð-Ñ.ó<:ñ@ ØØ	ØØ	ØØ"Ø"Ø	Ø
Ø	ØØ	ØØØ	ô€ô&c�"—)‘)ô cðP %;Ø$(ØØ$(Ø04ò!Ø�c˜3�h‘ð!à˜˜S˜‘/ð!ð ˜S‘Mð!ð ð	!ð
 ˜S‘Mð!ð ˜d 3¨ 8™nÑ-ð!ð ó!òH�u˜S #˜X‘ð °%¸¸S¸±/ð Ð\aó ð: %;ØØ!ØØØØ04ò(Ø�c˜3�h‘ð(à˜˜S˜‘/ð(ð ð(ð ð	(ð
 ð(ð ð(ð ð(ð ˜d 3¨ 8™nÑ-ð(ð ó(ðZ %;ØØò	!Ø�c˜3�h‘ð!à˜˜S˜‘/ð!ð ð!ð ð	!ð
 ó!ñH ó >á \Õ2ð>ñ  ]Õ3ð>ñ  ]Õ3ð	>ñ
  ^Õ4ð>ñ  _Õ5ð>ñ  _Õ5ð>ñ  _Õ5ð>ñ ˜|Õ,ð>ñ ˜}Õ-ð>ñ ˜}Õ-ð>ñ ˜~Õ.ð>ñ ˜Õ/ð>ñ  ˜Õ/ð!>ñ" ˜Õ/ð#>ñ$ ˜Õ/ð%>ñ* Ø s°rÈÙ $°1Ô5ÀõIð+>ñ0 Ø s°rÈØ¡d£f¸õ@ð1>ñ6 Ø¨°bÀtØ¡d£f¸õ@ð7>ñ< Ø¨°bÀtØ¡d£f¸õ@ð=>ñB Ø¨°rÈØ¡d£f¸õ@ðC>ñN  <Õ0ðO>ñP  <Õ0ðQ>ñR  <Ð:MÕNðS>ñT  <Ð:MÕNðU>ñV  >Ð<OÕPðW>ñX  >Ð<OÕPðY>ñ^  ,Õ/ð_>ñ`  ,Õ/ða>ñb  =Õ1ðc>ñf  L¸TÉtÐ]aÔObÕcðg>ñh  L¸TÉtÐ]aÔObÕcði>ñj  ]¸tÑQUÐ_cÔQdÕeðk>ñn  \¸eÑQUÓQWÕXðo>ñp  \¸eÑQUÓQWÕXðq>ñr  mÀÑSWÓSYÕZðs>ñv ØÐ&7À3ÐSTÐcgÙ $°1Ô5ÀõIðw>€
òB ð °$ð È#ð ÐR]ó ò.ˆsð  3ð ¨4°°S°©>ó ñ( %ð e&Ø™5ØØ{Ø ]ÀMÐ\^Ðjrôtðe&ð
 ™5ØØ{Ø ]ÀMÐ\`Ðltôvðe&ð ™5ØØ{Ø ]ÀMÐ\`Ðltôvðe&ð ™5ØØ{Ø }ÀmÐ^bÐnvôxðe&ð" ™5ØØ{Ø }ÀmÐ^cÐowôyð#e&ð* ™5ØØ{Ø }ÀmÐ^cÐowôyð+e&ð2 ™5ØØ{Ø }ÀmÐ^cÐowôyð3e&ð< ‘Ø˜&¨]ÈMô[ð=e&ð@ ‘Ø˜&¨]ÈMô[ðAe&ðD ‘Ø˜&¨]ÈMô[ðEe&ðH ‘Ø˜(¨}Èmô]ðIe&ðL ‘Ø˜(¨}Èmô]ðMe&ðP ‘Ø˜(¨}Èmô]ðQe&ðT ‘Ø˜(¨}Èmô]ðUe&ðX ‘Ø˜(¨}Èmô]ðYe&ð^ ™ØØxØ ]ÀMÐadôfð_e&ðf ™UØØ{Ø ]ÀMÐadôfñge&ðn ™UØØ{Ø ]ÀMÐadôfðoe&ðv ™UØØ{Ø }ÀmÐcfôhðwe&ð~ ‘EØØ }ÀmÐcfôhðe&ðF ‘EØ˜&¨]ÈMÐfqôsðGe&ðJ ™UØð AØ ]ÀMÐ^iôkðKe&ðR ‘EØ˜&¨]ÈMÐfqôsðSe&ðV ‘EØ˜&¨]ÈMÐfqôsðWe&ðZ ‘EØ˜(¨}ÈmÐhsôuð[e&ð^ ‘EØ˜(¨}ÈmÐhsôuð_e&ðd ‘5˜R¨KÔ8ðee&ðf ™EØØ{Ø ]ÀMÐ\`Ðmxôzðge&ðn ‘E˜b¨[Ô9ðoe&ðr ‘U˜r¨kÔ:ðse&ðt ‘U˜r¨kÔ:ðue&ðv ‘e ¨{Ô;ðwe&ðz ‘e ¨{Ô;ð{e&ð| ‘e ¨{Ô;ñ}e&ñ~  °Ô<á!ØØ /Ø -¸6ôCòCe&ó e€ðP òO˜Dð O°Cð O¸Kò Oó ðOð
 òO˜Dð O°Cð O¸Kò Oó ðOð
 òO˜Dð O°Cð O¸Kò Oó ðOð
 òO˜Dð O°Cð O¸Kò Oó ðOð
 òO˜Dð O°Cð O¸Kò Oó ðOð
 òO˜Dð O°Cð O¸Kò Oó ðOð
 òO˜Dð O°Cð O¸Kò Oó ðOð
 òL˜ð L°ð L¸ò Ló ðLð
 òL˜ð L°ð L¸ò Ló ðLð
 òL˜ð L°ð L¸ò Ló ðLð
 òL˜ð L°ð L¸ò Ló ðLð
 òL˜ð L°ð L¸ò Ló ðLð
 òL˜ð L°ð L¸ò Ló ðLð
 òL˜ð L°ð L¸ò Ló ðLð
 òL˜ð L°ð L¸ò Ló ðLð
 òL˜ð L°ð L¸ò Ló ðLð òP˜Tð P°Sð P¸[ò Pó ðPð òP˜Tð P°Sð P¸[ò Pó ðPð òP˜Tð P°Sð P¸[ò Pó ðPð òP˜Tð P°Sð P¸[ò Pó ðPð òP˜Tð P°Sð P¸[ò Pó ðPð òP˜Tð P°Sð P¸[ò Pó ðPð òP˜Tð P°Sð P¸[ò Pó ðPð òP˜Tð P°Sð P¸[ò Pó ðPð òP˜Tð P°Sð P¸[ò Pó ðPð òP˜Tð P°Sð P¸[ò Pó ðPð òO˜Dð O°Cð O¸Kò Oó ðOð òO˜Dð O°Cð O¸Kò Oó ðOð òP˜Tð P°Sð P¸[ò Pó ðPð òQ˜dð Q°cð Q¸kò Qó ðQð
 òQ˜dð Q°cð Q¸kò Qó ðQð
 òR˜tð R°sð R¸{ò Ró ðRð
 òR˜tð R°sð R¸{ó Ró ðRð
 òR˜tð R°sð R¸{ó Ró ðRð
 òS ð S¸ð SÀó Só ðSð
 òN˜4ð N°3ð N¸;ó Nó ñNrG   