Ë
    éÓ^j¤j  ã                   óô  — d Z ddlmZmZmZmZmZ ddlZddlm	Z	 ddlm
Z ddlmZmZmZmZmZmZmZmZmZmZmZmZ g d¢Zee	j4                     Zdee   defd	„Z G d
„ de	j4                  «      Z G d„ de	j4                  «      Z G d„ de	j4                  «      Z  G d„ de	j4                  «      Z! G d„ de	j4                  «      Z" G d„ de	j4                  «      Z# G d„ de!«      Z$ G d„ de	j4                  «      Z%y)z[ EfficientNet, MobileNetV3, etc Blocks

Hacked together by / Copyright 2019, Ross Wightman
é    )ÚCallableÚDictÚOptionalÚTypeÚUnionN)Ú
functional)Úcreate_conv2dÚDropPathÚmake_divisibleÚcreate_act_layerÚ	create_aaÚ	to_2tupleÚ	LayerTypeÚConvNormActÚget_norm_act_layerÚMultiQueryAttention2dÚAttention2dÚLayerScale2d)ÚSqueezeExciteÚ	ConvBnActÚDepthwiseSeparableConvÚInvertedResidualÚCondConvResidualÚEdgeResidualÚUniversalInvertedResidualÚMobileAttentionÚ
group_sizeÚchannelsc                 ó&   — | sy|| z  dk(  sJ ‚|| z  S )Né   r   © )r   r   s     úk/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/_efficientnet_blocks.pyÚ
num_groupsr#   "   s(   € ÙØð ˜*Ñ$¨Ò)Ð)Ð)Ø˜:Ñ%Ð%ó    c                   ó’   ‡ — e Zd ZdZddej
                  ej                  ddddfdedede	e   de
de
d	e	e
   d
e	e   fˆ fd„Zd„ Zˆ xZS )r   aç   Squeeze-and-Excitation w/ specific features for EfficientNet/MobileNet family

    Args:
        in_chs (int): input channels to layer
        rd_ratio (float): ratio of squeeze reduction
        act_layer (nn.Module): activation layer of containing block
        gate_layer (Callable): attention gate function
        force_act_layer (nn.Module): override block's activation fn if this is set/bound
        rd_round_fn (Callable): specify a fn to calculate rounding of reduced chs
    g      Ð?NÚin_chsÚrd_ratioÚrd_channelsÚ	act_layerÚ
gate_layerÚforce_act_layerÚrd_round_fnc
                 ó&  •— ||	dœ}
t         ‰| �  «        |€|xs t        } |||z  «      }|xs |}t        j                  ||dfddi|
¤Ž| _        t        |d¬«      | _        t        j                  ||dfddi|
¤Ž| _        t        |«      | _	        y )N©ÚdeviceÚdtyper    ÚbiasT)Úinplace)
ÚsuperÚ__init__ÚroundÚnnÚConv2dÚconv_reducer   Úact1Úconv_expandÚgate)Úselfr&   r'   r(   r)   r*   r+   r,   r/   r0   ÚddÚ	__class__s              €r"   r4   zSqueezeExcite.__init__7   s›   ø€ ð ¨Ñ/ˆÜ‰ÑÔØÐØ%Ò.¬ˆKÙ% f¨xÑ&7Ó8ˆKØ#Ò0 yˆ	ÜŸ9™9 V¨[¸!ÑMÀ$ÐMÈ"ÑMˆÔÜ$ Y¸Ô=ˆŒ	ÜŸ9™9 [°&¸!ÑMÀ$ÐMÈ"ÑMˆÔÜ$ ZÓ0ˆ�	r$   c                 ó¶   — |j                  dd¬«      }| j                  |«      }| j                  |«      }| j                  |«      }|| j	                  |«      z  S )N)é   é   T)Úkeepdim)Úmeanr8   r9   r:   r;   )r<   ÚxÚx_ses      r"   ÚforwardzSqueezeExcite.forwardN   sT   € Ø�v‰v�f dˆvÓ+ˆØ×Ñ Ó%ˆØ�y‰y˜‹ˆØ×Ñ Ó%ˆØ�4—9‘9˜T“?Ñ"Ð"r$   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r6   ÚReLUÚSigmoidÚintÚfloatr   r   r   r4   rF   Ú__classcell__©r>   s   @r"   r   r   +   s†   ø„ ñ	ð #Ø)-Ø#%§7¡7Ø$&§J¡JØ37Ø.2ØØñ1àð1ð ð1ð " #™ð	1ð
 !ð1ð "ð1ð & iÑ0ð1ð " (Ñ+õ1ö.#r$   r   c                   ó¶   ‡ — e Zd ZdZdddddej
                  ej                  ddddfded	ed
ededededeee	f   de
dee   dedee   defˆ fd„Zd„ Zd„ Zˆ xZS )r   z@ Conv + Norm Layer + Activation w/ optional skip connection
    r    r   Ú FNç        r&   Úout_chsÚkernel_sizeÚstrideÚdilationr   Úpad_typeÚskipr)   Ú
norm_layerÚaa_layerÚdrop_path_ratec                 óv  •— ||dœ}t         ‰| �  «        t        |
|	«      }t        ||«      }|xr |dk(  xr ||k(  | _        |d uxr |dkD  }t        |||f|rdn||||dœ|¤Ž| _         ||fddi|¤Ž| _        t        |f|||dœ|¤Ž| _	        |rt        |«      | _        y t        j                  «       | _        y )Nr.   r    ©rV   rW   ÚgroupsÚpaddingr2   T©r   rV   Úenable)r3   r4   r   r#   Úhas_skipr	   ÚconvÚbn1r   Úaar
   r6   ÚIdentityÚ	drop_path)r<   r&   rT   rU   rV   rW   r   rX   rY   r)   rZ   r[   r\   r/   r0   r=   Únorm_act_layerr_   Úuse_aar>   s                      €r"   r4   zConvBnAct.__init__Y   sâ   ø€ ð" ¨Ñ/ˆÜ‰ÑÔÜ+¨J¸	ÓBˆÜ˜J¨Ó/ˆØÒB ¨1¡ÒB°¸7Ñ1BˆŒØ Ð%Ò4¨&°1©*ˆä!ØØØð	
ñ ‘1 FØØØñ	
ð ñ	
ˆŒ	ñ " 'Ñ>°4Ð>¸2Ñ>ˆŒÜ˜HÐ[¨w¸vÈfÑ[ÐXZÑ[ˆŒÙ5Cœ .Ó1ˆ�ÌÏÉËˆ�r$   c                 ó’   — |dk(  r"t        dd| j                  j                  ¬«      S t        d| j                  j                  ¬«      S )NÚ	expansionre   rF   ©ÚmoduleÚ	hook_typeÚnum_chsrR   ©rn   rp   )Údictrd   Úout_channels©r<   Úlocations     r"   Úfeature_infozConvBnAct.feature_info   s:   € Ø�{Ò"Ü˜u°	À4Ç9Á9×CYÑCYÔZÐZä˜r¨4¯9©9×+AÑ+AÔBÐBr$   c                 ó°   — |}| j                  |«      }| j                  |«      }| j                  |«      }| j                  r| j	                  |«      |z   }|S ©N)rd   re   rf   rc   rh   ©r<   rD   Úshortcuts      r"   rF   zConvBnAct.forward…   sM   € ØˆØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØ�G‰G�A‹JˆØ�=Š=Ø—‘˜qÓ! HÑ,ˆAØˆr$   )rG   rH   rI   rJ   r6   rK   ÚBatchNorm2drM   r   ÚstrÚboolr   r   rN   r4   rv   rF   rO   rP   s   @r"   r   r   V   sÙ   ø„ ñð ØØØ(*ØØ-/¯W©WØ$&§N¡NØ,0Ø$&ØØñ$Wàð$Wð ð$Wð ð	$Wð
 ð$Wð ð$Wð ð$Wð ˜C ˜H‘oð$Wð ð$Wð   	Ñ*ð$Wð "ð$Wð ˜yÑ)ð$Wð "õ$WòLCör$   r   c            !       óÆ   ‡ — e Zd ZdZdddddddddej
                  ej                  dddddfd	ed
edededededede	dede	dede
de
dee
   dee   def ˆ fd„Zd„ Zd„ Zˆ xZS )r   zÕ Depthwise-separable block
    Used for DS convs in MobileNet-V1 and in the place of IR blocks that have no expansion
    (factor of 1.0). This is an alternative to having a IR with an optional first pw conv.
    rA   r    rR   Fr   NrS   r&   rT   Údw_kernel_sizerV   rW   r   rX   ÚnoskipÚpw_kernel_sizeÚpw_actÚs2dr)   rZ   r[   Úse_layerr\   c                 óì  •— ||dœ}t         ‰| �  «        t        ||«      }|dk(  xr ||k(  xr | | _        |
| _        |d uxr |dkD  }|dk(  rHt        |dz  «      }t        ||fddddœ|¤Ž| _         ||fi |¤Ž| _        |dz   dz  }|dk(  rdn|}|}d}nd | _        d | _        |}t        ||«      }t        |||f|rdn||||dœ|¤Ž| _
         ||fd	d
i|¤Ž| _        t        |f|||dœ|¤Ž| _        |r ||fd|i|¤Žnt        j                  «       | _        t        |||	fd|i|¤Ž| _         ||fd
| j                  dœ|¤Ž| _        |rt'        |«      | _        y t        j                  «       | _        y )Nr.   r    é   r@   Úsame©rU   rV   r`   F)rV   rW   r`   r_   r2   Tra   r)   r`   )r2   Ú	apply_act)r3   r4   r   rc   Ú
has_pw_actrM   r	   Úconv_s2dÚbn_s2dr#   Úconv_dwre   r   rf   r6   rg   ÚseÚconv_pwÚbn2r
   rh   )r<   r&   rT   r   rV   rW   r   rX   r€   r�   r‚   rƒ   r)   rZ   r[   r„   r\   r/   r0   r=   ri   rj   Úsd_chsÚdw_pad_typer_   r>   s                            €r"   r4   zDepthwiseSeparableConv.__init__”   sÇ  ø€ ð* ¨Ñ/ˆÜ‰ÑÔÜ+¨J¸	ÓBˆØ 1™Ò:¨°7Ñ):ÒJÀFÀ
ˆŒØ ˆŒØ Ð%Ò4¨&°1©*ˆð �!Š8Ü˜ !™“_ˆFÜ)¨&°&ÐhÀaÐPQÐ[aÑhÐegÑhˆDŒMÙ(¨Ñ6°2Ñ6ˆDŒKØ,¨qÑ0°QÑ6ˆNØ$2°aÒ$7™&¸XˆKØˆFØ‰Fà ˆDŒMØˆDŒKØ"ˆKä˜J¨Ó/ˆä$ØØØð	
ñ ‘1 FØØØñ	
ð ñ	
ˆŒñ " &Ñ=°$Ð=¸"Ñ=ˆŒÜ˜HÐ[¨w¸vÈfÑ[ÐXZÑ[ˆŒñ BJ‘(˜6Ñ=¨YÐ=¸"Ò=ÌrÏ{É{Ë}ˆŒä$ V¨W°nÑ]ÈhÐ]ÐZ\Ñ]ˆŒÙ! 'ÐY°4À4Ç?Á?ÑYÐVXÑYˆŒÙ5Cœ .Ó1ˆ�ÌÏÉËˆ�r$   c                 ó’   — |dk(  r"t        dd| j                  j                  ¬«      S t        d| j                  j                  ¬«      S ©Nrl   r�   Úforward_prerm   rR   rq   ©rr   r�   Úin_channelsrs   rt   s     r"   rv   z#DepthwiseSeparableConv.feature_infoÔ   ó:   € Ø�{Ò"Ü˜y°MÈ4Ï<É<×KcÑKcÔdÐdä˜r¨4¯<©<×+DÑ+DÔEÐEr$   c                 ór  — |}| j                   �"| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  r| j                  |«      |z   }|S rx   )
r‹   rŒ   r�   re   rf   rŽ   r�   r�   rc   rh   ry   s      r"   rF   zDepthwiseSeparableConv.forwardÚ   s™   € ØˆØ�=‰=Ð$Ø—‘˜aÓ ˆAØ—‘˜A“ˆAØ�L‰L˜‹OˆØ�H‰H�Q‹KˆØ�G‰G�A‹JˆØ�G‰G�A‹JˆØ�L‰L˜‹OˆØ�H‰H�Q‹KˆØ�=Š=Ø—‘˜qÓ! HÑ,ˆAØˆr$   )rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   r   r   Ú
ModuleTyperN   r4   rv   rF   rO   rP   s   @r"   r   r   �   s  ø„ ñð #$ØØØØØ Ø"#Ø ØØ#%§7¡7Ø$&§N¡NØ,0Ø-1Ø$&ØØñ'>Wàð>Wð ð>Wð  ð	>Wð
 ð>Wð ð>Wð ð>Wð ð>Wð ð>Wð  ð>Wð ð>Wð ð>Wð !ð>Wð "ð>Wð ˜yÑ)ð>Wð  ˜zÑ*ð!>Wð" "õ#>Wò@För$   r   c            %       óØ   ‡ — e Zd ZdZddddddddddej
                  ej                  dddd	ddfd
ededededededede	de
dedededededee   dee   dee   de
f$ˆ fd„Zd„ Zd„ Zˆ xZS )r   a„   Inverted residual block w/ optional SE

    Originally used in MobileNet-V2 - https://arxiv.org/abs/1801.04381v4, this layer is often
    referred to as 'MBConv' for (Mobile inverted bottleneck conv) and is also used in
      * MNasNet - https://arxiv.org/abs/1807.11626
      * EfficientNet - https://arxiv.org/abs/1905.11946
      * MobileNet-V3 - https://arxiv.org/abs/1905.02244
    rA   r    rR   Fç      ð?r   NrS   r&   rT   r   rV   rW   r   rX   r€   Ú	exp_ratioÚexp_kernel_sizer�   rƒ   r)   rZ   r[   r„   Úconv_kwargsr\   c                 óF  •— ||dœ}t         ‰| �  «        t        ||«      }|xs i }||k(  xr |dk(  xr | | _        |d uxr |dkD  }|dk(  rHt	        |dz  «      }t        ||fddddœ|¤Ž| _         ||fi |¤Ž| _        |dz   dz  }|dk(  rdn|}|}d}nd | _        d | _        |}t        ||	z  «      }t        ||«      }t        |||
fd|i|¤|¤Ž| _
         ||fd	d
i|¤Ž| _        t        |||f|rdn||||dœ|¤|¤Ž| _         ||fd	d
i|¤Ž| _        t        |f|||dœ|¤Ž| _        |r ||fd|i|¤Žnt!        j"                  «       | _        t        |||fd|i|¤|¤Ž| _         ||fddi|¤Ž| _        |rt+        |«      | _        y t!        j"                  «       | _        y )Nr.   r    r†   r@   r‡   rˆ   Fr`   r2   Tr^   ra   r)   r‰   )r3   r4   r   rc   rM   r	   r‹   rŒ   r   r#   r�   re   r�   r�   r   rf   r6   rg   rŽ   Úconv_pwlÚbn3r
   rh   )r<   r&   rT   r   rV   rW   r   rX   r€   r�   rž   r�   rƒ   r)   rZ   r[   r„   rŸ   r\   r/   r0   r=   ri   rj   r‘   r’   Úmid_chsr_   r>   s                               €r"   r4   zInvertedResidual.__init__ô   s$  ø€ ð. ¨Ñ/ˆÜ‰ÑÔÜ+¨J¸	ÓBˆØ!Ò' RˆØ 7Ñ*Ò:¨v¸©{ÒJÀFÀ
ˆŒØ Ð%Ò4¨&°1©*ˆð �!Š8Ü˜ !™“_ˆFÜ)¨&°&ÐhÀaÐPQÐ[aÑhÐegÑhˆDŒMÙ(¨Ñ6°2Ñ6ˆDŒKØ,¨qÑ0°QÑ6ˆNØ$2°aÒ$7™&¸XˆKØˆFØ‰Fà ˆDŒMØˆDŒKØ"ˆKä  ¨)Ñ!3Ó4ˆÜ˜J¨Ó0ˆô % V¨W°oÑmÈxÐmÐ[fÐmÐjlÑmˆŒÙ! 'Ñ>°4Ð>¸2Ñ>ˆŒô %ØØØð

ñ ‘1 FØØØñ

ð ð

ð ñ

ˆŒñ " 'Ñ>°4Ð>¸2Ñ>ˆŒÜ˜HÐ[¨w¸vÈfÑ[ÐXZÑ[ˆŒñ CK‘(˜7Ñ>¨iÐ>¸2Ò>ÔPR×P[ÑP[ÓP]ˆŒô & g¨w¸ÑnÐPXÐnÐ\gÐnÐkmÑnˆŒÙ! 'ÑA°UÐA¸bÑAˆŒÙ5Cœ .Ó1ˆ�ÌÏÉËˆ�r$   c                 ó’   — |dk(  r"t        dd| j                  j                  ¬«      S t        d| j                  j                  ¬«      S ©Nrl   r¡   r•   rm   rR   rq   ©rr   r¡   r—   rs   rt   s     r"   rv   zInvertedResidual.feature_info>  ó:   € Ø�{Ò"Ü˜z°]ÈDÏMÉM×LeÑLeÔfÐfä˜r¨4¯=©=×+EÑ+EÔFÐFr$   c                 ó¶  — |}| j                   �"| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  r| j                  |«      |z   }|S rx   )r‹   rŒ   r�   re   r�   r�   rf   rŽ   r¡   r¢   rc   rh   ry   s      r"   rF   zInvertedResidual.forwardD  s´   € ØˆØ�=‰=Ð$Ø—‘˜aÓ ˆAØ—‘˜A“ˆAØ�L‰L˜‹OˆØ�H‰H�Q‹KˆØ�L‰L˜‹OˆØ�H‰H�Q‹KˆØ�G‰G�A‹JˆØ�G‰G�A‹JˆØ�M‰M˜!ÓˆØ�H‰H�Q‹KˆØ�=Š=Ø—‘˜qÓ! HÑ,ˆAØˆr$   ©rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   rN   r   r   rš   r   r4   rv   rF   rO   rP   s   @r"   r   r   ê   s?  ø„ ñð #$ØØØØØ Ø"Ø#$Ø"#ØØ#%§7¡7Ø$&§N¡NØ,0Ø-1Ø*.Ø$&ØØñ+HWàðHWð ðHWð  ð	HWð
 ðHWð ðHWð ðHWð ðHWð ðHWð ðHWð !ðHWð  ðHWð ðHWð !ðHWð "ðHWð  ˜yÑ)ð!HWð" ˜zÑ*ð#HWð$ " $™ð%HWð& "õ'HWòTGör$   r   c            &       óÞ   ‡ — e Zd ZdZdddddddddej
                  ej                  dddd	d
ddfdededededededededede	de
dededee   dee   dee   de
dee
   f$ˆ fd„Zd„ Zd„ Zˆ xZS ) r   a   Universal Inverted Residual Block (aka Universal Inverted Bottleneck, UIB)

    For MobileNetV4 - https://arxiv.org/abs/, referenced from
    https://github.com/tensorflow/models/blob/d93c7e932de27522b2fa3b115f58d06d6f640537/official/vision/modeling/layers/nn_blocks.py#L778
    r   rA   r    rR   Frœ   NrS   çñhãˆµøä>r&   rT   Údw_kernel_size_startÚdw_kernel_size_midÚdw_kernel_size_endrV   rW   r   rX   r€   r�   r)   rZ   r[   r„   rŸ   r\   Úlayer_scale_init_valuec                 óÎ  •— ||dœ}t         ‰| �  «        |xs i }||k(  xr |dk(  xr |
 | _        |dkD  r|s|s|sJ ‚|r1|s|nd}t        ||«      }t	        |||f||||	d|||dœ|¤|¤Ž| _        nt        j                  «       | _        t        ||z  «      }t	        ||df|	||dœ|¤|¤Ž| _	        |r*t        ||«      }t	        |||f||||	|||dœ|¤|¤Ž| _
        nt        j                  «       | _
        |r ||fd|i|¤Žnt        j                  «       | _        t	        ||df|	d||dœ|¤|¤Ž| _        |r;|s|s|nd}t        ||«      }|dkD  r|rJ ‚t	        |||f||||	d||d	œ|¤|¤Ž| _        nt        j                  «       | _        |�t        ||fi |¤Ž| _        nt        j                  «       | _        |rt!        |«      | _        y t        j                  «       | _        y )
Nr.   r    F)rV   rW   r_   r`   r‰   r)   rZ   r[   )r`   r)   rZ   )rV   rW   r_   r`   r)   rZ   r[   r)   )r`   r‰   r)   rZ   )rV   rW   r_   r`   r‰   r)   rZ   )r3   r4   rc   r#   r   Údw_startr6   rg   r   Úpw_expÚdw_midrŽ   Úpw_projÚdw_endr   Úlayer_scaler
   rh   )r<   r&   rT   r¬   r­   r®   rV   rW   r   rX   r€   r�   r)   rZ   r[   r„   rŸ   r\   r¯   r/   r0   r=   Údw_start_strideÚdw_start_groupsr£   r_   Údw_end_strideÚdw_end_groupsr>   s                               €r"   r4   z"UniversalInvertedResidual.__init__]  s�  ø€ ð. ¨Ñ/ˆÜ‰ÑÔØ!Ò' RˆØ 7Ñ*Ò:¨v¸©{ÒJÀFÀ
ˆŒØ�AŠ:Ù'Ñ+=ÑASÐSÐSñ  Ù,>™fÀAˆOÜ(¨°VÓ<ˆOÜ'Ø˜Ð 4ðà&Ø!Ø&Ø ØØ#Ø%Ø!ñð ðð ñˆD�Mô ŸK™K›MˆDŒMô ! ¨)Ñ!3Ó4ˆÜ!Ø�G˜Qð
àØØ!ñ	
ð
 ð
ð ñ
ˆŒñ Ü 
¨GÓ4ˆFÜ%Ø˜Ð"4ðàØ!ØØ Ø#Ø%Ø!ñð ðð ñˆD�Kô Ÿ+™+›-ˆDŒKñ CK‘(˜7Ñ>¨iÐ>¸2Ò>ÔPR×P[ÑP[ÓP]ˆŒô #Ø�W˜að
àØØØ!ñ
ð ð
ð ñ
ˆŒñ Ù*>ÑGY™FÐ_`ˆMÜ& z°7Ó;ˆMØ˜qÒ Ù#Ð#�|Ü%Ø˜Ð"4ðà$Ø!Ø$Ø ØØ#Ø%ñð ðð ñˆD�Kô Ÿ+™+›-ˆDŒKà!Ð-Ü+¨GÐ5KÑRÈrÑRˆDÕä!Ÿ{™{›}ˆDÔÙ5Cœ .Ó1ˆ�ÌÏÉËˆ�r$   c                 óº   — |dk(  r,t        dd| j                  j                  j                  ¬«      S t        d| j                  j                  j                  ¬«      S )Nrl   zpw_proj.convr•   rm   rR   rq   )rr   r´   rd   r—   rs   rt   s     r"   rv   z&UniversalInvertedResidual.feature_infoÕ  sI   € Ø�{Ò"Ü˜~¸ÐPT×P\ÑP\×PaÑPa×PmÑPmÔnÐnä˜r¨4¯<©<×+<Ñ+<×+IÑ+IÔJÐJr$   c                 ó8  — |}| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }| j                  |«      }| j                  |«      }| j                  r| j                  |«      |z   }|S rx   )	r±   r²   r³   rŽ   r´   rµ   r¶   rc   rh   ry   s      r"   rF   z!UniversalInvertedResidual.forwardÛ  s…   € ØˆØ�M‰M˜!ÓˆØ�K‰K˜‹NˆØ�K‰K˜‹NˆØ�G‰G�A‹JˆØ�L‰L˜‹OˆØ�K‰K˜‹NˆØ×Ñ˜QÓˆØ�=Š=Ø—‘˜qÓ! HÑ,ˆAØˆr$   r©   rP   s   @r"   r   r   V  sC  ø„ ñð )*Ø&'Ø&'ØØØØØ Ø"Ø#%§7¡7Ø$&§N¡NØ,0Ø-1Ø*.Ø$&Ø6:ØØñ+vWàðvWð ðvWð #&ð	vWð
 !$ðvWð !$ðvWð ðvWð ðvWð ðvWð ðvWð ðvWð ðvWð !ðvWð "ðvWð ˜yÑ)ðvWð  ˜zÑ*ð!vWð" " $™ð#vWð$ "ð%vWð& %-¨U¡Oõ'vWòpKör$   r   c            1       óö   ‡ — e Zd ZdZdddddddddddddej
                  ej                  d	d
d
d
dddd	d	fdedededededededededede	dededede	de
de
dee
   deded ed!ee   d"e	d#e	f0ˆ fd$„Zd%„ Zd&„ Zˆ xZS )'r   zå Mobile Attention Block

    For MobileNetV4 - https://arxiv.org/abs/, referenced from
    https://github.com/tensorflow/models/blob/d93c7e932de27522b2fa3b115f58d06d6f640537/official/vision/modeling/layers/nn_blocks.py#L1504
    r    rA   rR   é   é@   F)r    r    NrS   r«   r&   rT   rV   r   rW   r   rX   Ú	num_headsÚkey_dimÚ	value_dimÚuse_multi_queryÚquery_stridesÚ	kv_strideÚcpe_dw_kernel_sizer€   r)   rZ   r[   r\   Ú	attn_dropÚ	proj_dropr¯   Úuse_biasÚuse_cpec                 ó¦  •— ||dœ}t         ‰| �  «        t        ||«      }|dk(  xr ||k(  xr | | _        t	        |«      | _        || _        t        | j
                  D �cg c]  }|dkD  ‘Œ	 c}«      | _        |rt        ||f||dddœ|¤Ž| _
        nd | _
         ||fddi|¤Ž| _        |€||	z  dk(  sJ ‚||	z  }|rt        |f|||	|
||||||||dœ|¤Ž| _        nt        |f|||||d	œ|¤Ž| _        |�t        ||fi |¤Ž| _        nt#        j$                  «       | _        |rt'        |«      | _        y t#        j$                  «       | _        y c c}w )
Nr.   r    T)rU   rW   Ú	depthwiser1   r‰   Fr   )Údim_outrÀ   rÁ   rÂ   rÄ   rÅ   r   rW   r`   rÇ   rÈ   rZ   )rÍ   rÀ   rÇ   rÈ   r1   )r3   r4   r   rc   r   rÄ   rÅ   ÚanyÚhas_query_strider	   Úconv_cpe_dwÚnormr   Úattnr   r   r¶   r6   rg   r
   rh   )r<   r&   rT   rV   r   rW   r   rX   rÀ   rÁ   rÂ   rÃ   rÄ   rÅ   rÆ   r€   r)   rZ   r[   r\   rÇ   rÈ   r¯   rÉ   rÊ   r/   r0   r=   ri   Úsr>   s                                 €r"   r4   zMobileAttention.__init__ï  s­  ø€ ð: ¨Ñ/ˆÜ‰ÑÔÜ+¨J¸	ÓBˆØ 1™Ò:¨°7Ñ):ÒJÀFÀ
ˆŒÜ& }Ó5ˆÔØ"ˆŒÜ #°D×4FÑ4FÖ$G¨q Q¨£UÒ$GÓ HˆÔñ Ü,Ø˜ð à.Ø!ØØñ ð ñ ˆDÕð  $ˆDÔá" 6ÑA°UÐA¸bÑAˆŒ	àÐØ˜GÑ# qÒ(Ð(Ð(Ø 'Ñ)ˆIáÜ-ØðàØ#ØØ#Ø+Ø#Ø-Ø!Ø Ø#Ø#Ø%ñð ñˆD�Iô$ $ØðàØ#Ø#Ø#Øñð ñˆDŒIð "Ð-Ü+¨GÐ5KÑRÈrÑRˆDÕä!Ÿ{™{›}ˆDÔá5Cœ .Ó1ˆ�ÌÏÉËˆ�ùòw %Hs   Á!Ec                 ó’   — |dk(  r"t        dd| j                  j                  ¬«      S t        d| j                  j                  ¬«      S r”   r–   rt   s     r"   rv   zMobileAttention.feature_infoO  r˜   r$   c                 óô   — | j                   �| j                  |«      }||z   }|}| j                  |«      }| j                  |«      }| j                  |«      }| j                  r| j                  |«      |z   }|S rx   )rÐ   rÑ   rÒ   r¶   rc   rh   )r<   rD   Úx_cperz   s       r"   rF   zMobileAttention.forwardU  su   € Ø×ÑÐ'Ø×$Ñ$ QÓ'ˆEØ�E‘	ˆAàˆØ�I‰I�a‹LˆØ�I‰I�a‹LˆØ×Ñ˜QÓˆØ�=Š=Ø—‘˜qÓ! HÑ,ˆAàˆr$   )rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   r   r   rN   r4   rv   rF   rO   rP   s   @r"   r   r   é  s”  ø„ ñð Ø"#ØØØØØØØ$)Ø!'ØØ&'Ø Ø#%§7¡7Ø$&§N¡NØ,0Ø$&Ø"Ø"Ø6:Ø"Ø!ØØñ7^Wàð^Wð ð^Wð ð	^Wð
  ð^Wð ð^Wð ð^Wð ð^Wð ð^Wð ð^Wð ð^Wð "ð^Wð ð^Wð ð^Wð !$ð^Wð  ð!^Wð" !ð#^Wð$ "ð%^Wð& ˜yÑ)ð'^Wð( "ð)^Wð* ð+^Wð, ð-^Wð. %-¨U¡Oð/^Wð0 ð1^Wð2 õ3^Wò@För$   r   c            #       óÆ   ‡ — e Zd ZdZdddddddddej
                  ej                  dddd	ddfd
ededededededede	de
dededededee   dee   dede
f"ˆ fd„Zd„ Zˆ xZS )r   z, Inverted residual block w/ CondConv routingrA   r    rR   Frœ   Nr   rS   r&   rT   r   rV   rW   r   rX   r€   r�   rž   r�   r)   rZ   r[   r„   Únum_expertsr\   c                 óØ   •— ||dœ}|| _         t        | j                   ¬«      }t        ‰| �  ||f|||||||	|
|||||||dœ|¤Ž t	        j
                  || j                   fi |¤Ž| _        y )Nr.   )rØ   )r   rV   rW   r   rX   r€   r�   rž   r�   r)   rZ   r[   r„   rŸ   r\   )rØ   rr   r3   r4   r6   ÚLinearÚ
routing_fn)r<   r&   rT   r   rV   rW   r   rX   r€   r�   rž   r�   r)   rZ   r[   r„   rØ   r\   r/   r0   r=   rŸ   r>   s                         €r"   r4   zCondConvResidual.__init__g  s—   ø€ ð, ¨Ñ/ˆØ&ˆÔÜ t×'7Ñ'7Ô8ˆÜ‰ÑØØð	
ð *ØØØ!ØØØØ+Ø)ØØ!ØØØ#Ø)ñ#	
ð$ ò%	
ô( Ÿ)™) F¨D×,<Ñ,<ÑCÀÑCˆ�r$   c                 óÐ  — |}t        j                  |d«      j                  d«      }t        j                  | j                  |«      «      }| j                  ||«      }| j                  |«      }| j                  ||«      }| j                  |«      }| j                  |«      }| j                  ||«      }| j                  |«      }| j                  r| j                  |«      |z   }|S )Nr    )ÚFÚadaptive_avg_pool2dÚflattenÚtorchÚsigmoidrÛ   r�   re   r�   r�   rŽ   r¡   r¢   rc   rh   )r<   rD   rz   Úpooled_inputsÚrouting_weightss        r"   rF   zCondConvResidual.forward–  s¾   € ØˆÜ×-Ñ-¨a°Ó3×;Ñ;¸AÓ>ˆÜŸ-™-¨¯©¸Ó(FÓGˆØ�L‰L˜˜OÓ,ˆØ�H‰H�Q‹KˆØ�L‰L˜˜OÓ,ˆØ�H‰H�Q‹KˆØ�G‰G�A‹JˆØ�M‰M˜!˜_Ó-ˆØ�H‰H�Q‹KˆØ�=Š=Ø—‘˜qÓ! HÑ,ˆAØˆr$   )rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   rN   r   r   rš   r4   rF   rO   rP   s   @r"   r   r   d  s  ø„ Ù6ð #$ØØØØØ Ø"Ø#$Ø"#Ø#%§7¡7Ø$&§N¡NØ,0Ø-1Ø Ø$&ØØñ)-Dàð-Dð ð-Dð  ð	-Dð
 ð-Dð ð-Dð ð-Dð ð-Dð ð-Dð ð-Dð !ð-Dð  ð-Dð !ð-Dð "ð-Dð ˜yÑ)ð-Dð  ˜zÑ*ð!-Dð" ð#-Dð$ "õ%-Dö^r$   r   c            !       óÆ   ‡ — e Zd ZdZdddddddddej
                  ej                  ddd	ddfd
edededededededede	de
dedededee   dee   de
f ˆ fd„Zd„ Zd„ Zˆ xZS )r   a(   Residual block with expansion convolution followed by pointwise-linear w/ stride

    Originally introduced in `EfficientNet-EdgeTPU: Creating Accelerator-Optimized Neural Networks with AutoML`
        - https://ai.googleblog.com/2019/08/efficientnet-edgetpu-creating.html

    This layer is also called FusedMBConv in the MobileDet, EfficientNet-X, and EfficientNet-V2 papers
      * MobileDet - https://arxiv.org/abs/2004.14525
      * EfficientNet-X - https://arxiv.org/abs/2102.05610
      * EfficientNet-V2 - https://arxiv.org/abs/2104.00298
    rA   r    r   rR   Frœ   NrS   r&   rT   rž   rV   rW   r   rX   Úforce_in_chsr€   r�   r�   r)   rZ   r[   r„   r\   c                 óR  •— ||dœ}t         ‰| �  «        t        ||«      }|dkD  rt        ||
z  «      }nt        ||
z  «      }t	        ||«      }||k(  xr |dk(  xr |	 | _        |d uxr |dkD  }t        |||f|rdn||||dœ|¤Ž| _         ||fddi|¤Ž| _        t        |f|||dœ|¤Ž| _
        |r ||fd|i|¤Žnt        j                  «       | _        t        |||fd	|i|¤Ž| _         ||fd
di|¤Ž| _        |rt!        |«      | _        y t        j                  «       | _        y )Nr.   r   r    r^   r2   Tra   r)   r`   r‰   F)r3   r4   r   r   r#   rc   r	   Úconv_expre   r   rf   r6   rg   rŽ   r¡   r�   r
   rh   )r<   r&   rT   rž   rV   rW   r   rX   rå   r€   r�   r�   r)   rZ   r[   r„   r\   r/   r0   r=   ri   r£   r_   rj   r>   s                           €r"   r4   zEdgeResidual.__init__²  sh  ø€ ð* ¨Ñ/ˆÜ‰ÑÔÜ+¨J¸	ÓBˆØ˜!ÒÜ$ \°IÑ%=Ó>‰Gä$ V¨iÑ%7Ó8ˆGÜ˜J¨Ó0ˆØ 7Ñ*Ò:¨v¸©{ÒJÀFÀ
ˆŒØ Ð%Ò4¨&°1©*ˆô &ØØØð	
ñ ‘1 FØØØñ	
ð ñ	
ˆŒñ " 'Ñ>°4Ð>¸2Ñ>ˆŒä˜HÐ[¨w¸vÈfÑ[ÐXZÑ[ˆŒñ CK‘(˜7Ñ>¨iÐ>¸2Ò>ÔPR×P[ÑP[ÓP]ˆŒô & g¨w¸Ñ_ÐPXÐ_Ð\^Ñ_ˆŒÙ! 'ÑA°UÐA¸bÑAˆŒÙ5Cœ .Ó1ˆ�ÌÏÉËˆ�r$   c                 ó’   — |dk(  r"t        dd| j                  j                  ¬«      S t        d| j                  j                  ¬«      S r¥   r¦   rt   s     r"   rv   zEdgeResidual.feature_infoé  r§   r$   c                 ó  — |}| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }| j                  |«      }| j                  r| j                  |«      |z   }|S rx   )rç   re   rf   rŽ   r¡   r�   rc   rh   ry   s      r"   rF   zEdgeResidual.forwardï  sv   € ØˆØ�M‰M˜!ÓˆØ�H‰H�Q‹KˆØ�G‰G�A‹JˆØ�G‰G�A‹JˆØ�M‰M˜!ÓˆØ�H‰H�Q‹KˆØ�=Š=Ø—‘˜qÓ! HÑ,ˆAØˆr$   )rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   rN   r   r   rš   r4   rv   rF   rO   rP   s   @r"   r   r   ¦  s  ø„ ñ	ð $%ØØØØØ !Ø Ø"Ø#$Ø#%§7¡7Ø$&§N¡NØ,0Ø-1Ø$&ØØñ'5Wàð5Wð ð5Wð !ð	5Wð
 ð5Wð ð5Wð ð5Wð ð5Wð ð5Wð ð5Wð ð5Wð !ð5Wð !ð5Wð "ð5Wð ˜yÑ)ð5Wð  ˜zÑ*ð!5Wð" "õ#5WònGö
r$   r   )&rJ   Útypingr   r   r   r   r   rà   Útorch.nnr6   r   rÝ   Útimm.layersr	   r
   r   r   r   r   r   r   r   r   r   r   Ú__all__ÚModulerš   rM   r#   r   r   r   r   r   r   r   r   r!   r$   r"   ú<module>rï      sê   ðñ÷ 9Õ 8ã Ý Ý $÷÷ ÷ ó ò€ð
 �"—)‘)‰_€
ð&˜8 C™=ð &°Có &ô(#�B—I‘Iô (#ôV6�—	‘	ô 6ôrX˜RŸY™Yô Xôvi�r—y‘yô iôXP §	¡	ô Pôfx�b—i‘iô xôv?Ð'ô ?ôDS�2—9‘9õ Sr$   