
    ^jj                        d Z ddlmZmZmZmZmZ ddlZddlm	Z	 ddlm
Z ddlmZmZmZmZmZmZmZmZmZmZmZmZ g dZee	j4                     Zdee   defd	Z G d
 de	j4                        Z G d de	j4                        Z G d de	j4                        Z  G d de	j4                        Z! G d de	j4                        Z" G d de	j4                        Z# G d de!      Z$ G d de	j4                        Z%y)z[ EfficientNet, MobileNetV3, etc Blocks

Hacked together by / Copyright 2019, Ross Wightman
    )CallableDictOptionalTypeUnionN)
functional)create_conv2dDropPathmake_divisiblecreate_act_layer	create_aa	to_2tuple	LayerTypeConvNormActget_norm_act_layerMultiQueryAttention2dAttention2dLayerScale2d)SqueezeExcite	ConvBnActDepthwiseSeparableConvInvertedResidualCondConvResidualEdgeResidualUniversalInvertedResidualMobileAttention
group_sizechannelsc                 &    | sy|| z  dk(  sJ || z  S )N   r    )r   r   s     k/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/_efficientnet_blocks.py
num_groupsr#   "   s(     *$))):%%    c                        e Zd ZdZddej
                  ej                  ddddfdedede	e   de
de
d	e	e
   d
e	e   f fdZd Z xZS )r   a   Squeeze-and-Excitation w/ specific features for EfficientNet/MobileNet family

    Args:
        in_chs (int): input channels to layer
        rd_ratio (float): ratio of squeeze reduction
        act_layer (nn.Module): activation layer of containing block
        gate_layer (Callable): attention gate function
        force_act_layer (nn.Module): override block's activation fn if this is set/bound
        rd_round_fn (Callable): specify a fn to calculate rounding of reduced chs
    g      ?Nin_chsrd_ratiord_channels	act_layer
gate_layerforce_act_layerrd_round_fnc
                 &   ||	d}
t         |           ||xs t        } |||z        }|xs |}t        j                  ||dfddi|
| _        t        |d      | _        t        j                  ||dfddi|
| _        t        |      | _	        y )Ndevicedtyper    biasT)inplace)
super__init__roundnnConv2dconv_reducer   act1conv_expandgate)selfr&   r'   r(   r)   r*   r+   r,   r/   r0   dd	__class__s              r"   r4   zSqueezeExcite.__init__7   s     /%.K%fx&78K#0y	99V[!M$M"M$Y=	99[&!M$M"M$Z0	r$   c                     |j                  dd      }| j                  |      }| j                  |      }| j                  |      }|| j	                  |      z  S )N)      T)keepdim)meanr8   r9   r:   r;   )r<   xx_ses      r"   forwardzSqueezeExcite.forwardN   sT    vvfdv+%yy%499T?""r$   )__name__
__module____qualname____doc__r6   ReLUSigmoidintfloatr   r   r   r4   rF   __classcell__r>   s   @r"   r   r   +   s    	 #)-#%77$&JJ37.211 1 "#	1
 !1 "1 &i01 "(+1.#r$   r   c                        e Zd ZdZdddddej
                  ej                  ddddfded	ed
ededededeee	f   de
dee   dedee   def fdZd Zd Z xZS )r   z@ Conv + Norm Layer + Activation w/ optional skip connection
    r    r    FN        r&   out_chskernel_sizestridedilationr   pad_typeskipr)   
norm_layeraa_layerdrop_path_ratec                 v   ||d}t         |           t        |
|	      }t        ||      }|xr |dk(  xr ||k(  | _        |d uxr |dkD  }t        |||f|rdn||||d|| _         ||fddi|| _        t        |f|||d|| _	        |rt        |      | _        y t        j                         | _        y )Nr.   r    rV   rW   groupspaddingr2   Tr   rV   enable)r3   r4   r   r#   has_skipr	   convbn1r   aar
   r6   Identity	drop_path)r<   r&   rT   rU   rV   rW   r   rX   rY   r)   rZ   r[   r\   r/   r0   r=   norm_act_layerr_   use_aar>   s                      r"   r4   zConvBnAct.__init__Y   s    " /+J	BJ/B1B71B%4&1*!	
 1F	
 	
	 "'>4>2>H[wvf[XZ[5C.1r$   c                     |dk(  r"t        dd| j                  j                        S t        d| j                  j                        S )N	expansionre   rF   module	hook_typenum_chsrR   rn   rp   )dictrd   out_channelsr<   locations     r"   feature_infozConvBnAct.feature_info   s:    {"u	499CYCYZZr499+A+ABBr$   c                     |}| j                  |      }| j                  |      }| j                  |      }| j                  r| j	                  |      |z   }|S N)rd   re   rf   rc   rh   r<   rD   shortcuts      r"   rF   zConvBnAct.forward   sM    IIaLHHQKGGAJ==q!H,Ar$   )rG   rH   rI   rJ   r6   rK   BatchNorm2drM   r   strboolr   r   rN   r4   rv   rF   rO   rP   s   @r"   r   r   V   s     (*-/WW$&NN,0$&$W$W $W 	$W
 $W $W $W CHo$W $W  	*$W "$W y)$W "$WLCr$   r   c            !            e Zd ZdZdddddddddej
                  ej                  dddddfd	ed
edededededede	dede	dede
de
dee
   dee   def  fdZd Zd Z xZS )r   z Depthwise-separable block
    Used for DS convs in MobileNet-V1 and in the place of IR blocks that have no expansion
    (factor of 1.0). This is an alternative to having a IR with an optional first pw conv.
    rA   r    rR   Fr   NrS   r&   rT   dw_kernel_sizerV   rW   r   rX   noskippw_kernel_sizepw_acts2dr)   rZ   r[   se_layerr\   c                    ||d}t         |           t        ||      }|dk(  xr ||k(  xr | | _        |
| _        |d uxr |dkD  }|dk(  rHt        |dz        }t        ||fdddd|| _         ||fi || _        |dz   dz  }|dk(  rdn|}|}d}nd | _        d | _        |}t        ||      }t        |||f|rdn||||d|| _
         ||fd	d
i|| _        t        |f|||d|| _        |r ||fd|i|nt        j                         | _        t        |||	fd|i|| _         ||fd
| j                  d|| _        |rt'        |      | _        y t        j                         | _        y )Nr.   r       r@   samerU   rV   r`   F)rV   rW   r`   r_   r2   Tra   r)   r`   )r2   	apply_act)r3   r4   r   rc   
has_pw_actrM   r	   conv_s2dbn_s2dr#   conv_dwre   r   rf   r6   rg   seconv_pwbn2r
   rh   )r<   r&   rT   r   rV   rW   r   rX   r   r   r   r   r)   rZ   r[   r   r\   r/   r0   r=   ri   rj   sd_chsdw_pad_typer_   r>   s                            r"   r4   zDepthwiseSeparableConv.__init__   s   * /+J	B1:7):JF
 %4&1* !8!_F)&&haPQ[aheghDM(626DK,q0Q6N$2a$7&XKFF DMDK"KJ/$	
 1F	
 	
 "&=$="=H[wvf[XZ[ BJ(6=Y="=r{{}$VWn]h]Z\]!'Y44??YVXY5C.1r$   c                     |dk(  r"t        dd| j                  j                        S t        d| j                  j                        S Nrl   r   forward_prerm   rR   rq   rr   r   in_channelsrs   rt   s     r"   rv   z#DepthwiseSeparableConv.feature_info   :    {"yM4<<KcKcddr4<<+D+DEEr$   c                 r   |}| j                   "| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  r| j                  |      |z   }|S rx   )
r   r   r   re   rf   r   r   r   rc   rh   ry   s      r"   rF   zDepthwiseSeparableConv.forward   s    ==$a AAALLOHHQKGGAJGGAJLLOHHQK==q!H,Ar$   )rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   r   r   
ModuleTyperN   r4   rv   rF   rO   rP   s   @r"   r   r      s    #$ "# #%77$&NN,0-1$&'>W>W >W  	>W
 >W >W >W >W >W  >W >W >W !>W ">W y)>W  z*!>W" "#>W@Fr$   r   c            %            e Zd ZdZddddddddddej
                  ej                  dddd	ddfd
ededededededede	de
dedededededee   dee   dee   de
f$ fdZd Zd Z xZS )r   a   Inverted residual block w/ optional SE

    Originally used in MobileNet-V2 - https://arxiv.org/abs/1801.04381v4, this layer is often
    referred to as 'MBConv' for (Mobile inverted bottleneck conv) and is also used in
      * MNasNet - https://arxiv.org/abs/1807.11626
      * EfficientNet - https://arxiv.org/abs/1905.11946
      * MobileNet-V3 - https://arxiv.org/abs/1905.02244
    rA   r    rR   F      ?r   NrS   r&   rT   r   rV   rW   r   rX   r   	exp_ratioexp_kernel_sizer   r   r)   rZ   r[   r   conv_kwargsr\   c                 F   ||d}t         |           t        ||      }|xs i }||k(  xr |dk(  xr | | _        |d uxr |dkD  }|dk(  rHt	        |dz        }t        ||fdddd|| _         ||fi || _        |dz   dz  }|dk(  rdn|}|}d}nd | _        d | _        |}t        ||	z        }t        ||      }t        |||
fd|i||| _
         ||fd	d
i|| _        t        |||f|rdn||||d||| _         ||fd	d
i|| _        t        |f|||d|| _        |r ||fd|i|nt!        j"                         | _        t        |||fd|i||| _         ||fddi|| _        |rt+        |      | _        y t!        j"                         | _        y )Nr.   r    r   r@   r   r   Fr`   r2   Tr^   ra   r)   r   )r3   r4   r   rc   rM   r	   r   r   r   r#   r   re   r   r   r   rf   r6   rg   r   conv_pwlbn3r
   rh   )r<   r&   rT   r   rV   rW   r   rX   r   r   r   r   r   r)   rZ   r[   r   r   r\   r/   r0   r=   ri   rj   r   r   mid_chsr_   r>   s                               r"   r4   zInvertedResidual.__init__   s$   . /+J	B!'R7*:v{JF
%4&1* !8!_F)&&haPQ[aheghDM(626DK,q0Q6N$2a$7&XKFF DMDK"K )!34J0 %VWomxm[fmjlm!'>4>2> %

 1F

 

 

 "'>4>2>H[wvf[XZ[ CK(7>i>2>PRP[P[P] &gwnPXn\gnkmn!'AUAbA5C.1r$   c                     |dk(  r"t        dd| j                  j                        S t        d| j                  j                        S Nrl   r   r   rm   rR   rq   rr   r   r   rs   rt   s     r"   rv   zInvertedResidual.feature_info>  :    {"z]DMMLeLeffr4==+E+EFFr$   c                    |}| j                   "| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  r| j                  |      |z   }|S rx   )r   r   r   re   r   r   rf   r   r   r   rc   rh   ry   s      r"   rF   zInvertedResidual.forwardD  s    ==$a AAALLOHHQKLLOHHQKGGAJGGAJMM!HHQK==q!H,Ar$   rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   rN   r   r   r   r   r4   rv   rF   rO   rP   s   @r"   r   r      s?    #$ "#$"##%77$&NN,0-1*.$&+HWHW HW  	HW
 HW HW HW HW HW HW !HW  HW HW !HW "HW  y)!HW" z*#HW$ "$%HW& "'HWTGr$   r   c            &            e Zd ZdZdddddddddej
                  ej                  dddd	d
ddfdededededededededede	de
dededee   dee   dee   de
dee
   f$ fdZd Zd Z xZS ) r   a   Universal Inverted Residual Block (aka Universal Inverted Bottleneck, UIB)

    For MobileNetV4 - https://arxiv.org/abs/, referenced from
    https://github.com/tensorflow/models/blob/d93c7e932de27522b2fa3b115f58d06d6f640537/official/vision/modeling/layers/nn_blocks.py#L778
    r   rA   r    rR   Fr   NrS   h㈵>r&   rT   dw_kernel_size_startdw_kernel_size_middw_kernel_size_endrV   rW   r   rX   r   r   r)   rZ   r[   r   r   r\   layer_scale_init_valuec                    ||d}t         |           |xs i }||k(  xr |dk(  xr |
 | _        |dkD  r|s|s|sJ |r1|s|nd}t        ||      }t	        |||f||||	d|||d||| _        nt        j                         | _        t        ||z        }t	        ||df|	||d||| _	        |r*t        ||      }t	        |||f||||	|||d||| _
        nt        j                         | _
        |r ||fd|i|nt        j                         | _        t	        ||df|	d||d||| _        |r;|s|s|nd}t        ||      }|dkD  r|rJ t	        |||f||||	d||d	||| _        nt        j                         | _        |t        ||fi || _        nt        j                         | _        |rt!        |      | _        y t        j                         | _        y )
Nr.   r    F)rV   rW   r_   r`   r   r)   rZ   r[   )r`   r)   rZ   )rV   rW   r_   r`   r)   rZ   r[   r)   )r`   r   r)   rZ   )rV   rW   r_   r`   r   r)   rZ   )r3   r4   rc   r#   r   dw_startr6   rg   r   pw_expdw_midr   pw_projdw_endr   layer_scaler
   rh   )r<   r&   rT   r   r   r   rV   rW   r   rX   r   r   r)   rZ   r[   r   r   r\   r   r/   r0   r=   dw_start_stridedw_start_groupsr   r_   dw_end_stridedw_end_groupsr>   s                               r"   r4   z"UniversalInvertedResidual.__init__]  s   . /!'R7*:v{JF
A:'+=ASSS  ,>fAO(V<O' 4&!& #%!  DM KKMDM !)!34!GQ
!	

 
 
 
G4F%"4! #%!  DK ++-DK CK(7>i>2>PRP[P[P] #Wa
!
 
 
 *>GYF_`M&z7;Mq ##|%"4$!$ #%  DK ++-DK!-+G5KRrRD!{{}D5C.1r$   c                     |dk(  r,t        dd| j                  j                  j                        S t        d| j                  j                  j                        S )Nrl   zpw_proj.convr   rm   rR   rq   )rr   r   rd   r   rs   rt   s     r"   rv   z&UniversalInvertedResidual.feature_info  sI    {"~PTP\P\PaPaPmPmnnr4<<+<+<+I+IJJr$   c                 8   |}| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }| j                  |      }| j                  r| j                  |      |z   }|S rx   )	r   r   r   r   r   r   r   rc   rh   ry   s      r"   rF   z!UniversalInvertedResidual.forward  s    MM!KKNKKNGGAJLLOKKNQ==q!H,Ar$   r   rP   s   @r"   r   r   V  sC    )*&'&' "#%77$&NN,0-1*.$&6:+vWvW vW #&	vW
 !$vW !$vW vW vW vW vW vW vW !vW "vW y)vW  z*!vW" "$#vW$ "%vW& %-UO'vWpKr$   r   c            1            e Zd ZdZdddddddddddddej
                  ej                  d	d
d
d
dddd	d	fdedededededededededede	dededede	de
de
dee
   deded ed!ee   d"e	d#e	f0 fd$Zd% Zd& Z xZS )'r   z Mobile Attention Block

    For MobileNetV4 - https://arxiv.org/abs/, referenced from
    https://github.com/tensorflow/models/blob/d93c7e932de27522b2fa3b115f58d06d6f640537/official/vision/modeling/layers/nn_blocks.py#L1504
    r    rA   rR      @   F)r    r    NrS   r   r&   rT   rV   r   rW   r   rX   	num_headskey_dim	value_dimuse_multi_queryquery_strides	kv_stridecpe_dw_kernel_sizer   r)   rZ   r[   r\   	attn_drop	proj_dropr   use_biasuse_cpec                    ||d}t         |           t        ||      }|dk(  xr ||k(  xr | | _        t	        |      | _        || _        t        | j
                  D cg c]  }|dkD  	 c}      | _        |rt        ||f||ddd|| _
        nd | _
         ||fddi|| _        |||	z  dk(  sJ ||	z  }|rt        |f|||	|
||||||||d|| _        nt        |f|||||d	|| _        |t        ||fi || _        nt#        j$                         | _        |rt'        |      | _        y t#        j$                         | _        y c c}w )
Nr.   r    T)rU   rW   	depthwiser1   r   Fr   )dim_outr   r   r   r   r   r   rW   r`   r   r   rZ   )r   r   r   r   r1   )r3   r4   r   rc   r   r   r   anyhas_query_strider	   conv_cpe_dwnormr   attnr   r   r   r6   rg   r
   rh   )r<   r&   rT   rV   r   rW   r   rX   r   r   r   r   r   r   r   r   r)   rZ   r[   r\   r   r   r   r   r   r/   r0   r=   ri   sr>   s                                 r"   r4   zMobileAttention.__init__  s   : /+J	B1:7):JF
&}5" #D4F4F$GqQU$G H , .!   D  $D"6AUAbA	G#q(((')I-##+#-! ##% DI$ $### DI "-+G5KRrRD!{{}D5C.1w %Hs   !Ec                     |dk(  r"t        dd| j                  j                        S t        d| j                  j                        S r   r   rt   s     r"   rv   zMobileAttention.feature_infoO  r   r$   c                     | j                   | j                  |      }||z   }|}| j                  |      }| j                  |      }| j                  |      }| j                  r| j                  |      |z   }|S rx   )r   r   r   r   rc   rh   )r<   rD   x_cperz   s       r"   rF   zMobileAttention.forwardU  su    '$$Q'EE	AIIaLIIaLQ==q!H,Ar$   )rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   r   r   rN   r4   rv   rF   rO   rP   s   @r"   r   r     s    "#$)!'&' #%77$&NN,0$&""6:"!7^W^W ^W 	^W
  ^W ^W ^W ^W ^W ^W ^W "^W ^W ^W !$^W  !^W" !#^W$ "%^W& y)'^W( ")^W* +^W, -^W. %-UO/^W0 1^W2 3^W@Fr$   r   c            #            e Zd ZdZdddddddddej
                  ej                  dddd	ddfd
ededededededede	de
dededededee   dee   dede
f" fdZd Z xZS )r   z, Inverted residual block w/ CondConv routingrA   r    rR   Fr   Nr   rS   r&   rT   r   rV   rW   r   rX   r   r   r   r   r)   rZ   r[   r   num_expertsr\   c                     ||d}|| _         t        | j                         }t        |   ||f|||||||	|
|||||||d| t	        j
                  || j                   fi || _        y )Nr.   )r   )r   rV   rW   r   rX   r   r   r   r   r)   rZ   r[   r   r   r\   )r   rr   r3   r4   r6   Linear
routing_fn)r<   r&   rT   r   rV   rW   r   rX   r   r   r   r   r)   rZ   r[   r   r   r\   r/   r0   r=   r   r>   s                         r"   r4   zCondConvResidual.__init__g  s    , /&t'7'78	
 *!+)!#)#	
$ %	
( ))FD,<,<CCr$   c                    |}t        j                  |d      j                  d      }t        j                  | j                  |            }| j                  ||      }| j                  |      }| j                  ||      }| j                  |      }| j                  |      }| j                  ||      }| j                  |      }| j                  r| j                  |      |z   }|S )Nr    )Fadaptive_avg_pool2dflattentorchsigmoidr   r   re   r   r   r   r   r   rc   rh   )r<   rD   rz   pooled_inputsrouting_weightss        r"   rF   zCondConvResidual.forward  s    --a3;;A>--(FGLLO,HHQKLLO,HHQKGGAJMM!_-HHQK==q!H,Ar$   )rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   rN   r   r   r   r4   rF   rO   rP   s   @r"   r   r   d  s   6 #$ "#$"##%77$&NN,0-1 $&)-D-D -D  	-D
 -D -D -D -D -D -D !-D  -D !-D "-D y)-D  z*!-D" #-D$ "%-D^r$   r   c            !            e Zd ZdZdddddddddej
                  ej                  ddd	ddfd
edededededededede	de
dedededee   dee   de
f  fdZd Zd Z xZS )r   a(   Residual block with expansion convolution followed by pointwise-linear w/ stride

    Originally introduced in `EfficientNet-EdgeTPU: Creating Accelerator-Optimized Neural Networks with AutoML`
        - https://ai.googleblog.com/2019/08/efficientnet-edgetpu-creating.html

    This layer is also called FusedMBConv in the MobileDet, EfficientNet-X, and EfficientNet-V2 papers
      * MobileDet - https://arxiv.org/abs/2004.14525
      * EfficientNet-X - https://arxiv.org/abs/2102.05610
      * EfficientNet-V2 - https://arxiv.org/abs/2104.00298
    rA   r    r   rR   Fr   NrS   r&   rT   r   rV   rW   r   rX   force_in_chsr   r   r   r)   rZ   r[   r   r\   c                 R   ||d}t         |           t        ||      }|dkD  rt        ||
z        }nt        ||
z        }t	        ||      }||k(  xr |dk(  xr |	 | _        |d uxr |dkD  }t        |||f|rdn||||d|| _         ||fddi|| _        t        |f|||d|| _
        |r ||fd|i|nt        j                         | _        t        |||fd	|i|| _         ||fd
di|| _        |rt!        |      | _        y t        j                         | _        y )Nr.   r   r    r^   r2   Tra   r)   r`   r   F)r3   r4   r   r   r#   rc   r	   conv_expre   r   rf   r6   rg   r   r   r   r
   rh   )r<   r&   rT   r   rV   rW   r   rX   r   r   r   r   r)   rZ   r[   r   r\   r/   r0   r=   ri   r   r_   rj   r>   s                           r"   r4   zEdgeResidual.__init__  sh   * /+J	B!$\I%=>G$Vi%78GJ07*:v{JF
%4&1* &	
 1F	
 	
 "'>4>2>H[wvf[XZ[ CK(7>i>2>PRP[P[P] &gw_PX_\^_!'AUAbA5C.1r$   c                     |dk(  r"t        dd| j                  j                        S t        d| j                  j                        S r   r   rt   s     r"   rv   zEdgeResidual.feature_info  r   r$   c                    |}| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }| j                  r| j                  |      |z   }|S rx   )r   re   rf   r   r   r   rc   rh   ry   s      r"   rF   zEdgeResidual.forward  sv    MM!HHQKGGAJGGAJMM!HHQK==q!H,Ar$   )rG   rH   rI   rJ   r6   rK   r{   rM   r|   r}   rN   r   r   r   r4   rv   rF   rO   rP   s   @r"   r   r     s   	 $% ! "#$#%77$&NN,0-1$&'5W5W 5W !	5W
 5W 5W 5W 5W 5W 5W 5W !5W !5W "5W y)5W  z*!5W" "#5WnG
r$   r   )&rJ   typingr   r   r   r   r   r   torch.nnr6   r   r   timm.layersr	   r
   r   r   r   r   r   r   r   r   r   r   __all__Moduler   rM   r#   r   r   r   r   r   r   r   r   r!   r$   r"   <module>r      s    9 8   $   
 "))_
&8C= &C &(#BII (#V6		 6rXRYY Xviryy iXP		 Pfxbii xv?' ?DS299 Sr$   