
    ^jh                         d Z ddlmZmZmZ ddlZddlmc mZ	 ddlmZ ddl
mZ  G d dej                        Z G d	 d
ej                        Zy)a'   Split Attention Conv2d (for ResNeSt Models)

Paper: `ResNeSt: Split-Attention Networks` - /https://arxiv.org/abs/2004.08955

Adapted from original PyTorch impl at https://github.com/zhanghang1989/ResNeSt

Modified for torchscript compat, performance, and consistency with timm by Ross Wightman
    )OptionalTypeUnionN)nn   )make_divisiblec                   .     e Zd Zdedef fdZd Z xZS )RadixSoftmaxradixcardinalityc                 >    t         |           || _        || _        y )N)super__init__r   r   )selfr   r   	__class__s      a/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/split_attn.pyr   zRadixSoftmax.__init__   s    
&    c                 6   |j                  d      }| j                  dkD  rc|j                  || j                  | j                  d      j	                  dd      }t        j                  |d      }|j                  |d      }|S t        j                  |      }|S )Nr   r      dim)
sizer   viewr   	transposeFsoftmaxreshapetorchsigmoid)r   xbatchs      r   forwardzRadixSoftmax.forward   s    q	::>ud..

B?II!QOA		!#A		%$A  a Ar   )__name__
__module____qualname__intr   r#   __classcell__r   s   @r   r
   r
      s    'c ' '
r   r
   c            !           e Zd ZdZdddddddddddej
                  ddfd	ed
ee   dededee   dedededede	dee   dede
ej                     dee
ej                        dee
ej                        f fdZd Z xZS )	SplitAttnz Split-Attention (aka Splat)
    N   r   Fr   g      ?   in_channelsout_channelskernel_sizestridepaddingdilationgroupsbiasr   rd_ratiord_channels
rd_divisor	act_layer
norm_layer
drop_layerc                    |j                  dd       |j                  dd       d}t        | 	          |xs |}|	| _        ||	z  }|t	        ||	z  |
z  d|      }n||	z  }||dz  n|}t        j                  ||||||f||	z  |d||| _        |r	 ||fi |nt        j                         | _	        | |       nt        j                         | _
         |d	      | _        t        j                  ||d
fd|i|| _        |r	 ||fi |nt        j                         | _         |d	      | _        t        j                  ||d
fd|i|| _        t!        |	|      | _        y )Ndevicedtype)r=   r>       )	min_valuedivisorr   )r4   r5   T)inplacer   r4   )popr   r   r   r   r   Conv2dconvIdentitybn0dropact0fc1bn1act1fc2r
   rsoftmax)r   r.   r/   r0   r1   r2   r3   r4   r5   r   r6   r7   r8   r9   r:   r;   kwargsddmid_chsattn_chsr   s                       r   r   zSplitAttn.__init__&   s}   & 

8T2VZZQU=VW#2{
&%kE&9H&DPR\fgH"U*H&-o+"7II
 E>
 
 
	 1;:g,,$.$:JL	d+	99\8QLvLL1;:h-"-d+	99XwG&GBG$UF3r   c                 *   | j                  |      }| j                  |      }| j                  |      }| j                  |      }|j                  \  }}}}| j
                  dkD  r@|j                  || j
                  || j
                  z  ||f      }|j                  d      }n|}|j                  dd      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      j                  |ddd      }| j
                  dkD  rP||j                  || j
                  || j
                  z  ddf      z  j                  d      }|j                         S ||z  }|j                         S )Nr   r   )r   r,   T)keepdimr   )rE   rG   rH   rI   shaper   r   summeanrJ   rK   rL   rM   rN   r   
contiguous)	r   r!   BRCHWx_gapx_attnouts	            r   r#   zSplitAttn.forwardY   s^   IIaLHHQKIIaLIIaLgg2q!::>		1djj"

*:AqABAEEaELEE

64
0		% %v&++Ar1a8::>v~~q$**bDJJ6F1&MNNSSXYSZC ~~ f*C~~r   )r$   r%   r&   __doc__r   ReLUr'   r   boolfloatr   Moduler   r#   r(   r)   s   @r   r+   r+   #   s   
 +/ %)")-)+4848!1414 #3-14 	14
 14 c]14 14 14 14 14 14 "#14 14 BII14 !bii114  !bii1!14f r   r+   )r`   typingr   r   r   r   torch.nn.functionalr   
functionalr   helpersr   rd   r
   r+    r   r   <module>rj      sC    ) (     #299 "M 		 M r   