
    ^j5                         d dl mZmZmZmZ d dlZd dlmZ d dlmZ	 ddl
mZ ddlmZ ddlmZ dd	lmZ  G d
 dej"                        Z G d dej"                        Z G d dej"                        Zy)    )ListOptionalTypeUnionN)nn)
functional   )use_fused_attn)create_conv2d)	to_2tuple)create_pool2dc                        e Zd ZdZ	 	 	 	 	 	 	 	 ddedee   dededededef fd	Zd
 Zd Z	ddee
j                     fdZ xZS )MultiQueryAttentionV2a  Multi Query Attention.

    Fast Transformer Decoding: One Write-Head is All You Need
    https://arxiv.org/pdf/1911.02150.pdf

    This is an acceletor optimized version - removing multiple unnecessary
    tensor transpose by re-arranging indices according to the following rules: 1)
    contracted indices are at the end, 2) other indices have the same order in the
    input and output tensores.

    Compared to V1, this gives 3x speed up.
    dimdim_out	num_headskey_dim	value_dim	attn_drop	proj_dropc
                    ||	d}
t         |           |xs |}|| _        || _        || _        |dz  | _        t        j                  t        j                  | j                  | j                  |ffi |
      | _
        t        j                  t        j                  || j                  ffi |
      | _        t        j                  t        j                  || j                  ffi |
      | _        t        j                  |      | _        t        j                  t        j                  || j                  | j                  ffi |
      | _        t        j                  |      | _        | j#                          y)zInitializer.devicedtype      N)super__init__r   r   r   scaler   	Parametertorchempty
query_projkey_proj
value_projDropoutr   out_projr   reset_parameters)selfr   r   r   r   r   r   r   r   r   dd	__class__s              b/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/attention2d.pyr   zMultiQueryAttentionV2.__init__   s    /.S""_
,,u{{DNNDLLRU3V']Z\']^U[[#t||1D%K%KL,,u{{C3H'OB'OPI.U[['4>>4>>1Z%a^`%abI.    c                    | j                   j                  d   dz  }t        j                  j	                  | j
                  |       t        j                  j	                  | j                   |       t        j                  j	                  | j                  |       t        j                  j	                  | j                  | j                  j                  d   dz         y )Nr   r   )std)r#   shaper   initnormal_r"   r$   r&   )r(   r   s     r+   r'   z&MultiQueryAttentionV2.reset_parameters9   s    ##A&$.
U3
51
U3
4==+>+>q+AT+IJr,   c                 l    |j                   }|j                  |d   |d   d      j                  dd      S )zBReshapes a tensor to three dimensions, keeping the first and last.r   r	      )r/   reshape	transposer(   tss      r+   _reshape_inputz$MultiQueryAttentionV2._reshape_input@   s5    GG yy1qtR(221a88r,   mc                    |j                   \  }}}}||n|}| j                  |      }| j                  |      }t        j                  d|| j                        }	t        j                  d|| j
                        }
t        j                  d|	|
      | j                  z  }|j                  d      }| j                  |      }t        j                  d|| j                        }t        j                  d||      }t        j                  d|| j                        }| j                  |      }|j                  |d||      S )	Run layer computation.zbnd,hkd->bnhkzbmd,dk->bmkzbnhk,bmk->bnhmr3   r   zbmd,dv->bmvzbnhm,bmv->bnhvzbnhv,dhv->bdn)r/   r:   r    einsumr"   r#   r   softmaxr   r$   r&   r   r5   )r(   xr;   b_hw
reshaped_x
reshaped_mqkattnvoresults                  r+   forwardzMultiQueryAttentionV2.forwardH   s   WW
1aAA((+
((+
LL*dooFLL
DMMB||,a3djj@|||#~~d#LL
DOODLL)43oq$--@'~~aQ**r,   )N   @   rP           rQ   NNN)__name__
__module____qualname____doc__intr   floatr   r'   r:   r    TensorrN   __classcell__r*   s   @r+   r   r      s      &*!!   c]  	 
        <K9+HU\\2 +r,   r   c                       e Zd ZU dZej
                  j                  e   ed<   ddddddddddde	j                  d	ddfd
edee   dedee   dee   dededededeeeee   f   dededee	j$                     def fdZd Zdej*                  fdZdej*                  dedefdZdej*                  dededefdZd"d eej*                     fd!Z xZS )#MultiQueryAttention2da  Multi Query Attention with spatial downsampling.

     3 parameters are introduced for the spatial downsampling:
     1. kv_stride: downsampling factor on Key and Values only.
     2. query_strides: horizontal & vertical strides on Query only.

    This is an optimized version.
    1. Projections in Attention is explicit written out as 1x1 Conv2D.
    2. Additional reshapes are introduced to bring a up to 3x speed up.
    
fused_attnNrO   r	       rQ   Fr   r   r   r   r   query_strides	kv_stridedw_kernel_sizedilationpaddingr   r   
norm_layeruse_biasc                    ||d}t         |           |xs |}|| _        |xs ||z  | _        |xs ||z  | _        t        |      | _        || _        t        | j                  D cg c]  }|dkD  	 c}      | _	        | j                  dz  | _
        t               | _        || _        t        j                         | _        | j                  r|
dk(  r3| j                   j#                  dt%        d| j                  d             n0| j                   j#                  dt        j&                  |             | j                   j#                  d	 ||fi |       | j                   j#                  d
t)        || j                  | j                  z  fd|d|       t        j                         | _        |dkD  rP| j*                  j#                  dt)        ||f|||	|
dd|       | j*                  j#                  d	 ||fi |       | j*                  j#                  d
t)        || j                  fd|
|d|       t        j                         | _        |dkD  rP| j,                  j#                  dt)        ||f|||	|
dd|       | j,                  j#                  d	 ||fi |       | j,                  j#                  d
t)        || j                  fd|d|       t        j.                  |      | _        t        j                         | _        | j                  r<| j2                  j#                  dt        j4                  | j                  dd             | j2                  j#                  d
t)        | j                  | j                  z  |fd|d|       | j2                  j#                  dt        j.                  |             d| _        | j9                          yc c}w )a{  Initializer.

        Args:
          num_heads: Number of attention heads.
          key_dim: Size of the attention key dimension.
          value_dim: Size of the attention value dimension.
          query_strides: Vertical stride size for query only.
          kv_stride: Key and value stride size.
          dw_kernel_size: Spatial dimension of the depthwise kernel.
        r   r	   r   same	down_poolavg)kernel_sizere   )rl   normproj)rl   bias	down_convT)rl   striderd   re   	depthwise)rl   re   ro   upsamplebilinearF)scale_factormodealign_cornersdropN)r   r   r   r   r   r   ra   rb   anyhas_query_stridesr   r
   r^   rx   r   
Sequentialquery
add_moduler   	AvgPool2dr   keyvaluer%   r   outputUpsampler?   init_weights)r(   r   r   r   r   r   ra   rb   rc   rd   re   r   r   rf   rg   r   r   r)   r9   r*   s                      r+   r   zMultiQueryAttention2d.__init__k   s   : /.S"2#"2"6cY&6&}5"!$T5G5G%Ha!e%H!I\\T)
(*	]]_
!!& 

%%k= $ 2 2"4  

%%k2<<M3Z[JJ!!&*S*?B*?@

fmNNT\\)'
 	'

 '
 	 ==?q=HH]	. + !	. 	. 	 HH
3(="(=>FMLL%
 %
 %
 	 ]]_
q=JJ!!+}	0 + !	0 	0 	 JJ!!&*S*?B*?@

fmNN'
 	'

 '
 	 I.mmo!!KK"":r{{!//#0 
 	v}NNT^^+(
 	(

 (
 	 	vrzz)'<=q &Is   (O#c                    t         j                  j                  | j                  j                  j
                         t         j                  j                  | j                  j                  j
                         t         j                  j                  | j                  j                  j
                         | j                  dkD  rzt         j                  j                  | j                  j                  j
                         t         j                  j                  | j                  j                  j
                         t         j                  j                  | j                  j                  j
                         y )Nr	   )r   r0   xavier_uniform_r|   rn   weightr   r   rb   rp   r   )r(   s    r+   r   z"MultiQueryAttention2d.init_weights   s    


 6 67
 4 45


 6 67>>AGG##DHH$6$6$=$=>GG##DJJ$8$8$?$?@
 0 0 7 78r,   r8   c                     |j                   }|j                  |d   |d   d      j                  dd      }| j                  r|S |j	                  d      j                         S )zFReshapes a tensor to three dimensions, keeping the batch and channels.r   r	   r3   r4   )r/   r5   r6   r?   	unsqueeze
contiguousr7   s      r+   r:   z$MultiQueryAttention2d._reshape_input   sU    GGIIadAaD"%//15;;H;;q>,,..r,   c                     |j                   }|j                  |d   ||d      }| j                  r"|j                  dddd      j	                         S |j                  dd      j	                         S )z?Reshapes projected query: [b, n, n, h x k] -> [b, n x n, h, k].r   r3   r_   r	   r4   )r/   r5   r?   permuter   r6   )r(   r8   r   r   r9   s        r+   _reshape_projected_queryz.MultiQueryAttention2d._reshape_projected_query   sb    GGIIadIw3;;99Q1a(3355;;r2&1133r,   h_pxw_pxc                     |j                   }|d   |z  }| j                  s|j                  dd      }|j                  |d   |||      j	                  dddd      j                         S )z2Reshape output:[b, n x n x h, k] -> [b, n, n, hk].r3   r	   r4   r   r_   )r/   r?   r6   r5   r   r   )r(   r8   r   r   r   r9   feat_dims          r+   _reshape_outputz%MultiQueryAttention2d._reshape_output  sc    GGR59${{Aq!Ayy1tT84<<Q1aHSSUUr,   	attn_maskc                    |j                   x\  }}}}}| j                  |      }| j                  || j                  | j                        }| j                  |      }	| j                  |	      }	| j                  |      }
| j                  |
      }
| j                  rft        j                  d||	      | j                  z  }|||z   }|j                  d      }| j                  |      }t        j                  d||
      }n| j                  r=t        j                  ||	|
|| j                   r| j                  j"                  nd      }nS|| j                  z  }||	j%                  dd      z  }|||z   }|j                  d      }| j                  |      }||
z  }| j'                  || j                  || j(                  d   z  || j(                  d	   z        }| j+                  |      }|S )
r=   zblhk,bpk->blhpr3   r>   zblhp,bpk->blhkrQ   r   	dropout_pr   r   r	   )r/   r|   r   r   r   r   r:   r   r?   r    r   r@   r   r^   Fscaled_dot_product_attentiontrainingpr6   r   ra   r   )r(   rA   r   BCHWr9   rH   rI   rK   rJ   rL   s                r+   rN   zMultiQueryAttention2d.forward  s    
1aQJJqM))!T^^T\\JHHQK"JJqM"
 ;;<< 0!Q7$**DD$i'<<B<'D>>$'D-tQ7A22q!'26--dnn..R 

N1;;r2..()+D|||+~~d+1H   DNNA9K9KA9N4NPQUYUgUghiUjPjkKKNr,   rR   )rS   rT   rU   rV   r    jitFinalbool__annotations__r   BatchNorm2drW   r   r   strr   rX   r   Moduler   r   rY   r:   r   r   rN   rZ   r[   s   @r+   r]   r]   ^   s   	 		%%
 &*%)'+!""#24!!*,.."#}} c]} 	}
 c]}  }} } }  } } 3T#Y./} } } RYY} }~9/ /4%,, 43 4QT 4V V# VS VPS V/HU\\$: /r,   r]   c                        e Zd ZU ej                  j
                  e   ed<   	 	 	 	 	 	 	 	 	 	 ddede	e   dedededede
d	e
f fd
Zdde	ej                     fdZ xZS )Attention2dr^   r   r   r   ro   expand_first
head_firstr   r   c                 |   |	|
d}t         |           |xs |}|r|n|}|| _        ||z  | _        || _        t               | _        t        j                  ||dz  dfd|i|| _	        t        j                  |      | _        t        j                  ||dfd|i|| _        t        j                  |      | _        y )Nr   r_   r	   ro   )r   r   r   dim_headr   r
   r^   r   Conv2dqkvr%   r   rn   r   )r(   r   r   r   ro   r   r   r   r   r   r   r)   dim_attnr*   s                r+   r   zAttention2d.__init__D  s     /.S*7" I-$(*99S(Q,CCCI.IIhDDD	I.r,   r   c                    |j                   \  }}}}| j                  rP| j                  |      j                  || j                  | j
                  dz  d      j                  dd      \  }}}	nK| j                  |      j                  |d| j                  | j
                  d      j                  d      \  }}}	| j                  rt        j                  j                  j                  |j                  dd      j                         |j                  dd      j                         |	j                  dd      j                         || j                   r| j"                  j$                  nd      j                  dd      j                  |d||      }n|j                  dd      }|	j                  dd      }	||z  |j'                  d      d	z  z  }
||
|z   }
|
j)                  d      }
| j#                  |
      }
|
|	z  j                  dd      j                  |d||      }| j+                  |      }| j-                  |      }|S )
Nr_   r3   r4   r>   r	   r   rQ   r   r   )r/   r   r   viewr   r   chunkr5   unbindr^   r    r   r   r   r6   r   r   r   r   sizer@   rn   r   )r(   rA   r   r   r   r   r   rH   rI   rK   rJ   s              r+   rN   zAttention2d.forward_  s   WW
1a??hhqk&&q$..$--!:KRPVVWX^_V`GAq!hhqk))!QrRYYZ[\GAq!??##@@B#..0B#..0B#..0#.2mm$..** A  iB2q! 4  B#AB#Aq5166":--D$i'<<B<'D>>$'D$$R,44QAqAAIIaLNN1r,   )	N    TFFrQ   rQ   NNrR   )rS   rT   rU   r    r   r   r   r   rW   r   rX   r   rY   rN   rZ   r[   s   @r+   r   r   @  s    		%%3 &*!&$!!// c]/ 	/
 / / / / /6HU\\$: r,   r   )typingr   r   r   r   r    r   torch.nnr   r   configr
   r   helpersr   pool2d_samer   r   r   r]   r    r,   r+   <module>r      sV    . .   $ " (  &N+BII N+b_BII _D<")) <r,   