
    ^j/2                         d Z ddlmZmZmZ ddlZddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZmZ dd	lmZ  G d
 dej&                        Z G d dej&                        Zy)a:   Attention Pool 2D

Implementations of 2D spatial feature pooling using multi-head attention instead of average pool.

Based on idea in CLIP by OpenAI, licensed Apache 2.0
https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

Hacked together by / Copyright 2021 Ross Wightman
    )OptionalUnionTupleN   )use_fused_attn)	to_2tuple)resample_abs_pos_embed)apply_rot_embed_catcreate_rope_embed)trunc_normal_c                   P    e Zd ZU dZej
                  j                  e   ed<   	 	 	 	 	 	 	 	 	 	 	 	 	 dde	de
e	   dee	ee	e	f   f   de
e	   de
e	   de
e	   d	ed
ededededef fdZddefdZdde
e	   de
e   fdZdej$                  de	de	dej$                  fdZddefdZ xZS )RotAttentionPool2da   Attention based 2D feature pooling w/ rotary (relative) pos embedding.
    This is a multi-head attention based replacement for (spatial) average pooling in NN architectures.

    Adapted from the AttentionPool2d in CLIP w/ rotary embedding instead of learned embed.
    https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

    NOTE: While this impl does not require a fixed feature size, performance at differeing resolutions from
    train varies widely and falls off dramatically. I'm not sure if there is a way around this... -RW

    Setting out_features=0 disables the output projection (pre_logits mode).
    
fused_attnin_featuresout_featuresref_feat_size	embed_dimhead_dim	num_headsqkv_biasqkv_separate	pool_typeclass_token	drop_rate	rope_typec           
         ||d}t         |           |	dv sJ |xs |x| _        }|| _        ||| _        n|dkD  r|| _        n|| _        t        |      }|||z  dk(  sJ ||z  }n||z  dk(  sJ ||z  }|| _        || _        |	j                         | _	        | j                  dz  | _
        t               | _        || _        |
r0t        j                  t!        j"                  d|fi |      | _        nd | _        |rbt        j&                  ||fd|i|| _        t        j&                  ||fd|i|| _        t        j&                  ||fd|i|| _        d | _        n!t        j&                  ||dz  fd|i|| _        t        j0                  |      | _        |dk7  r!t        j&                  || j                  fi |nt        j4                         | _        t9        d
|||d|dd	|| _        y )Ndevicedtype tokenr         r   bias   F)r   dimr   	in_pixelsref_feat_shaperotate_half )super__init__r   r   r   r   r   r   lowerr   scaler   r   r   nn	Parametertorchzeros	cls_tokenLinearqkvqkvDropoutdropIdentityprojr   	pos_embed)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   dd	__class__s                   g/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/attention_pool2d.pyr,   zRotAttentionPool2d.__init__$   s   " /M)))%.%=+=& +DA ,D )D!-0 y(A--- I-Hx'1,,,!X-I" "*]]d*
(*"\\%++a*Ib*IJDN!DNYY{IKHKKDFYY{IKHKKDFYY{IKHKKDFDHyyi!mQ(QbQDHJJy)	EQUVEVBIIi):):AbA\^\g\g\i	* 
(
 
    zero_init_lastc                 0   | j                   | j                  j                  }t        | j                  j                  |dz         t
        j                  j                  | j                  j                         t        | j                  j                  |dz         t
        j                  j                  | j                  j                         t        | j                  j                  |dz         t
        j                  j                  | j                  j                         y | j                   j                  }t        | j                   j                  |dz         t
        j                  j                  | j                   j                         y Nr#   )std)r8   r5   r   r   weightr/   initzeros_r$   r6   r7   r>   rC   r   s      rA   init_weightszRotAttentionPool2d.init_weightsg   s    88&&,,K$&&--[D-@AGGNN466;;'$&&--[D-@AGGNN466;;'$&&--[D-@AGGNN466;;'((..K$((//{d/BCGGNN488==)rB   num_classesc                     ||dv sJ || _         |W|dkD  r t        j                  | j                  |      nt        j                         | _        |dkD  r|n| j                  | _        y y Nr    r   r   r/   r4   r   r;   r<   r   r>   rL   r   s      rA   resetzRotAttentionPool2d.resetu   e     ---&DN"BMPQ/		$..+>WYWbWbWdDI/:QDNND #rB   xHWreturnc                     | j                   dk(  r|d d df   }|S |d d dd f   j                  |j                  d   ||d      j                  dddd      }|S Nr"   r   r   r%      r   reshapeshapepermuter>   rS   rT   rU   s       rA   _poolzRotAttentionPool2d._pool~   b    >>W$!Q$A  !QR%  Q26>>q!QJArB   
pre_logitsc                 ~   |j                   \  }}}}||z  }|j                  d      j                  dd      }| j                  +t	        j
                  |j                  dd      |gd      }nAt	        j
                  | j                  j                  |j                   d   dd      |gd      }| j                  | j                  |      j                  ||dz   | j                  | j                        j                  dd      }| j                  |      j                  ||dz   | j                  | j                        j                  dd      }	| j                  |      j                  ||dz   | j                  | j                        j                  dd      }
nc| j                  |      j                  ||dz   d| j                  | j                        j                  ddddd	      }|j!                  d      \  }}	}
| j"                  j%                  ||f      }t'        |t(              rt	        j
                  |d      }t	        j
                  |d d d d d dd d f   t+        |d d d d dd d d f   |      gd      j-                  |
      }t	        j
                  |	d d d d d dd d f   t+        |	d d d d dd d d f   |      gd      j-                  |
      }	| j.                  r"t0        j2                  j5                  ||	|
      }n;|| j6                  z  }||	j                  d
d      z  }|j9                  d      }||
z  }|j                  dd      j                  ||dz   d      }| j;                  |      }|r| j=                  |||      }|S | j?                  |      }| j=                  |||      }|S )NrZ   r   Tkeepdimr&   r   rY   r%      ) r]   flatten	transposer3   r1   catmeanexpandr8   r5   r\   r   r   r6   r7   r^   unbindr=   	get_embed
isinstancetupler
   type_asr   r/   
functionalscaled_dot_product_attentionr.   softmaxr:   r`   r<   )r>   rS   rb   B_rT   rU   Nr5   r6   r7   ropeattns                rA   forwardzRotAttentionPool2d.forward   s(   WW
1aEIIaL""1a(>>!		166!T62A6A>A		4>>00RDaHaPA88q	!!!QUDNNDMMJTTUVXYZAq	!!!QUDNNDMMJTTUVXYZAq	!!!QUDNNDMMJTTUVXYZA##Aq1uaOWWXY[\^_abdefAhhqkGAq!~~''A/dE"99Tr*DIIqArr1~':1Q12q[>4'PQWXYaabcdIIqArr1~':1Q12q[>4'PQWXYaabcd??::1aCADJJAq{{2r**D<<B<'DqAKK1%%aQ3IIaL

1a#AHIIaLJJq!QrB   )N   N@   NTFr"   F        rk   NNFNN)__name__
__module____qualname____doc__r1   jitFinalbool__annotations__intr   r   r   strfloatr,   rK   rQ   Tensorr`   r{   __classcell__r@   s   @rA   r   r      sP   
 		%%
 +/9:'+&('+!!&$ %!"A
A
 #3-A
 !eCHo!56	A

  }A
 smA
  }A
 A
 A
 A
 A
 A
 A
F*4 *S# S(3- Su||    %T %rB   r   c                   J    e Zd ZU dZej
                  j                  e   ed<   	 	 	 	 	 	 	 	 	 	 	 	 dde	de
e	ee	e	f   f   dee	   dee	   dee	   dee	   d	ed
edededef fdZddefdZddee	   dee   fdZdej$                  de	de	dej$                  fdZddefdZ xZS )AttentionPool2da   Attention based 2D feature pooling w/ learned (absolute) pos embedding.
    This is a multi-head attention based replacement for (spatial) average pooling in NN architectures.

    It was based on impl in CLIP by OpenAI
    https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

    NOTE: This requires feature size upon construction and well prevent adaptive sizing of the network.

    Setting out_features=0 disables the output projection (pre_logits mode).
    r   r   	feat_sizer   r   r   r   r   r   r   r   r   c                    ||d}t         |           |	dv sJ |xs |x| _        }|| _        ||| _        n|dkD  r|| _        n|| _        |||z  dk(  sJ ||z  }n||z  dk(  sJ ||z  }t        |      | _        | j                  d   | j                  d   z  | _        || _        || _	        |	| _
        | j                  dz  | _        t               | _        |
r0t        j                  t!        j"                  d|fi |      | _        nd | _        |rbt        j&                  ||fd|i|| _        t        j&                  ||fd|i|| _        t        j&                  ||fd|i|| _        d | _        n6d x| _        x| _        | _        t        j&                  ||dz  fd|i|| _        t        j0                  |      | _        |dk7  r!t        j&                  || j                  fi |nt        j4                         | _        t        j                  t!        j"                  | j                  dz   |fi |      | _        | j;                          y )Nr   r    r   r   r#   r$   r%   )r+   r,   r   r   r   r   r   seq_lenr   r   r   r.   r   r   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   rK   )r>   r   r   r   r   r   r   r   r   r   r   r   r   r   r?   r@   s                  rA   r,   zAttentionPool2d.__init__   s4     /M)))%.%=+=& +DA ,D )D y(A--- I-Hx'1,,,!X-I"9-~~a(4>>!+<<" "]]d*
(*\\%++a*Ib*IJDN!DNYY{IKHKKDFYY{IKHKKDFYY{IKHKKDFDH'++DF+TVdfyyi!mQ(QbQDHJJy)	EQUVEVBIIi):):AbA\^\g\g\i	ekk$,,2BK&VSU&VWrB   rC   c                 d   | j                   | j                  j                  }t        | j                  j                  |dz         t
        j                  j                  | j                  j                         t        | j                  j                  |dz         t
        j                  j                  | j                  j                         t        | j                  j                  |dz         t
        j                  j                  | j                  j                         nm| j                   j                  }t        | j                   j                  |dz         t
        j                  j                  | j                   j                         t        | j                  |dz         y rE   )r8   r5   r   r   rG   r/   rH   rI   r$   r6   r7   r=   rJ   s      rA   rK   zAttentionPool2d.init_weights   s    88&&,,K$&&--[D-@AGGNN466;;'$&&--[D-@AGGNN466;;'$&&--[D-@AGGNN466;;'((..K$((//{d/BCGGNN488==)dnn+*=>rB   rL   c                     ||dv sJ || _         |W|dkD  r t        j                  | j                  |      nt        j                         | _        |dkD  r|n| j                  | _        y y rN   rO   rP   s      rA   rQ   zAttentionPool2d.reset  rR   rB   rS   rT   rU   rV   c                     | j                   dk(  r|d d df   }|S |d d dd f   j                  |j                  d   ||d      j                  dddd      }|S rX   r[   r_   s       rA   r`   zAttentionPool2d._pool  ra   rB   rb   c                    |j                   \  }}}}||z  }|j                  d      j                  dd      }| j                  +t	        j
                  |j                  dd      |gd      }nAt	        j
                  | j                  j                  |j                   d   dd      |gd      }t        | j                  j                  d      ||fd      }||z   }| j                  | j                  |      j                  ||dz   | j                  | j                        j                  dd      }	| j!                  |      j                  ||dz   | j                  | j                        j                  dd      }
| j#                  |      j                  ||dz   | j                  | j                        j                  dd      }n`| j                  |      j                  |dd	| j                  | j                        j%                  ddd	dd
      }|j'                  d      \  }	}
}| j(                  r"t*        j,                  j/                  |	|
|      }n;|	| j0                  z  }	|	|
j                  dd      z  }|j3                  d      }||z  }|j                  dd      j                  ||dz   d      }| j5                  |      }|r| j7                  |||      }|S | j9                  |      }| j7                  |||      }|S )NrZ   r   Trd   rf   r   rY   )num_prefix_tokensr%   rg   rh   )r]   ri   rj   r3   r1   rk   rl   rm   r	   r=   	unsqueezer8   r5   r\   r   r   r6   r7   r^   rn   r   r/   rs   rt   r.   ru   r:   r`   r<   )r>   rS   rb   rv   rw   rT   rU   rx   r=   r5   r6   r7   rz   s                rA   r{   zAttentionPool2d.forward  s   WW
1aEIIaL""1a(>>!		166!T62A6A>A		4>>00RDaHaPA*4>>+C+CA+FAbcd		M88q	!!!QUDNNDMMJTTUVXYZAq	!!!QUDNNDMMJTTUVXYZAq	!!!QUDNNDMMJTTUVXYZA##Ar1dnndmmLTTUVXY[\^_abcAhhqkGAq!??::1aCADJJAq{{2r**D<<B<'DqAKK1%%aQ3IIaL

1a#AHIIaLJJq!QrB   )r|   NNr}   NTFr"   Fr~   NNr   r   )r   r   r   r   r1   r   r   r   r   r   r   r   r   r   r   r,   rK   rQ   r   r`   r{   r   r   s   @rA   r   r      s6   	 		%%
 67*.'+&('+!!&$ %!:: S%S/12: #3-	:
  }: sm:  }: : : : : :x?4 ?S# S(3- Su||    !T !rB   r   )r   typingr   r   r   r1   torch.nnr/   configr   helpersr   r=   r	   pos_embed_sincosr
   r   weight_initr   Moduler   r   r*   rB   rA   <module>r      sL    * )   "  - D &U UpJbii JrB   