
    ^j                         d dl mZmZ d dlZd dlmZ d dlmc mZ ddl	m
Z
 ddlmZ ddlmZ ddlmZ  G d d	ej"                        Z G d
 dej"                        Zy)    )OptionalTypeN   )maybe_add_mask)use_fused_attn)Mlp)trunc_normal_tf_c            !       J    e Zd ZU dZej
                  j                  e   ed<   ddddddddddd	d
de	j                  dddfdededededee   dededededededededeee	j                         deee	j                         def  fdZd Zd deej&                     fdZ xZS )!AttentionPoolLatentz Attention pooling w/ latent query

    Setting out_features=0 disables the output projection, norm, and MLP layers (pre_logits mode).
    
fused_attnN   g      @TFr    tokeng        in_featuresout_features	embed_dim	num_heads	feat_size	mlp_ratioqkv_biasq_projqk_norm
latent_len
latent_dim	pos_embed	pool_type
norm_layer	act_layerdropc                    ||d}t         |           |xs |}||}||z  dk(  sJ || _        ||z  | _        || _        | j                  dz  | _        || _        t               | _        |dk(  r4|J t        j                  t        j                  ||fi |      | _        nd | _        |xs || _        |s'| j                  |k(  sJ d| j                   d| d       |
| _        t        j                  t        j                  d| j                  | j                  fi |      | _        |r#t        j"                  | j                  |fd	|i|nt        j$                         | _        t        j"                  ||d
z  fd	|i|| _        |	rE|xs t        j*                  } || j                  fi || _         || j                  fi || _        n2t        j$                         | _        t        j$                         | _        |dkD  r|t        j"                  ||fi || _        t        j2                  |      | _        |	 ||fi |nt        j$                         | _        t9        |t;        ||z        f||d|| _        nUt        j$                         | _        t        j2                  |      | _        t        j$                         | _        d | _        |}|| _        | jA                          y )Ndevicedtyper         abszdq_proj=False uses the latent directly as the query; latent_dim must equal embed_dim (got latent_dim=z, embed_dim=z).r   bias   )r   r   )!super__init__r   head_dimr   scalepoolr   r   nn	Parametertorchzerosr   r   r   latentLinearIdentityqkv	LayerNormq_normk_normprojDropout	proj_dropnormr   intmlpr   init_weights)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r"   r#   ddqk_norm_layer	__class__s                        e/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/attention_pool.pyr)   zAttentionPoolLatent.__init__   s   * /,	&L9$)))"!Y."]]d*
	(*(((\\%++i*SPR*STDN!DN$1	??i/ N##'??"3<	{"NN/ %ll5;;q$//4??#YVX#YZ PV4??IKHKK[][f[f[h))Iy1}J8JrJ&6",,M'<<DK'<<DK++-DK++-DK!		)\@R@DIZZ-DN:D:P
<626VXVaVaVcDI<\I-E)F}Uamv}z|}DHDIZZ-DNDIDH$L(    c                     | j                   1t        | j                   | j                   j                  d   dz         t        | j                  | j                  dz         y )Nr   r$   )std)r   r	   shaper1   r   )r@   s    rD   r?   z AttentionPoolLatent.init_weightsa   sE    >>%T^^1E1Ea1HD1PQ$//T*ABrE   	attn_maskc                    |j                   \  }}}| j                  7|| j                  j                  d      j                  |j                        z   }| j
                  j                  |dd      }| j                  |      j                  || j                  | j                  | j                        j                  dd      }| j                  |      j                  ||d| j                  | j                        j                  ddddd      }|j                  d      \  }	}
| j!                  |      | j#                  |	      }	}| j$                  rt'        j(                  ||	|
|      }nG|| j*                  z  }||	j                  dd      z  }t-        ||      }|j/                  d	      }||
z  }|j                  dd      j                  || j                  |      }| j1                  |      }| j3                  |      }| j4                  #|| j5                  | j7                  |            z   }| j8                  d
k(  r|d d df   }|S | j8                  dk(  r|j;                  d      }|S )Nr   r   r'         )rI   dimr   avg)rH   r   	unsqueezetor#   r1   expandr4   reshaper   r   r*   	transposer5   permuteunbindr7   r8   r   Fscaled_dot_product_attentionr+   r   softmaxr9   r;   r>   r<   r,   mean)r@   xrI   BNCq_latentr4   r5   kvattns               rD   forwardzAttentionPoolLatent.forwardf   s   ''1a>>%DNN,,Q/22177;;A;;%%aR0FF8$$QWaabcefgWWQZ1aGOOPQSTVWYZ\]^yy|1{{1~t{{1~1??..q!Q)LADJJAq{{2r**D!$	2D<<B<'DqAKK1%%a!<IIaLNN188DHHTYYq\**A 99!Q$A  YY%q	ArE   )N)__name__
__module____qualname____doc__r/   jitFinalbool__annotations__r-   GELUr=   r   floatstrr   Moduler)   r?   Tensorre   __classcell__rC   s   @rD   r   r      sD    		%%
 !%!'+"!!"$483577'KK K 	K
 K  }K K K K K K K K K !bii1K   RYY0!K" #KZC
#HU\\$: #rE   r   c                        e Zd ZU dZej
                  j                  e   ed<   	 	 	 	 	 	 	 dde	de	de
dededeeej                        f fd	Zd
ej                   dej                   fdZ xZS )AttentionPoolPrruF   Patch Representation Refinement (PRR) attention pool.

    From "Locality-Attending Vision Transformer" (ICLR 2026).

    Parameter-free multi-head self-attention that refines all patch representations
    before pooling. No Q/K/V projections — input is reshaped directly into multi-head
    format for self-attention.
    r   rP   r   r   pre_norm	post_normr   c	                    ||d}	t         
|           |dv sJ d| d       ||z  dk(  sJ d| d| d       ||s|rt        j                  }|| _        ||z  | _        | j
                  d	z  | _        || _        t               | _	        || _
        |r	 ||fi |	nt        j                         | _        |r ||fi |	| _        y t        j                         | _        y )
Nr!   )r   rQ   z)pool_type must be 'token' or 'avg', got ''r   zdim (z") must be divisible by num_heads ()r$   )r(   r)   r-   r6   r   r*   r+   r,   r   r   r   r3   rw   rx   )r@   rP   r   r   rw   rx   r   r"   r#   rA   rC   s             rD   r)   zAttentionPoolPrr.__init__   s     /,,f0YZcYdde.ff,Y!#`uSE1ST]S^^_%``#8yJ"y(]]d*
	(*19
3-"-r{{}2;C.2.rE   r]   returnc                 N   |j                   \  }}}| j                  |      }|j                  ||| j                  | j                        j                  dd      }| j                  rt        j                  |||      }n9|| j                  z  |j                  dd      z  }|j                  d      }||z  }|j                  dd      j                  |||      }| j                  |      }| j                  dk(  r|d d df   }|S | j                  dk(  r|j                  d      }|S )	Nr   r'   rN   rK   rO   r   r   rQ   )rH   rw   rU   r   r*   rV   r   rY   rZ   r+   r[   rx   r,   r\   )r@   r]   r^   r_   r`   qkvrd   s          rD   re   zAttentionPoolPrr.forward   s   ''1aMM! ii1dnndmm<FFq!L??..sC=A$**$b"(==D<<B<'Ds
AKK1%%aA.NN1 99!Q$A  YY%q	ArE   )r   r   FFNNN)rf   rg   rh   ri   r/   rj   rk   rl   rm   r=   rp   r   r   r-   rq   r)   rr   re   rs   rt   s   @rD   rv   rv      s     		%%
 $"#48OO O 	O
 O O !bii1O: %,, rE   rv   )typingr   r   r/   torch.nnr-   torch.nn.functional
functionalrY   	attentionr   configr   r>   r   weight_initr	   rq   r   rv    rE   rD   <module>r      sC    !     % "  )|")) |~?ryy ?rE   