Ë
    éÓ^jö  ã                   ó˜   — d Z ddlmZmZ ddlZddlmZ ddlmc mZ ddl	m
Z
 ddlmZmZ ddlmZ dd	„Z G d
„ dej"                  «      Zy)an   Lambda Layer

Paper: `LambdaNetworks: Modeling Long-Range Interactions Without Attention`
    - https://arxiv.org/abs/2102.08602

@misc{2102.08602,
Author = {Irwan Bello},
Title = {LambdaNetworks: Modeling Long-Range Interactions Without Attention},
Year = {2021},
}

Status:
This impl is a WIP. Code snippets in the paper were used as reference but
good chance some details are missing/wrong.

I've only implemented local lambda conv based pos embeddings.

For a PyTorch impl that includes other embedding options checkout
https://github.com/lucidrains/lambda-networks

Hacked together by / Copyright 2021 Ross Wightman
é    )ÚOptionalÚTupleN)Únné   )Úndgrid)Ú	to_2tupleÚmake_divisible)Útrunc_normal_c           
      ó’  — t        | «      } t        j                  t        t        j                  | d   |t        j
                  ¬«      t        j                  | d   |t        j
                  ¬«      «      «      j                  d«      }|d d …d d d …f   |d d …d d …d f   z
  }|dxx   | d   dz
  z  cc<   |dxx   | d   dz
  z  cc<   |S )Nr   ©ÚdeviceÚdtyper   )r   ÚtorchÚstackr   ÚarangeÚlongÚflatten)Úsizer   ÚposÚrel_poss       úc/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/lambda_layer.pyÚrel_pos_indicesr   "   s¯   € Ü�T‹?€DÜ
�+‰+”fÜ�‰�T˜!‘W V´5·:±:Ô>Ü�‰�T˜!‘W V´5·:±:Ô>óó ÷ �wˆqƒzð ð ’!�Tš1�*‰o ¢A¢q¨$ J¡Ñ/€GØˆAƒJ�$�q‘'˜A‘+ÑƒJØˆAƒJ�$�q‘'˜A‘+ÑƒJØ€Nó    c                   ó’   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 ddedee   deeeef      dedededed	ed
efˆ fd„Z	dd„Z
dd„Zd„ Zdd„Zˆ xZS )ÚLambdaLayera  Lambda Layer

    Paper: `LambdaNetworks: Modeling Long-Range Interactions Without Attention`
        - https://arxiv.org/abs/2102.08602

    NOTE: intra-depth parameter 'u' is fixed at 1. It did not appear worth the complexity to add.

    The internal dimensions of the lambda module are controlled via the interaction of several arguments.
      * the output dimension of the module is specified by dim_out, which falls back to input dim if not set
      * the value (v) dimension is set to dim_out // num_heads, the v projection determines the output dim
      * the query (q) and key (k) dimension are determined by
        * dim_head = (dim_out * attn_ratio // num_heads) if dim_head is None
        * q = num_heads * dim_head, k = dim_head
      * as seen above, attn_ratio determines the ratio of q and k relative to the output if dim_head not set

    Args:
        dim: input dimension to the module
        dim_out: output dimension of the module, same as dim if not set
        feat_size: size of input feature_map for relative pos variant H, W
        stride: output stride of the module, avg pool used if stride == 2
        num_heads: parallel attention heads.
        dim_head: dimension of query and key heads, calculated from dim_out * attn_ratio // num_heads if not set
        r: local lambda convolution radius. Use lambda conv if set, else relative pos if not. (default: 9)
        qk_ratio: ratio of q and k dimensions to output dimension when dim_head not set. (default: 1.0)
        qkv_bias: add bias to q, k, and v projections
    ÚdimÚdim_outÚ	feat_sizeÚstrideÚ	num_headsÚdim_headÚrÚqk_ratioÚqkv_biasc                 ór  •— |
|dœ}t         ‰| �  «        |xs |}||z  dk(  sJ d«       ‚|xs t        ||z  d¬«      |z  | _        || _        ||z  | _        t        j                  ||| j                  z  | j                  z   | j
                  z   fd|	dœ|¤Ž| _        t        j                  || j                  z  fi |¤Ž| _
        t        j                  | j
                  fi |¤Ž| _        |�Kt        j                  d| j                  ||dffd|d	z  |d	z  dfi|¤Ž| _        d | _        d | _        d | _        n¸|€J ‚t#        |«      }|| _        |D �cg c]
  }d	|z  dz
  ‘Œ }}|d   |d   z  }d | _        t        j$                  t'        j(                  |d   |d   | j                  fi |¤Ž«      | _        | j+                  d
t'        j(                  d	||f|
t&        j,                  ¬«      d¬«       |d	k(  rt        j.                  d	d	«      nt        j0                  «       | _        | j5                  «        y c c}w )Nr   r   z should be divided by num_headsé   )Údivisorr   )Úkernel_sizeÚbiasÚpaddingé   r   F)Ú
persistent)ÚsuperÚ__init__r	   Údim_qkr    Údim_vr   ÚConv2dÚqkvÚBatchNorm2dÚnorm_qÚnorm_vÚConv3dÚconv_lambdaÚpos_embr   r   r   Ú	Parameterr   ÚemptyÚregister_bufferr   Ú	AvgPool2dÚIdentityÚpoolÚreset_parameters)Úselfr   r   r   r   r    r!   r"   r#   r$   r   r   ÚddÚsÚrel_sizeÚMÚ	__class__s                   €r   r.   zLambdaLayer.__init__I   s2  ø€ ð ¨Ñ/ˆÜ‰ÑÔØ’.˜SˆØ˜Ñ" aÒ'ÐJÐ)JÓJÐ'ØÒ\¤.°¸8Ñ1CÈQÔ"OÐS\Ñ"\ˆŒØ"ˆŒØ 	Ñ)ˆŒ
ä—9‘9ØØ˜Ÿ™Ñ# d§k¡kÑ1°D·J±JÑ>ð
ð Øñ	
ð
 ñ
ˆŒô —n‘n Y°·±Ñ%<ÑCÀÑCˆŒÜ—n‘n T§Z¡ZÑ6°2Ñ6ˆŒàˆ=ä!Ÿy™y¨¨D¯K©K¸!¸QÀ¸ÑfÈQÐRSÉVÐUVÐZ[ÑU[Ð]^ÐL_ÐfÐceÑfˆDÔØˆDŒLØ#'ˆDÔ Ø!ˆD�Nð Ð(Ð(Ð(Ü! )Ó,ˆIØ&ˆDŒNØ+4Ö5 a˜˜A™ ›	Ð5ˆHÐ5Ø˜!‘˜y¨™|Ñ+ˆAØ#ˆDÔÜŸ<™<¬¯©°H¸Q±KÀÈ!ÁÈdÏkÉkÑ(`Ð]_Ñ(`ÓaˆDŒLØ× Ñ Ø!Ü—‘˜Q  1˜I¨f¼E¿J¹JÔGØ ð !ô ð +1°Aª+”B—L‘L  AÔ&¼2¿;¹;»=ˆŒ	ð 	×ÑÕùò 6s   ÅH4c                 óh  — t        | j                  j                  | j                  j                  j                  d   dz  ¬«       | j                  �.t        | j                  j                  | j
                  dz  ¬«       | j                  �t        | j                  d¬«       | j                  «        y)z"Initialize parameters and buffers.r   g      à¿)ÚstdNg{®Gáz”?)r
   r2   ÚweightÚshaper7   r/   r8   Ú_init_buffers©r@   s    r   r?   zLambdaLayer.reset_parametersƒ   s{   € ä�d—h‘h—o‘o¨4¯8©8¯?©?×+@Ñ+@ÀÑ+CÀtÑ+KÕLØ×ÑÐ'Ü˜$×*Ñ*×1Ñ1°t·{±{ÀdÑ7JÕKØ�<‰<Ð#Ü˜$Ÿ,™,¨CÕ0Ø×ÑÕr   c                 ó¦   — | j                   �E| j                   j                  t        | j                  | j                   j                  ¬«      «       yy)z.Compute and fill non-persistent buffer values.N)r   )r   Úcopy_r   r   rK   s    r   rJ   zLambdaLayer._init_buffersŒ   sA   € à×ÑÐ+Ø× Ñ ×&Ñ&Ü §¡°t×7KÑ7K×7RÑ7RÔSõð ,r   c           	      ó  — |j                   \  }}}}||z  }| j                  |«      }t        j                  || j                  | j
                  z  | j
                  | j                  gd¬«      \  }}	}
| j                  |«      j                  || j                  | j
                  |«      j                  dd«      }| j                  |
«      j                  || j                  |«      j                  dd«      }
t        j                  |	j                  || j
                  |«      d¬«      }	|	|
z  }||j                  d«      z  }| j                  €k| j                  |
j                  |d||| j                  «      «      }|j                  |d| j
                  ||z  | j                  «      j                  dd«      }np| j                  | j                   d   | j                   d   f   j#                  |ddd«      }|j                  dd«      |
j                  d«      z  j                  d«      }|j                  d«      |z  j%                  d«      }||z   j                  dd«      j                  ||||«      }| j'                  |«      }|S )Nr   )r   éÿÿÿÿéþÿÿÿr+   é   r   )rI   r2   r   Úsplitr    r/   r0   r4   ÚreshapeÚ	transposer5   ÚFÚsoftmaxÚ	unsqueezer8   r7   r   ÚexpandÚsqueezer>   )r@   ÚxÚBÚCÚHÚWrD   r2   ÚqÚkÚvÚcontent_lamÚcontent_outÚposition_lamr8   Úposition_outÚouts                    r   ÚforwardzLambdaLayer.forward“   s+  € Ø—W‘W‰
ˆˆ1ˆa�Ø�‰EˆØ�h‰h�q‹kˆÜ—+‘+˜cØ�N‰N˜TŸ[™[Ñ(¨$¯+©+°t·z±zð$CØHIôK‰ˆˆ1ˆaà�K‰K˜‹N×"Ñ" 1 d§n¡n°d·k±kÀ1ÓE×OÑOÐPRÐTVÓWˆØ�K‰K˜‹N×"Ñ" 1 d§j¡j°!Ó4×>Ñ>¸rÀ2ÓFˆÜ�I‰I�a—i‘i  4§;¡;°Ó2¸Ô;ˆà˜!‘eˆØ˜+×/Ñ/°Ó2Ñ2ˆà�<‰<ÐØ×+Ñ+¨A¯I©I°a¸¸A¸qÀ$Ç*Á*Ó,MÓNˆLØ'×/Ñ/°°1°d·k±kÀ1ÀqÁ5È$Ï*É*ÓU×_Ñ_Ð`aÐcdÓe‰Lð —l‘l 4×#7Ñ#7¸Ñ#:¸D×<PÑ<PÐQRÑ<SÐ#SÑT×[Ñ[Ð\]Ð_aÐceÐgiÓjˆGØ#×-Ñ-¨b°"Ó5¸¿¹ÀA»ÑF×QÑQÐRSÓTˆLØŸ™ B›¨,Ñ6×?Ñ?ÀÓCˆà˜\Ñ)×4Ñ4°R¸Ó<×DÑDÀQÈÈ1ÈaÓPˆØ�i‰i˜‹nˆØˆ
r   c                 ó$   — | j                  «        y)z"Initialize non-persistent buffers.N)rJ   rK   s    r   Úinit_non_persistent_buffersz'LambdaLayer.init_non_persistent_buffers­   s   € à×ÑÕr   )
NNr   é   é   é	   g      ð?FNN)ÚreturnN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr   r   ÚfloatÚboolr.   r?   rJ   rg   ri   Ú__classcell__)rE   s   @r   r   r   .   s«   ø„ ñð: &*Ø37ØØØØØ!Ø"ØØñ8 àð8 ð ˜c‘]ð8 ð    c¨3 h¡Ñ0ð	8 ð
 ð8 ð ð8 ð ð8 ð ð8 ð ð8 ð õ8 ótóò÷4r   r   )N)rq   Útypingr   r   r   r   Útorch.nn.functionalÚ
functionalrU   Úgridr   Úhelpersr   r	   Úweight_initr
   r   ÚModuler   © r   r   ú<module>r~      s;   ðñ÷, #ã Ý ß Ð å ß .Ý &ó	ôA�"—)‘)õ Ar   