
    ^j3"                        d dl Z d dlmZ d dlmZ d dlmZmZ d dlm	Z	 d dl
Z
d dlmZ d dlmZ dd	lmZmZmZ  G d
 d      Ze G d d             ZdedefdZdedededefdZd+dedededefdZdedededefdZ	 d,de
j6                  dee   dee   deddf
d Zd!ed"ed#ed$ed%e	dee   fd&Zd'e j>                  dee   fd(Z d) Z!ed*        Z"y)-    N)contextmanager)	dataclass)ceillog2)Any)PretrainedConfig)is_torch_greater_or_equal   )FutureRequestStateRequestStateRequestStatusc                       e Zd ZdZddZddZdeedf   dej                  j                  dz  fdZdeedf   d	ej                  j                  ddfd
Zy)CudaGraphBufferz`A dict for CUDA graphs with a special __del__ method to make sure the graphs are properly reset.returnNc                     i | _         y N_storage)selfs    |/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/generation/continuous_batching/utils.py__init__zCudaGraphBuffer.__init__   s	    EG    c                     | j                   r;| j                   j                         \  }}|j                          | j                   r:y y r   )r   popitemreset)r   _graphs      r   __del__zCudaGraphBuffer.__del__"   s1    mm}},,.HAuKKM mmr   key.c                 8    | j                   j                  |      S r   )r   get)r   r   s     r   	get_graphzCudaGraphBuffer.get_graph'   s    }}  %%r   r   c                 "    || j                   |<   y r   r   )r   r   r   s      r   	set_graphzCudaGraphBuffer.set_graph*   s    "cr   )r   N)__name__
__module____qualname____doc__r   r   tupleinttorchcuda	CUDAGraphr"   r$    r   r   r   r      sf    jH
&U38_ &1E1E1L &#U38_ #UZZ5I5I #d #r   r   c                   <    e Zd ZU dZdZeed<   dZeed<   dZeed<   y)WorkloadHintszRA tiny dataclass containing hints to help choose good continuous batching defaultsr   max_prompt_lengthmax_generated_lengthnum_requestsN)	r%   r&   r'   r(   r1   r*   __annotations__r2   r3   r.   r   r   r0   r0   .   s%    \s !#!L#r   r0   configr   c                     | j                   dv S )z:Checks if attention mask is needed for the given (config).)zpaged|eagerz
paged|sdpa)_attn_implementation)r5   s    r   attn_mask_is_neededr8   7   s    &&*GGGr   sizeinterval_size	max_valuec                 X    |dk  r|S | dkD  rt        | |z        |z  n|}t        ||      S )zQReturn the smallest multiple of (interval_size) >= (size), capped at (max_value).r   )r   min)r9   r:   r;   paddeds       r   pad_to_intervalr?   <   s9    ;?!8T$&'-7Fvy!!r   value	min_valuec                     t        | t        d|            } dt        t        t        |                   z  }t	        ||      S )zReturn the smallest power of 2 >= (value), capped at (max_value). If a minimum value is provided, the value is at
    least padded to that value.r
      )maxr*   r   r   r=   )r@   r;   rA   r>   s       r   pad_to_pow2rE   D   s<     s1i()E#d4;'((Fvy!!r   x	divide_byalign_toc                 T    t        t        | |z              } | |z  r| || |z  z
  z  } | S r   )r*   r   )rF   rG   rH   s      r   aligned_dividerJ   L   s4    DY A8|	XX&&Hr   attention_maskcumulative_seqlens_qcumulative_seqlens_ksliding_windowc                 J   t        j                  | j                        j                  }t	        t        |      dz
        D ]  }||dz      ||   z
  }||dz      ||   z
  }||k  r|dk\  r	||z
  dz   }nd}t        ||   ||dz            }	t        ||   ||dz            }
t        j                  | d|	|
f   j                  || j                  | j                        }t        j                  ||      }|dkD  r"||z
  |z
  }|t        j                  ||      z  }|| d|	|
f<    y)u  Builds an attention mask inplace using the cumulative seqlens of the query and key. If given a sliding window, it
    will also apply a sliding window mask on top. The attention mask is not boolean, it uses zeroes and -inf (or its
    equivalent) so it's more of an attention score bias tensor.
    The attention mask is a block-diagonal matrix, with each block an attention mask for a single query-key pair.
    Each of those block is built from a causal mask and, if there is a sliding window, a sliding window mask.

    An example is represented below, with seqlen_k = 8, seqlen_q = 4 and sliding_window = 6:

    CAUSAL MASK:

           █ █ █ █ █ ░ ░ ░
           █ █ █ █ █ █ ░ ░
           █ █ █ █ █ █ █ ░
           █ █ █ █ █ █ █ █

    SLIDING WINDOW MASK:
         ┌──────────────────────── seqlen_k - seqlen_q - sliding_window = 8 - 4 - 6 = -2 offset to the left
       <─┴─>
     ░ █ | █ █ █ █ █ █ █ █
     ░ ░ | █ █ █ █ █ █ █ █
     ░ ░ | ░ █ █ █ █ █ █ █
     ░ ░ | ░ ░ █ █ █ █ █ █

    ATTENTION MASK (sum of causal and sliding window masks):

           █ █ █ █ █ ░ ░ ░
           █ █ █ █ █ █ ░ ░
           ░ █ █ █ █ █ █ ░
           ░ ░ █ █ █ █ █ █

    Another example with seqlen_k = 5, seqlen_q = 3 and sliding_window = 2:

    CAUSAL MASK:

           █ █ █ ░ ░
           █ █ █ █ ░
           █ █ █ █ █

    SLIDING WINDOW MASK:
         ┌──────────────────────── seqlen_k - seqlen_q - sliding_window = 5 - 3 - 2 = 0 offset to the left
        <┴>
         | ░ █ █ █ █
         | ░ ░ █ █ █
         | ░ ░ ░ █ █

    ATTENTION MASK (sum of causal and sliding window masks):

           ░ █ █ ░ ░
           ░ ░ █ █ ░
           ░ ░ ░ █ █

    r
   .)dtypedevice)diagonalN)r+   finforP   r=   rangelenslicefullshaperQ   triutril)rK   rL   rM   rN   rA   iseqlen_qseqlen_kcausal_diagonalquery_range	key_range	minus_infmaskedsliding_diagonals                 r   build_attention_maskrd   S   sS   t N00155I3+,q01 ='A.1Ea1HH'A.1Ea1HHh8q=&1A5OO035I!a%5PQ.q13GA3NO	JJ3Y67== &&!((	
	 I@A'(2^Cejj5EFFF6<sK23-=r   numstatusnum_q_tokensmax_kv_readcachec           	         t        |       D cg c]  }d|j                   d| d }}||z   }t        ||j                  z        }g }	|D ]n  }
t	        |
dg|z  d      }||_        dg|z  |_        ||_        |j                  ||j                  d      }||	c S |	j                  t        |dd|             p |	S c c}w )	zPA utility function to create a list of FutureRequestStates for the warmup of CB.	__warmup_r   __r   r
   )
request_idinitial_tokensmax_new_tokensT)has_new_tokencomplete_blocksquery_length)rT   namer   
block_sizer   _statustokens_to_processposition_offsetallocate_blocksrm   appendr   )re   rf   rg   rh   ri   r[   request_idstotal_tokensblocks_neededfuture_statesreq_idstate	allocateds                r   create_warmup_future_statesr      s     =B#JGqYv{{m1QCr2GKG+-L(8(889MM 
s\?Qbcd#$#"4 +))-9I9I1M	  uD!Zfg	

 # Hs   B>request_queuec                     g }| j                         s5	 | j                         }||j                  |       | j                         s5|S # t        j                  $ r Y |S w xY w)z3Drains a queue and returns a list of RequestStates.)empty
get_nowaitry   queueEmpty)r   
new_statesr   s      r   drain_queuer      sn    %'J!!#	!,,.E !!%(	 !!#  {{ 		s   #A
 
A! A!c                      t        d      r.t        j                  j                         } | j                  }| |fS d} t        j                  j                         }| |fS )zReturns a tuple of (mem_pool, graph_pool_id) for CUDA graphs. Since the MemPool object is only available in torch
    2.5+, we only return a graph_pool_id for older versions.z2.5.0N)r	   r+   r,   MemPoolidgraph_pool_handle)mem_poolgraph_pool_ids     r   get_cuda_poolsr      sT     !)::%%' &&

446&&r   c              #      K   | -t         j                  j                  |       5  d ddd       yd y# 1 sw Y   yxY ww)zA context manager to use a CUDA mem pool. If the mem pool is None, it is a no-op. No type hint because it would
    make torch 2.4 or below crash.N)r+   r,   use_mem_pool)r   s    r   mem_pool_ctxr      sA      ZZ$$X. 		 	 		 	s   "A7AA A)r   )r
   )#r   
contextlibr   dataclassesr   mathr   r   typingr   r+    transformers.configuration_utilsr   transformers.utilsr	   requestsr   r   r   r   r0   boolr8   r*   r?   rE   rJ   Tensorlistrd   r   Queuer   r   r   r.   r   r   <module>r      s    % !    = 8 E E# #$   H 0 HT H
"# "c "c "c ""s "s "s "3 "c c S S  	Q=LLQ=s)Q= s)Q= 	Q=
 
Q=h	  	
  

:
u{{ 
tL/A 

'  r   