
    ^jY?                         d dl Z d dlmZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 dd	lmZ dd
lmZmZ ddlmZmZ ddlmZmZmZmZmZ  G d d      Zy)    N)Callable)nullcontext)nn   )ContinuousBatchingConfig   )PagedAttentionCache)%ContinuousBatchingLogitsProcessorList)ContinuousBatchingAsyncIOsContinuousBatchingIOs)RequestStatuslogger)create_warmup_future_statesget_cuda_poolsmem_pool_ctxpad_to_intervalpad_to_pow2c                   (   e Zd ZdZdededeez  dede	de	dd	fd
Z
dedede	deeef   fdZdej                  de	fdZdej                  dedd	fdZde	deee	f   fdZdedej,                  j.                  dd	fdZdej                  dedej2                  dej2                  dej2                  dd	fdZdej2                  dej2                  dd	fdZ ej8                         dej                  dd	fd       Zdej                  deded	z  defdZy	) ModelRunnerzThis class is the continuous batching entry point for running the model. As a rule of thumb, anything running on
    the device should happen from this class.logit_processor	cb_configinputs_and_outputscache	do_samplereturn_logprobsreturnNc                 j   || _         || _        || _        || _        || _        | j                  j
                  \  | _        | _        || _        d | _	        | j                  xs | j                  }| j                  j                  d uxs | j                  j                  d u}|xs || _        | j                  s| j                  rt               \  | _        | _        nd\  | _        | _        d | _        | j                  j                  Gt#        j$                  | j&                  fi | j                  j                  j)                         | _        d | _        | j                  j                  Ht#        j$                  | j&                  fi | j                  j                  j)                         | _        y y )N)NN)r   r   r   r   r   cuda_graph_booleansuse_cuda_graph_varlenuse_cuda_graph_decoder   _model_supports_logits_to_keepvarlen_compile_configdecode_compile_config
pad_inputsr   mem_poolgraph_pool_id_compiled_varlentorchcompile_forward_process_and_sampleto_dict_compiled_decode)	selfr   r   r   r   r   r   
cuda_graphr)   s	            /var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/generation/continuous_batching/model_runner.py__init__zModelRunner.__init__!   sw     /""4".AEAcAc>"D$>
;?+ //M43M3M
..66dBvdnnFjFjrvFv$/ %%)C)C0>0@-DM4-0:-DM4- !%>>//;$)MM00%48NN4X4X4`4`4b%D!
 !%>>//;$)MM00%48NN4X4X4`4`4b%D! <    num_q_tokensmax_kv_readuse_decode_fast_pathc                 R   | j                   s||fS | j                  j                  }|sZt        || j                  j
                  |      }t        || j                  j                  | j                  j                        }||fS t        || j                  j                        }d}||fS )zZPads the input sizes for the next batch if it is needed. Often it is, for max performance.r   )
r$   r   max_batch_tokensr   r   q_padding_interval_sizekv_padding_interval_size	num_pagesr   max_requests_per_batch)r-   r2   r3   r4   r6   s        r/   maybe_pad_inputszModelRunner.maybe_pad_inputsN   s    ,,::66#*<9_9_aqrL)+t~~7^7^`d`j`j`t`tuK
 [(( '|T^^5Z5Z[LK[((r1   modelc                 x    | j                   #t        |d      xr |j                         | _         | j                   S )zJReturns True if the model accepts the logits_to_keep kwarg in its forward._supports_logits_to_keep)r!   hasattrr>   )r-   r<   s     r/   supports_logits_to_keepz#ModelRunner.supports_logits_to_keep]   s=    ..69:_u?]?]?_ / 222r1   
batch_datac                    | j                   j                         \  }}}| j                   j                  }| j                  |      r|d   |d<   | j	                  | j                   j
                        \  }}|sC|t        j                  j                  |      n	t               }	|	5   ||||||       ddd       y| j                   j                         }
|
9t        j                  j                  |      5  |
j                          ddd       y|||||f} | j                  ||g|  y# 1 sw Y   yxY w# 1 sw Y   yxY w)zRuns the forward pass, processes the logits and samples the next tokens. It also handles which version of
        the forward pass to use (varlen or decode), whether to use CUDA graphs (with the eventual capture of the graph)
        and torch compile.logits_indiceslogits_to_keep)use_block_tableN)r   get_cb_kwargscompute_streamr@   _get_forward_fnrE   r(   cudastreamr   	get_graphreplay_capture_graph)r-   r<   rA   carry_over_idsprev_output_ids
output_idsrG   
forward_fnuse_cuda_graphmaybe_streamgraphargss               r/   compute_batchzModelRunner.compute_batche   sG   
 7;6M6M6[6[6]300?? ''.+56F+GJ'( &*%9%9$JaJaJqJq%9%r"
N @N@Z5::,,^<`k`mL [5*nozZ[ [
 ++557E ZZ&&~6 #LLN# # z>?JW###JFF[ [# #s   &D-7D9-D69ErE   c                     |r4| j                   | j                  n| j                   }| j                  }||fS | j                  | j                  n| j                  }| j                  }||fS )zfHelper function to get the appropriate forward function based on the block table and compile behavior.)r,   r*   r    r'   r   )r-   rE   rQ   rR   s       r/   rH   zModelRunner._get_forward_fn   st    =A=R=R=Z99`d`u`uJ!77N >)) >B=R=R=Z99`d`u`uJ!77N>))r1   rQ   rG   c                    t         j                  j                  |      5  t        | j                        5   ||  ddd       ddd       t         j                  j                         }t         j                  j                  ||| j                  d      5   ||  ddd       | j                  j                  |       y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   =xY w)zJHelper function to capture and store a graph for a given forward function.Nthread_local)rJ   poolcapture_error_mode)
r(   rI   rJ   r   r%   	CUDAGraphrT   r&   r   	set_graph)r-   rQ   rG   rU   rT   s        r/   rM   zModelRunner._capture_graph   s     ZZ~. 	T]]0K 		 	 

$$&ZZ.t/A/AVd  
 	 	
 	))%0	 	 	 		 	s-   CCCCC
	CCC"rN   rO   rP   c                    | j                   j                  |d   ||        |di |j                  }|d   }d|vr|dd|ddf   }|j                         }| j                  j
                  rY|j                  \  }}	}
|j                  ||	z  |
      }|d   d|f   }| j	                  |||d         }|j                  ||	|
      }n|}| j                  ||       y)zThis function performs the forward pass, logits processing, and sampling. This is what is either captured
        and/or compiled.	input_idsrC   rD   Nr   logits_processor_args )	r   carry_over_tokenslogitsfloatr   do_processingshapeview_sample)r-   r<   rA   rN   rO   rP   rc   rC   	dummy_dim
num_logits
vocab_size	logits_2dsliced_input_ids_2dscoress                 r/   r*   z'ModelRunner._forward_process_and_sample   s     	11*[2I>[jk $$++ $$45:-A~q01F --06-Iz:I
$:JGI",["9!^:K"L,,-@)ZXoMpqI^^Iz:FFF 	VZ(r1   rn   c                    | j                   s| j                  r%t        j                  j	                  |d   d      }n|j                  d      }| j                   rt        j                  |d      }nt        j                  |dd      }| j                  r2|j                  d|      j                  d      }|j                         }|j                  d      }|j                  d      }|dd	|f   j                  |       | j                  r7|dd	|f   j                  j                  t        j                  
             y	y	)z5Private method to sample next tokens from the scores.r   )dimr   )num_samplesT)rq   keepdim)rq   indexN)dtype)r   r   r   
functionalsoftmaxsqueezer(   multinomialargmaxgatherlogsizecopy_rg   int32)r-   rn   rP   probsnext_tokensper_token_probslogprobstokenss           r/   rh   zModelRunner._sample   s    >>T11MM))&))<ENN1%E >>++EqAK,,u"dCK #llqlDLLRPO&**,H "))"- !!!$1gvg:$$[1 q'6'z"((U[[)IJ  r1   c                    | j                   syd}t        | j                  t              rdnd}t	        |      D ]	  }| j
                  j                  }| j
                  j                  | j
                  j                  z  }||z  }|| j                  |||      z  }| j
                  j                  dk(  r}d}	 || j                  ||d      z  }|| j                  j                  k\  rn$t        d|z  | j                  j                        }Ut        | j                  t              s| j                  j                           t        j                   d|dd       y)	zPre-capture CUDA graphs and/or trigger compile warmup for varlen and decode paths (if available). Unless the
        force_warmup flag is set, the warmup is only performed if the CUDA graphs or compile are enabled.Nr      r   )r<   r2   r3   Warmup completed in .2fs)r$   
isinstancer   r   ranger   r6   
num_blocks
block_sizerun_one_warmupmax_blocks_per_requestr   r:   minswap_io_pairsr   info)r-   r<   total_duration
iterations_r2   r3   num_requestss           r/   warmupzModelRunner.warmup   sI   
  $T%<%<>XYQ_`
z" 	8A::66L**//$**2G2GGK<'Kd11Lfq1rrN zz00A5 L$"5"5EP\jn"5"oo4>>#H#HH"1|#3T^^5Z5Z[	  $113MN''557+	8, 	*>#*>a@Ar1   c           	      "   |du }|rD|}t         j                  }d}| j                  j                  }t	        j
                  d|d       n.d}t         j                  }t	        j
                  d|d|d       t        ||||| j                        }|s t	        j                  d|d|d|d       y	| j                  ||z  ||
      \  }}	t        j                         }
	 | j                  j                  || j                  |||	d       | j                  j                  d      }| j!                  ||       t        j                         |
z
  }t	        j
                  d|dd       |D ]1  }| j                  j%                  |j&                  j(                         3 |S # t"        $ r%}d	}t	        j                  d| d       Y d}~ad}~ww xY w# |D ]1  }| j                  j%                  |j&                  j(                         3 w xY w)a.  Warms up the decode fast path (if max_kv_read is None) or varlen path (if max_kv_read is an int) for a
        specific number of query and cache-resident tokens. `max_kv_read` is the number of tokens already in cache,
        matching the terminology used by `prepare_batch_tensors` and the scheduler.Nr   z/Warming up decode fast path for num_requests = .z*Warming up varlen path for num_q_tokens = z, max_kv_read = z:Failed to warm up: no blocks allocated for num_requests = z, num_q_tokens = g        )r2   r3   r4   T)use_paddingr   r   r   zFailed to warm up: z-.
Graph pool may fragment and OOM under load.)r   DECODINGr   r   r   debug
PREFILLINGr   warningr;   timeperf_counterr   prepare_batch_tensorsr   get_model_kwargsrV   	Exceptionfree_blocksstate
request_id)r-   r<   r2   r3   r4   r   statusfuture_statespadded_q	padded_kvstartrA   durationefss                  r/   r   zModelRunner.run_one_warmup  s"   
  +d2'L"++FL**//KLLKL;LANOL"--FLLF|6GGX+IYYZ[\3L&,XceieoeopNNMl=NN`Q]Paardocsstu  #33%4#!5 4 
) !!#	<##99t335I8U^lp :  00AAdASJuj1((*U2HLL/~Q?@ $ <

&&rxx':':;<  	dHNN03abcc	d $ <

&&rxx':':;<s+   #B
F% %	G.G	G GG 8H) __name__
__module____qualname____doc__r
   r   r   r   r	   boolr0   inttupler;   r   Moduler@   dictrV   r   rH   r(   rI   StreamrM   Tensorr*   rh   no_gradr   rd   r   ra   r1   r/   r   r      s   1+>+ ,+ 24NN	+
 #+ + + 
+Z)S )s )Z^ )chilnqiqcr )3RYY 34 3 G299  G$  G4  GD*t *hn8M *1 15::CTCT 1`d 1%)yy%) %) 	%)
 %) LL%) 
%)NKell K K K< U]]_ BBII  B$  B  BD2BII 2S 2sUYz 2^c 2r1   r   )r   collections.abcr   
contextlibr   r(   r   generation.configuration_utilsr   r   r	   cb_logits_processorsr
   input_outputsr   r   requestsr   r   utilsr   r   r   r   r   r   ra   r1   r/   <module>r      s7     $ "   F & G L + j j\ \r1   