Ë
    ÛÓ^jY?  ã                   ó¬   — d dl Z d dlmZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 dd	lmZ dd
lmZmZ ddlmZmZ ddlmZmZmZmZmZ  G d„ d«      Zy)é    N)ÚCallable)Únullcontext)Únné   )ÚContinuousBatchingConfigé   )ÚPagedAttentionCache)Ú%ContinuousBatchingLogitsProcessorList)ÚContinuousBatchingAsyncIOsÚContinuousBatchingIOs)ÚRequestStatusÚlogger)Úcreate_warmup_future_statesÚget_cuda_poolsÚmem_pool_ctxÚpad_to_intervalÚpad_to_pow2c                   ó(  — e Zd ZdZdededeez  dede	de	dd	fd
„Z
dedede	deeef   fd„Zdej                  de	fd„Zdej                  dedd	fd„Zde	deee	f   fd„Zdedej,                  j.                  dd	fd„Zdej                  dedej2                  dej2                  dej2                  dd	fd„Zdej2                  dej2                  dd	fd„Z ej8                  «       dej                  dd	fd„«       Zdej                  deded	z  defd„Zy	) ÚModelRunnerzžThis class is the continuous batching entry point for running the model. As a rule of thumb, anything running on
    the device should happen from this class.Úlogit_processorÚ	cb_configÚinputs_and_outputsÚcacheÚ	do_sampleÚreturn_logprobsÚreturnNc                 ój  — || _         || _        || _        || _        || _        | j                  j
                  \  | _        | _        || _        d | _	        | j                  xs | j                  }| j                  j                  d uxs | j                  j                  d u}|xs || _        | j                  s| j                  rt        «       \  | _        | _        nd\  | _        | _        d | _        | j                  j                  �Gt#        j$                  | j&                  fi | j                  j                  j)                  «       ¤Ž| _        d | _        | j                  j                  �Ht#        j$                  | j&                  fi | j                  j                  j)                  «       ¤Ž| _        y y )N)NN)r   r   r   r   r   Úcuda_graph_booleansÚuse_cuda_graph_varlenÚuse_cuda_graph_decoder   Ú_model_supports_logits_to_keepÚvarlen_compile_configÚdecode_compile_configÚ
pad_inputsr   Úmem_poolÚgraph_pool_idÚ_compiled_varlenÚtorchÚcompileÚ_forward_process_and_sampleÚto_dictÚ_compiled_decode)	Úselfr   r   r   r   r   r   Ú
cuda_graphr)   s	            úƒ/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/generation/continuous_batching/model_runner.pyÚ__init__zModelRunner.__init__!   sw  € ð  /ˆÔØ"ˆŒØ"4ˆÔà"ˆŒØ.ˆÔØAEÇÁ×AcÑAcÑ>ˆÔ" DÔ$>ØˆŒ
Ø;?ˆÔ+ð ×/Ñ/ÒM°4×3MÑ3Mˆ
Ø—.‘.×6Ñ6¸dÐBÒvÀdÇnÁn×FjÑFjÐrvÐFvˆØ$Ò/¨ˆŒð ×%Ò%¨×)CÒ)CÜ0>Ó0@Ñ-ˆDŒM˜4Õ-à0:Ñ-ˆDŒM˜4Ô-ð !%ˆÔØ�>‰>×/Ñ/Ð;Ü$)§M¡MØ×0Ñ0ñ%Ø48·N±N×4XÑ4X×4`Ñ4`Ó4bñ%ˆDÔ!ð
 !%ˆÔØ�>‰>×/Ñ/Ð;Ü$)§M¡MØ×0Ñ0ñ%Ø48·N±N×4XÑ4X×4`Ñ4`Ó4bñ%ˆDÕ!ð <ó    Únum_q_tokensÚmax_kv_readÚuse_decode_fast_pathc                 óR  — | j                   s||fS | j                  j                  }|sZt        || j                  j
                  |«      }t        || j                  j                  | j                  j                  «      }||fS t        || j                  j                  «      }d}||fS )zZPads the input sizes for the next batch if it is needed. Often it is, for max performance.r   )
r$   r   Úmax_batch_tokensr   r   Úq_padding_interval_sizeÚkv_padding_interval_sizeÚ	num_pagesr   Úmax_requests_per_batch)r-   r2   r3   r4   r6   s        r/   Úmaybe_pad_inputszModelRunner.maybe_pad_inputsN   sš   € à�ŠØ Ð,Ð,ØŸ:™:×6Ñ6Ðá#Ü*¨<¸¿¹×9_Ñ9_ÐaqÓrˆLÜ)¨+°t·~±~×7^Ñ7^Ð`d×`jÑ`j×`tÑ`tÓuˆKð
 ˜[Ð(Ð(ô ' |°T·^±^×5ZÑ5ZÓ[ˆLØˆKØ˜[Ð(Ð(r1   Úmodelc                 óx   — | j                   €#t        |d«      xr |j                  «       | _         | j                   S )zJReturns True if the model accepts the logits_to_keep kwarg in its forward.Ú_supports_logits_to_keep)r!   Úhasattrr>   )r-   r<   s     r/   Úsupports_logits_to_keepz#ModelRunner.supports_logits_to_keep]   s=   € à×.Ñ.Ð6ä˜Ð9Ó:Ò_¸u×?]Ñ?]Ó?_ð Ô/ð ×2Ñ2Ð2r1   Ú
batch_datac                 óŠ  — | j                   j                  «       \  }}}| j                   j                  }| j                  |«      r|d   |d<   | j	                  | j                   j
                  ¬«      \  }}|sC|�t        j                  j                  |«      n	t        «       }	|	5   ||||||«       ddd«       y| j                   j                  «       }
|
�9t        j                  j                  |«      5  |
j                  «        ddd«       y|||||f} | j                  ||g|¢­Ž  y# 1 sw Y   yxY w# 1 sw Y   yxY w)züRuns the forward pass, processes the logits and samples the next tokens. It also handles which version of
        the forward pass to use (varlen or decode), whether to use CUDA graphs (with the eventual capture of the graph)
        and torch compile.Úlogits_indicesÚlogits_to_keep)Úuse_block_tableN)r   Úget_cb_kwargsÚcompute_streamr@   Ú_get_forward_fnrE   r(   ÚcudaÚstreamr   Ú	get_graphÚreplayÚ_capture_graph)r-   r<   rA   Úcarry_over_idsÚprev_output_idsÚ
output_idsrG   Ú
forward_fnÚuse_cuda_graphÚmaybe_streamÚgraphÚargss               r/   Úcompute_batchzModelRunner.compute_batche   sG  € ð
 7;×6MÑ6M×6[Ñ6[Ó6]Ñ3ˆ˜¨à×0Ñ0×?Ñ?ˆð ×'Ñ'¨Ô.Ø+5Ð6FÑ+GˆJÐ'Ñ(ð &*×%9Ñ%9È$×JaÑJa×JqÑJqÐ%9Ó%rÑ"ˆ
�Nñ Ø@NÐ@Zœ5Ÿ:™:×,Ñ,¨^Ô<Ô`kÓ`mˆLØñ [Ù˜5 *¨n¸oÈzÔZ÷[ð [ð
 ×+Ñ+×5Ñ5Ó7ˆEàÐ Ü—Z‘Z×&Ñ& ~Ó6ñ #Ø—L‘L”N÷#ð #ð ˜z¨>¸?ÈJÐW�Ø#�×#Ñ# J°ÐFÀÔF÷[ð [ú÷#ð #ús   Â&D-Ã7D9Ä-D6Ä9ErE   c                 óÖ   — |r4| j                   €| j                  n| j                   }| j                  }||fS | j                  €| j                  n| j                  }| j                  }||fS )zfHelper function to get the appropriate forward function based on the block table and compile behavior.)r,   r*   r    r'   r   )r-   rE   rQ   rR   s       r/   rH   zModelRunner._get_forward_fn‡   st   € áØ=A×=RÑ=RÐ=Z˜×9Ò9Ð`d×`uÑ`uˆJØ!×7Ñ7ˆNð ˜>Ð)Ð)ð >B×=RÑ=RÐ=Z˜×9Ò9Ð`d×`uÑ`uˆJØ!×7Ñ7ˆNØ˜>Ð)Ð)r1   rQ   rG   c                 óÊ  — t         j                  j                  |«      5  t        | j                  «      5   ||Ž  ddd«       ddd«       t         j                  j                  «       }t         j                  j                  ||| j                  d¬«      5   ||Ž  ddd«       | j                  j                  |«       y# 1 sw Y   Œ†xY w# 1 sw Y   ŒŠxY w# 1 sw Y   Œ=xY w)zJHelper function to capture and store a graph for a given forward function.NÚthread_local)rJ   ÚpoolÚcapture_error_mode)
r(   rI   rJ   r   r%   Ú	CUDAGraphrT   r&   r   Ú	set_graph)r-   rQ   rG   rU   rT   s        r/   rM   zModelRunner._capture_graph‘   sÃ   € ô �Z‰Z×Ñ˜~Ó.ñ 	´¸T¿]¹]Ó0Kñ 	Ù˜Ñ÷	÷ 	ô —
‘
×$Ñ$Ó&ˆÜ�Z‰Z×ÑØ˜.¨t×/AÑ/AÐVdð ó 
ñ 	ñ ˜Ñ÷	ð
 	×Ñ×)Ñ)¨%Õ0÷	ð 	ú÷ 	ð 	ú÷	ð 	ús-    C¶C¼CÂCÃC
	ÃCÃCÃC"rN   rO   rP   c                 ó¸  — | j                   j                  |d   ||«        |di |¤Žj                  }|d   }d|vr|dd…|dd…f   }|j                  «       }| j                  j
                  rY|j                  \  }}	}
|j                  ||	z  |
«      }|d   d|f   }| j	                  |||d   «      }|j                  ||	|
«      }n|}| j                  ||«       y)z‚This function performs the forward pass, logits processing, and sampling. This is what is either captured
        and/or compiled.Ú	input_idsrC   rD   Nr   Úlogits_processor_args© )	r   Úcarry_over_tokensÚlogitsÚfloatr   Údo_processingÚshapeÚviewÚ_sample)r-   r<   rA   rN   rO   rP   rc   rC   Ú	dummy_dimÚ
num_logitsÚ
vocab_sizeÚ	logits_2dÚsliced_input_ids_2dÚscoress                 r/   r*   z'ModelRunner._forward_process_and_sampleŸ   sõ   € ð 	×Ñ×1Ñ1°*¸[Ñ2IÈ>Ð[jÔkñ Ñ$˜Ñ$×+Ñ+ˆð $Ð$4Ñ5ˆØ :Ñ-ØšA˜~ªqÐ0Ñ1ˆFà—‘“ˆð ×Ñ×-Ò-à06·±Ñ-ˆI�z :ØŸ™ I°
Ñ$:¸JÓGˆIØ",¨[Ñ"9¸!¸^Ð:KÑ"LÐà×,Ñ,Ð-@À)ÈZÐXoÑMpÓqˆIà—^‘^ I¨z¸:ÓF‰FàˆFð 	�‰�V˜ZÕ(r1   rn   c                 óŽ  — | j                   s| j                  r%t        j                  j	                  |d   d¬«      }n|j                  d«      }| j                   rt        j                  |d¬«      }nt        j                  |dd¬«      }| j                  r2|j                  d|¬«      j                  d«      }|j                  «       }|j                  d«      }|j                  d«      }|dd	|…f   j                  |«       | j                  r7|dd	|…f   j                  j                  t        j                  ¬
«      «       y	y	)z5Private method to sample next tokens from the scores.r   éÿÿÿÿ)Údimr   )Únum_samplesT)rq   Úkeepdim)rq   ÚindexN)Údtype)r   r   r   Ú
functionalÚsoftmaxÚsqueezer(   ÚmultinomialÚargmaxÚgatherÚlogÚsizeÚcopy_rg   Úint32)r-   rn   rP   ÚprobsÚnext_tokensÚper_token_probsÚlogprobsÚtokenss           r/   rh   zModelRunner._sampleÆ   s  € ð �>Š>˜T×1Ò1Ü—M‘M×)Ñ)¨&°©)¸Ð)Ó<‰Eà—N‘N 1Ó%ˆEð �>Š>Ü×+Ñ+¨E¸qÔA‰KäŸ,™, u°"¸dÔCˆKð ×ÒØ#Ÿl™l¨q¸˜lÓD×LÑLÈRÓPˆOØ&×*Ñ*Ó,ˆHð "×)Ñ)¨"Ó-ˆð ×!Ñ! !Ó$ˆØ�1�g�v�g�:Ñ×$Ñ$ [Ô1Ø×Òð �q˜'˜6˜'�zÑ"×(Ñ(¨¯©¼U¿[¹[¨Ó)IÕJð  r1   c                 óÂ  — | j                   syd}t        | j                  t        «      rdnd}t	        |«      D �]	  }| j
                  j                  }| j
                  j                  | j
                  j                  z  }||z  }|| j                  |||¬«      z  }| j
                  j                  dk(  rŒ}d}	 || j                  ||d¬«      z  }|| j                  j                  k\  rn$t        d|z  | j                  j                  «      }ŒUt        | j                  t        «      sŒð| j                  j                  «        �Œ t        j                   d|d›d�«       y)	zÖPre-capture CUDA graphs and/or trigger compile warmup for varlen and decode paths (if available). Unless the
        force_warmup flag is set, the warmup is only performed if the CUDA graphs or compile are enabled.Nr   é   r   )r<   r2   r3   úWarmup completed in ú.2fÚs)r$   Ú
isinstancer   r   Úranger   r6   Ú
num_blocksÚ
block_sizeÚrun_one_warmupÚmax_blocks_per_requestr   r:   ÚminÚswap_io_pairsr   Úinfo)r-   r<   Útotal_durationÚ
iterationsÚ_r2   r3   Únum_requestss           r/   ÚwarmupzModelRunner.warmupä   sI  € ð
 �ŠØð ˆÜ$ T×%<Ñ%<Ô>XÔY‘QÐ_`ˆ
Ü�zÓ"ó 	8ˆAàŸ:™:×6Ñ6ˆLØŸ*™*×/Ñ/°$·*±*×2GÑ2GÑGˆKØ˜<Ñ'ˆKØ˜d×1Ñ1¸ÈLÐfqÐ1ÓrÑrˆNð �z‰z×0Ñ0°AÒ5Øð ˆLØØ $×"5Ñ"5¸EÐP\ÐjnÐ"5Ó"oÑo�Ø 4§>¡>×#HÑ#HÒHØÜ" 1 |Ñ#3°T·^±^×5ZÑ5ZÓ[�ð	 ô ˜$×1Ñ1Ô3MÕNØ×'Ñ'×5Ñ5Ö7ð+	8ô, 	�‰Ð*¨>¸#Ð*>¸aÐ@ÕAr1   c           	      ó"  — |du }|rD|}t         j                  }d}| j                  j                  }t	        j
                  d|›d�«       n.d}t         j                  }t	        j
                  d|›d|›d�«       t        ||||| j                  «      }|s t	        j                  d|›d|›d|›d�«       y	| j                  ||z  ||¬
«      \  }}	t        j                  «       }
	 | j                  j                  || j                  |||	d¬«       | j                  j                  d¬«      }| j!                  ||«       t        j                  «       |
z
  }t	        j
                  d|d›d�«       |D ]1  }| j                  j%                  |j&                  j(                  «       Œ3 |S # t"        $ r%}d	}t	        j                  d|› d�«       Y d}~Œad}~ww xY w# |D ]1  }| j                  j%                  |j&                  j(                  «       Œ3 w xY w)a.  Warms up the decode fast path (if max_kv_read is None) or varlen path (if max_kv_read is an int) for a
        specific number of query and cache-resident tokens. `max_kv_read` is the number of tokens already in cache,
        matching the terminology used by `prepare_batch_tensors` and the scheduler.Nr   z/Warming up decode fast path for num_requests = ú.z*Warming up varlen path for num_q_tokens = z, max_kv_read = z:Failed to warm up: no blocks allocated for num_requests = z, num_q_tokens = g        )r2   r3   r4   T)Úuse_paddingr‡   rˆ   r‰   zFailed to warm up: z-.
Graph pool may fragment and OOM under load.)r   ÚDECODINGr   r�   r   ÚdebugÚ
PREFILLINGr   Úwarningr;   ÚtimeÚperf_counterr   Úprepare_batch_tensorsr   Úget_model_kwargsrV   Ú	ExceptionÚfree_blocksÚstateÚ
request_id)r-   r<   r2   r3   r4   r–   ÚstatusÚfuture_statesÚpadded_qÚ	padded_kvÚstartrA   ÚdurationÚeÚfss                  r/   rŽ   zModelRunner.run_one_warmup  s"  € ð
  +¨dÐ2ÐÙØ'ˆLÜ"×+Ñ+ˆFØˆLØŸ*™*×/Ñ/ˆKÜ�L‰LÐK¸LÐ;LÈAÐNÕOàˆLÜ"×-Ñ-ˆFÜ�L‰LÐF°|Ð6GÐGXÈ+ÐIYÐYZÐ[Ô\Ü3°LÀ&È,ÐXcÐei×eoÑeoÓpˆÙÜ�N‰NØM¸lÐ=NÐN`ÐQ]ÐPaÐarÐdoÐcsÐstÐuôð ð #×3Ñ3Ø%¨Ñ4Ø#Ø!5ð 4ó 
Ñˆ�)ô ×!Ñ!Ó#ˆð	<Ø×#Ñ#×9Ñ9Ø˜t×3Ñ3Ð5IÈ8ÐU^Ðlpð :ô ð ×0Ñ0×AÑAÈdÐAÓSˆJØ×Ñ˜u jÔ1Ü×(Ñ(Ó*¨UÑ2ˆHÜ�L‰LÐ/°¸¨~¸QÐ?Ô@ð $ò <�Ø—
‘
×&Ñ& r§x¡x×':Ñ':Õ;ð<àˆøô ò 	dØˆHÜ�N‰NÐ0°°Ð3aÐb×cÑcûð	dûð $ò <�Ø—
‘
×&Ñ& r§x¡x×':Ñ':Õ;ñ<ús+   Ã#B
F% Æ%	GÆ.GÇ	G ÇGÇG Ç8H) Ú__name__Ú
__module__Ú__qualname__Ú__doc__r
   r   r   r   r	   Úboolr0   ÚintÚtupler;   r   ÚModuler@   ÚdictrV   r   rH   r(   rI   ÚStreamrM   ÚTensorr*   rh   Úno_gradr—   rd   rŽ   ra   r1   r/   r   r      sÕ  „ ñ1ð+à>ð+ð ,ð+ð 2Ð4NÑNð	+ð
 #ð+ð ð+ð ð+ð 
ó+ðZ)¨Sð )¸sð )ÐZ^ð )ÐchÐilÐnqÐiqÑcró )ð3¨R¯Y©Yð 3¸4ó 3ð G 2§9¡9ð  G¸$ð  GÀ4ó  GðD*¨tð *¸¸hÈ¸nÑ8Mó *ð1¨ð 1À5Ç:Á:×CTÑCTð 1Ð`dó 1ð%)à�y‰yð%)ð ð%)ð Ÿ™ð	%)ð
 Ÿ™ð%)ð —L‘Lð%)ð 
ó%)ðNK˜eŸl™lð K¸¿¹ð KÈó Kð< €U‡]�]ƒ_ð B˜BŸI™Ið  B¨$ò  Bó ð BðD2 B§I¡Ið 2¸Sð 2ÈsÐUYÉzð 2Ð^cô 2r1   r   )rŸ   Úcollections.abcr   Ú
contextlibr   r(   r   Úgeneration.configuration_utilsr   r   r	   Úcb_logits_processorsr
   Úinput_outputsr   r   Úrequestsr   r   Úutilsr   r   r   r   r   r   ra   r1   r/   ú<module>rÂ      s7   ðó Ý $Ý "ã Ý å FÝ &Ý Gß Lß +ß jÕ j÷\ò \r1   