
    ^j%                         d Z ddlZddlZddlZddlZddlmZ ddlZddlm	Z	 ddl
mZ ddlmZ  e	j                  e      Z G d d	eej$                        Z G d
 d      Zde_         y)z+
CLI entry point for `transformers serve`.
    N)	Annotated)logging)is_serve_available   )set_torch_seedc                       e Zd ZdZdZdZy)ReasoningModeonoffautoN)__name__
__module____qualname__ONOFFAUTO     a/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/cli/serve.pyr	   r	   #   s    	B
CDr   r	   c            2          e Zd Zdddddej                  j
                  ddddddddddddddddfdeedz   ej                  d	
      f   dee
 ej                  d
      f   deedz   ej                  d
      f   dee
 ej                  d
      f   deedz   ej                  d
      f   dee ej                  d
      f   deedz   ej                  d
      f   dee ej                  d
      f   deedz   ej                  d
      f   dee
 ej                  d
      f   dee ej                  d
      f   deedz   ej                  d 
      f   d!eedz   ej                  d"
      f   d#eedz   ej                  d$
      f   d%eedz   ej                  d&
      f   d'ee
dz   ej                  d(
      f   d)ee ej                  d*
      f   d+ee ej                  d,
      f   d-ee
 ej                  d.
      f   d/ee ej                  d0
      f   d1eedz   ej                  d2
      f   d3ee
 ej                  d4d56      f   d7df.d8Zd9 Zd: Zd; Zy)<ServeNFr   i,  	localhosti@  warningforce_modelz*Model to preload and use for all requests.)helpcontinuous_batchingzMEnable continuous batching with paged attention. Configure with --cb-* flags.attn_implementationz2Attention implementation (e.g. flash_attention_2).compilez*Enable torch.compile for faster inference.quantizationz.Quantization method: 'bnb-4bit' or 'bnb-8bit'.	reasoningu   Reasoning mode. 'auto' uses the chat template default. Only applies to models that support reasoning via their chat template (e.g. Qwen3, Gemma 4) — for other models this flag has no effect.chat_template_kwargszDefault JSON kwargs forwarded to apply_chat_template (e.g. '{"enable_thinking": true}'); per-request chat_template_kwargs override these.devicez4Device for inference (e.g. 'auto', 'cuda:0', 'cpu').dtypez2Override model dtype. 'auto' derives from weights.trust_remote_codezTrust remote code when loading.model_timeoutzGSeconds before idle model is unloaded. Ignored when force_model is set.cb_block_sizez6KV cache block size in tokens for continuous batching.cb_num_blocksz2Number of KV cache blocks for continuous batching.cb_max_batch_tokensz1Maximum tokens per batch for continuous batching.cb_max_memory_percentz/Max GPU memory fraction for KV cache (0.0-1.0).cb_use_cuda_graphz+Enable CUDA graphs for continuous batching.hostzServer listen address.portzServer listen port.enable_corszEnable permissive CORS.	log_levelz'Logging level (e.g. 'info', 'warning').default_seedzDefault torch seed.non_blockingTz1Run server in a background thread. Used by tests.)hiddenr   returnc           	      :   t               st        d      dd l}ddlm} ddlm} ddlm} ddl	m
} ddlm} dd	lm} dd
lm} |t#        |       t%        j&                  d      }|j)                  t$        j*                  |j-                                    |||	|
||||      | _        ddlm}  |||||dj5                         D !"ci c]
  \  }!}"|"|!|" }#}!}"|#r | di |#nd }$ ||||$      | _        |r:t9        j:                  |      }t=        |t>              stA        jB                  d      i }|tD        jF                  k(  rd|d<   n|tD        jH                  k(  rd|d<    || j.                  | j6                  |      | _%         || j.                  | j6                        | _&         || j.                  | j6                  |      | _'         || j.                  | j6                        | _(         || j.                  | jJ                  | jL                  | jN                  | jP                  | j6                  |      }%|jS                  |%||d      }&|jU                  |&      | _+        |r| jY                          y | jV                  j[                          y c c}"}!w )NzRMissing dependencies for serving. Install with `pip install transformers[serving]`r   r   )ChatCompletionHandler)CompletionHandler)ModelManager)ResponseHandler)build_server)TranscriptionHandler)GenerationStatetransformers)r"   r#   r$   r   r   r%   r   )ContinuousBatchingConfig)
block_size
num_blocksmax_batch_tokensmax_memory_percentuse_cuda_graph)r   r   	cb_configz,--chat-template-kwargs must be a JSON objectTenable_thinkingF)model_managergeneration_stater!   )rD   rE   )completion_handlerresponse_handlertranscription_handlerrE   r-   info)r+   r,   r.   r   ).r   ImportErroruvicornserving.chat_completionr4   serving.completionr5   serving.model_managerr6   serving.responser7   serving.serverr8   serving.transcriptionr9   serving.utilsr:   r   r   
get_loggersetLevel
log_levelslower_model_managerr;   r<   items_generation_statejsonloads
isinstancedicttyperBadParameterr	   r   r   _chat_handler_completion_handler_response_handler_transcription_handlerConfigServerserverstart_serverrun)'selfr   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   rK   r4   r5   r6   r7   r8   r9   r:   transformers_loggerr<   kv	cb_kwargsrB   appconfigs'                                          r   __init__zServe.__init__*   s   D "#rssB9750?2 #<( &00@$$W%7%7	8I%JK*/ 3%'#
 	:
 ,+$7&;"3 eg

1 } qD

	 

 >G,9y9D	!0 3"
  #'::.B#C 2D9(()WXX#% (((6: !23-+++6; !232--!33!5
 $5--!33$
 
 "1--!33!5"
 ';4;N;NPTPfPf&g##77!33"&"="=!33#
 $TVLnnV,KKOOC

s   	Jc                 ~      fd}t        j                  |dd       _         j                  j                          y )Nc                      t        j                         } t        j                  |        | j                  j                  j                                y )N)asyncionew_event_loopset_event_looprun_until_completerf   serve)loopri   s    r   _runz Serve.start_server.<locals>._run   s:    ))+D""4(##DKK$5$5$78r   zuvicorn-threadF)targetnamedaemon)	threadingThread_threadstart)ri   ry   s   ` r   rg   zServe.start_server   s2    	9
 !''t:JSXYr   c                 8    | j                   j                          y)z$Clear all loaded models from memory.N)rW   shutdownri   s    r   reset_loaded_modelszServe.reset_loaded_models   s    $$&r   c                    | j                   j                          | j                  j                          | j                  r| j                  j	                         sy d| j
                  _        | j                  j                  d       y )NT   )timeout)rY   r   rW   r   is_aliverf   should_exitjoinr   s    r   kill_serverzServe.kill_server   s`    '')$$&||4<<#8#8#:"&!$r   )r   r   r   r	   r   valuer   strr^   ArgumentboolOptionintfloatrp   rg   r   r   r   r   r   r   r   )   s    qu
  di  $$ lrpvch      LWIMUZbkX\ AcsTz>5>>?k+llmc 'ELLmnp
	c '$J*^__
c 43_!``ac  $J*Z[[
c ELL/	
c2 ($JELLm
3cD #|u||1ghhiEcF t\U\\7k%llmGcH %T<5<<=^+_%_`IcJ !#lmm
KcR !$J*bcc
ScX !$J*^__
Yc^ '$J*]^^
_cd  )DL,%,,,]^^ 
ecj %4K+XYY
kcr \U\\/GHHIsct \U\\/DEEFucv t\U\\7P%QQRwcx S,%,,4]"^^_ycz  d
LELL>S,T TU{c|  ,%,,d1dee
}cB 
CcJ'%r   r   u  
Run a FastAPI server to serve models on-demand with an OpenAI compatible API.
Models will be loaded and unloaded automatically based on usage and a timeout.


Endpoints:
    POST /v1/chat/completions — Chat completions (streaming + non-streaming).
    POST /v1/completions      — Legacy text completions from a prompt.
    GET  /v1/models           — Lists available models.
    GET  /health              — Health check.

Requires FastAPI and Uvicorn: pip install transformers[serving]
)__doc__rs   enumrZ   r}   typingr   r^   transformers.utilsr   transformers.utils.import_utilsr   rR   r   rS   r   loggerr   Enumr	   r   r   r   r   <module>r      sc          & > ) 
		H	%C y% y%xr   