Ë
    ÛÓ^j%  ã                   óÒ   — d Z ddlZddlZddlZddlZddlmZ ddlZddlm	Z	 ddl
mZ ddlmZ  e	j                  e«      Z G d„ d	eej$                  «      Z G d
„ d«      Zde_         y)z+
CLI entry point for `transformers serve`.
é    N)Ú	Annotated)Úlogging)Úis_serve_availableé   )Úset_torch_seedc                   ó   — e Zd ZdZdZdZy)ÚReasoningModeÚonÚoffÚautoN)Ú__name__Ú
__module__Ú__qualname__ÚONÚOFFÚAUTO© ó    úa/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/cli/serve.pyr	   r	   #   s   „ Ø	€BØ
€CØ�Dr   r	   c            2       óä  — e Zd Zdddddej                  j
                  ddddddddddddddddfdeedz   ej                  d	¬
«      f   dee
 ej                  d¬
«      f   deedz   ej                  d¬
«      f   dee
 ej                  d¬
«      f   deedz   ej                  d¬
«      f   dee ej                  d¬
«      f   deedz   ej                  d¬
«      f   dee ej                  d¬
«      f   deedz   ej                  d¬
«      f   dee
 ej                  d¬
«      f   dee ej                  d¬
«      f   deedz   ej                  d ¬
«      f   d!eedz   ej                  d"¬
«      f   d#eedz   ej                  d$¬
«      f   d%eedz   ej                  d&¬
«      f   d'ee
dz   ej                  d(¬
«      f   d)ee ej                  d*¬
«      f   d+ee ej                  d,¬
«      f   d-ee
 ej                  d.¬
«      f   d/ee ej                  d0¬
«      f   d1eedz   ej                  d2¬
«      f   d3ee
 ej                  d4d5¬6«      f   d7df.d8„Zd9„ Zd:„ Zd;„ Zy)<ÚServeNFr   i,  Ú	localhosti@  ÚwarningÚforce_modelz*Model to preload and use for all requests.)ÚhelpÚcontinuous_batchingzMEnable continuous batching with paged attention. Configure with --cb-* flags.Úattn_implementationz2Attention implementation (e.g. flash_attention_2).Úcompilez*Enable torch.compile for faster inference.Úquantizationz.Quantization method: 'bnb-4bit' or 'bnb-8bit'.Ú	reasoninguÀ   Reasoning mode. 'auto' uses the chat template default. Only applies to models that support reasoning via their chat template (e.g. Qwen3, Gemma 4) â€” for other models this flag has no effect.Úchat_template_kwargsz‰Default JSON kwargs forwarded to apply_chat_template (e.g. '{"enable_thinking": true}'); per-request chat_template_kwargs override these.Údevicez4Device for inference (e.g. 'auto', 'cuda:0', 'cpu').Údtypez2Override model dtype. 'auto' derives from weights.Útrust_remote_codezTrust remote code when loading.Úmodel_timeoutzGSeconds before idle model is unloaded. Ignored when force_model is set.Úcb_block_sizez6KV cache block size in tokens for continuous batching.Úcb_num_blocksz2Number of KV cache blocks for continuous batching.Úcb_max_batch_tokensz1Maximum tokens per batch for continuous batching.Úcb_max_memory_percentz/Max GPU memory fraction for KV cache (0.0-1.0).Úcb_use_cuda_graphz+Enable CUDA graphs for continuous batching.ÚhostzServer listen address.ÚportzServer listen port.Úenable_corszEnable permissive CORS.Ú	log_levelz'Logging level (e.g. 'info', 'warning').Údefault_seedzDefault torch seed.Únon_blockingTz1Run server in a background thread. Used by tests.)Úhiddenr   Úreturnc           	      ó:  — t        «       st        d«      ‚dd l}ddlm} ddlm} ddlm} ddl	m
} ddlm} dd	lm} dd
lm} |�t#        |«       t%        j&                  d«      }|j)                  t$        j*                  |j-                  «          «        |||	|
||||¬«      | _        ddlm}  |||||dœj5                  «       D �!�"ci c]
  \  }!}"|"�|!|"“Œ }#}!}"|#r | di |#¤Žnd }$ ||||$¬«      | _        |r:t9        j:                  |«      }t=        |t>        «      stA        jB                  d«      ‚i }|tD        jF                  k(  rd|d<   n|tD        jH                  k(  rd|d<    || j.                  | j6                  |¬«      | _%         || j.                  | j6                  ¬«      | _&         || j.                  | j6                  |¬«      | _'         || j.                  | j6                  «      | _(         || j.                  | jJ                  | jL                  | jN                  | jP                  | j6                  |¬«      }%|jS                  |%||d¬«      }&|jU                  |&«      | _+        |r| jY                  «        y | jV                  j[                  «        y c c}"}!w )NzRMissing dependencies for serving. Install with `pip install transformers[serving]`r   r   )ÚChatCompletionHandler)ÚCompletionHandler)ÚModelManager)ÚResponseHandler)Úbuild_server)ÚTranscriptionHandler)ÚGenerationStateÚtransformers)r"   r#   r$   r   r   r%   r   )ÚContinuousBatchingConfig)Ú
block_sizeÚ
num_blocksÚmax_batch_tokensÚmax_memory_percentÚuse_cuda_graph)r   r   Ú	cb_configz,--chat-template-kwargs must be a JSON objectTÚenable_thinkingF)Úmodel_managerÚgeneration_stater!   )rD   rE   )Úcompletion_handlerÚresponse_handlerÚtranscription_handlerrE   r-   Úinfo)r+   r,   r.   r   ).r   ÚImportErrorÚuvicornÚserving.chat_completionr4   Úserving.completionr5   Úserving.model_managerr6   Úserving.responser7   Úserving.serverr8   Úserving.transcriptionr9   Úserving.utilsr:   r   r   Ú
get_loggerÚsetLevelÚ
log_levelsÚlowerÚ_model_managerr;   r<   ÚitemsÚ_generation_stateÚjsonÚloadsÚ
isinstanceÚdictÚtyperÚBadParameterr	   r   r   Ú_chat_handlerÚ_completion_handlerÚ_response_handlerÚ_transcription_handlerÚConfigÚServerÚserverÚstart_serverÚrun)'Úselfr   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   rK   r4   r5   r6   r7   r8   r9   r:   Útransformers_loggerr<   ÚkÚvÚ	cb_kwargsrB   ÚappÚconfigs'                                          r   Ú__init__zServe.__init__*   s‹  € ôD "Ô#ÜÐrÓsÐsãåBÝ9Ý7Ý5Ý0Ý?Ý2ð Ð#Ü˜<Ô(ô &×0Ñ0°Ó@ÐØ×$Ñ$¤W×%7Ñ%7¸	¿¹Ó8IÑ%JÔKá*ØØØ/Ø 3Ø%Ø'Ø#ô
ˆÔõ 	:ð
 ,Ø+Ø$7Ø&;Ø"3ñ÷ ‰e‹g÷

á��1ð ˆ}ð ˆq‰Dð

ˆ	ñ 

ñ >GÑ,Ñ9¨yÒ9ÈDˆ	Ù!0Ø 3ØØô"
ˆÔñ  Ü#'§:¡:Ð.BÓ#CÐ ÜÐ2´DÔ9Ü×(Ñ(Ð)WÓXÐXà#%Ð àœ×(Ñ(Ò(Ø6:Ð Ð!2Ò3Øœ-×+Ñ+Ò+Ø6;Ð Ð!2Ñ3á2Ø×-Ñ-Ø!×3Ñ3Ø!5ô
ˆÔñ $5Ø×-Ñ-Ø!×3Ñ3ô$
ˆÔ ñ
 "1Ø×-Ñ-Ø!×3Ñ3Ø!5ô"
ˆÔñ ';¸4×;NÑ;NÐPT×PfÑPfÓ&gˆÔ#áØ×ÑØ×ÑØ#×7Ñ7Ø!×3Ñ3Ø"&×"=Ñ"=Ø!×3Ñ3Ø#ô
ˆð —‘ ¨$°TÀV�ÓLˆØ—n‘n VÓ,ˆŒáØ×ÑÕà�K‰K�O‰OÕùóC

s   Ã	Jc                 ó~   ‡ — ˆ fd„}t        j                  |dd¬«      ‰ _        ‰ j                  j                  «        y )Nc                  óª   •— t        j                  «       } t        j                  | «       | j                  ‰j                  j                  «       «       y )N)ÚasyncioÚnew_event_loopÚset_event_loopÚrun_until_completerf   Úserve)Úloopri   s    €r   Ú_runz Serve.start_server.<locals>._runÐ   s:   ø€ Ü×)Ñ)Ó+ˆDÜ×"Ñ" 4Ô(Ø×#Ñ# D§K¡K×$5Ñ$5Ó$7Õ8r   zuvicorn-threadF)ÚtargetÚnameÚdaemon)Ú	threadingÚThreadÚ_threadÚstart)ri   ry   s   ` r   rg   zServe.start_serverÏ   s2   ø€ ô	9ô
 !×'Ñ'¨tÐ:JÐSXÔYˆŒØ�‰×ÑÕr   c                 ó8   — | j                   j                  «        y)z$Clear all loaded models from memory.N)rW   Úshutdown©ri   s    r   Úreset_loaded_modelszServe.reset_loaded_modelsØ   s   € à×Ñ×$Ñ$Õ&r   c                 ó  — | j                   j                  «        | j                  j                  «        | j                  r| j                  j	                  «       sy d| j
                  _        | j                  j                  d¬«       y )NTé   )Útimeout)rY   r‚   rW   r   Úis_aliverf   Úshould_exitÚjoinrƒ   s    r   Úkill_serverzServe.kill_serverÜ   s`   € Ø×Ñ×'Ñ'Ô)Ø×Ñ×$Ñ$Ô&Ø�|Š| 4§<¡<×#8Ñ#8Ô#:ØØ"&ˆ�‰ÔØ�‰×Ñ !ÐÕ$r   )r   r   r   r	   r   Úvaluer   Ústrr^   ÚArgumentÚboolÚOptionÚintÚfloatrp   rg   r„   r‹   r   r   r   r   r   )   sç  „ ð quð
 ð Ødið ð ×Ñ×$Ñ$ð ØlrØpvØchð ð ð ð ð ð àLWØIMØUZØbkØX\ð ñAcà˜s T™z¨>¨5¯>©>Ð?kÔ+lÐlÑmðcð 'ØØˆE�L‰LÐmÔnðpñ
ð	cð 'Ø�$‰J˜˜Ÿ™Ð*^Ô_Ð_ñ
ðcð ˜4  §¡Ð3_Ô!`Ð`Ñaðcð  Ø�$‰J˜˜Ÿ™Ð*ZÔ[Ð[ñ
ðcð ØØˆE�L‰Lð/ôðñ	
ðcð2 (Ø�$‰JØˆE�L‰Lðmôðñ
ð3cðD ˜#˜|˜uŸ|™|Ð1gÔhÐhÑiðEcðF ˜˜t™ \ U§\¡\Ð7kÔ%lÐlÑmðGcðH % T¨<¨5¯<©<Ð=^Ô+_Ð%_Ñ`ðIcðJ !Ø��—‘Ð#lÔmÐmñ
ðKcðR !Ø�$‰J˜˜Ÿ™Ð*bÔcÐcñ
ðScðX !Ø�$‰J˜˜Ÿ™Ð*^Ô_Ð_ñ
ðYcð^ 'Ø�$‰J˜˜Ÿ™Ð*]Ô^Ð^ñ
ð_cðd  )Ø�D‰L˜,˜%Ÿ,™,Ð,]Ô^Ð^ñ 
ðecðj %Ø�4‰K˜˜Ÿ™Ð+XÔYÐYñ
ðkcðr ˜˜\˜UŸ\™\Ð/GÔHÐHÑIðscðt ˜˜\˜UŸ\™\Ð/DÔEÐEÑFðucðv ˜t \ U§\¡\Ð7PÔ%QÐQÑRðwcðx ˜S , %§,¡,Ð4]Ô"^Ð^Ñ_ðycðz    d¡
¨L¨E¯L©LÐ>SÔ,TÐ TÑUð{cð|  Ø�,�%—,‘, dÐ1dÔeÐeñ
ð}cðB 
óCcòJò'ó%r   r   uð  
Run a FastAPI server to serve models on-demand with an OpenAI compatible API.
Models will be loaded and unloaded automatically based on usage and a timeout.


Endpoints:
    POST /v1/chat/completions â€” Chat completions (streaming + non-streaming).
    POST /v1/completions      â€” Legacy text completions from a prompt.
    GET  /v1/models           â€” Lists available models.
    GET  /health              â€” Health check.

Requires FastAPI and Uvicorn: pip install transformers[serving]
)Ú__doc__rs   ÚenumrZ   r}   Útypingr   r^   Útransformers.utilsr   Útransformers.utils.import_utilsr   rR   r   rS   r   Úloggerr�   ÚEnumr	   r   r   r   r   ú<module>rš      sc   ðñó Û Û Û Ý ã å &Ý >å )ð 
ˆ×	Ñ	˜HÓ	%€ô�C˜Ÿ™ô ÷y%ñ y%ðx€…r   