
    ^jE                     b   d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z
  e
       r4ddlmZmZ dd	lmZmZmZ dd
lmZ ddlmZmZmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% erddl&m'Z'm(Z(m)Z)m*Z*  G d ded      Z+h dZ, ejZ                  e.      Z/ G d de      Z0y)z
Handler for the /v1/chat/completions endpoint.

Supports streaming (SSE via DirectStreamer) and non-streaming (JSON) responses.
    N)AsyncGenerator)TYPE_CHECKING   )logging)is_serve_available)JSONResponseStreamingResponse)ChatCompletionChatCompletionMessageChatCompletionMessageToolCall)Choice)ChatCompletionChunkChoiceDeltaChoiceDeltaToolCall)CompletionCreateParamsStreaming)CompletionUsage   )	BaseGenerateManagerBaseHandlerModalityReasoningText_StreamErrorget_reasoning_configget_tool_call_configparse_reasoningparse_tool_calls)GenerationConfigPreTrainedModelPreTrainedTokenizerFastProcessorMixinc                   ,    e Zd ZU eed<   eed<   eed<   y)+TransformersCompletionCreateParamsStreaminggeneration_configseedchat_template_kwargsN)__name__
__module____qualname__str__annotations__intdict     s/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/cli/serving/chat_completion.pyr"   r"   @   s    
Ir.   r"   F)total>   nuseraudiostorelogprobsmetadata	functions
modalities
predictiontool_choiceservice_tiertop_logprobsfunction_callstream_optionsresponse_formatpresence_penaltyreasoning_effortweb_search_optionsparallel_tool_callsmax_completion_tokensc                       e Zd ZdZeZeZdede	de
ez  fdZ	 	 dde	ddd	d
de	dedddededz  dedz  de
fdZ	 	 dde	ddd	d
de	dedddededz  dedz  defdZd dedddef fdZ	 	 	 d!de	de	de	de	dedz  dee   dz  de	dz  defdZ	 	 	 	 	 	 	 	 d"de	de	dz  de	dz  de	dz  de	dz  dedz  dedz  de	dz  de	fdZ xZS )#ChatCompletionHandlerz|Handler for the `/v1/chat/completions` endpoint.

    Supports both streaming (SSE) and non-streaming (JSON) responses.
    body
request_idreturnc                 2  K   | j                  |       | j                  |      \  }}}| j                  j                  ||      }| j                  j                  ||      }t        j                  d| d|        | j                  j                  ||      }| j                  |d   |      }	t        d |	D              }
i }|
rd|d<   |j                  | j                         |j                  |j                  d	i               |j                  |	fd
|j                  d      |rdndd
d
|t        j                   k(  xr |
d|}|s|j#                  |j$                        }| j'                  ||j(                  |      }|r|j+                  ||       |j                  d      rt-        ||      nd}t/        |||d         }|j                  d      }|r| j1                  |||||||||	      S | j3                  |||||||||	       d{   S 7 w)a  Validate the request, load the model, and dispatch to streaming or non-streaming.

        Args:
            body (`dict`): The raw JSON request body (OpenAI chat completion format).
            request_id (`str`): Unique request identifier (from header or auto-generated).

        Returns:
            `StreamingResponse | JSONResponse`: SSE stream or JSON depending on ``body["stream"]``.
        )	processorz[Request received] Model: z, CB: use_cbmessagesc              3      K   | ]O  }t        |j                  d       t              r|j                  d       ng D ]  }|j                  d      dk(    Q yw)contenttypevideoN)
isinstancegetlist).0msgcs      r/   	<genexpr>z7ChatCompletionHandler.handle_request.<locals>.<genexpr>   sX      
,6swwy7I4,Pcggi(VX
  EE&MW$
$
s   AA    
num_framesr%   TtoolsNpt)add_generation_promptr\   return_tensorsreturn_dicttokenizeload_audio_from_video	input_idsstream)gen_managertool_configreasoning_config)_validate_request_resolve_modelmodel_managerget_model_modalitygeneration_stateuse_continuous_batchingloggerwarningget_manager"get_processor_inputs_from_messagesanyupdater%   rT   apply_chat_templater   
MULTIMODALtodevice_build_generation_configr#   init_cbr   r   
_streaming_non_streaming)selfrG   rH   model_idmodelrK   modalityrM   re   processor_inputs	has_videor%   inputs
gen_configrf   rg   	streamings                    r/   handle_requestz$ChatCompletionHandler.handle_requestm   sT     	t$%)%8%8%>"%%%88)8T&&>>uhO3H:VF8LM++777PBB4
CSU]^ 
'
 
	 &(13 .##D$=$=>##DHH-CR$HI...	
"&((7##)4t"*h.A.A"A"Oi	
 #	
 YYu||,F2249P9PY_2`
z2@D@Q*9e<W[/	5&BUVHHX&	??''!1 # 
 
 ,,''!1 - 
 
 
 
s   HHHHNr~   r   rK   z(ProcessorMixin | PreTrainedTokenizerFastr}   r   r   r   re   rf   rg   c
           
          |j                  |||	      \  |d   }
t        |
t              rt        |
      n|
j                  d   dt
        t        df   f f	d}t         |       d      S )	z(Stream tokens as SSE via DirectStreamer.)rH   rf   rg   rc   rI   Nc                   	K   	 j                  d       d} | s=j                          d {   }|g}	 	 |j                  j                                 d}rdt        j                  d
         }|rHd}t        |      D ]8  \  }}j                  t        |d d| |d   |d   d      g       : j                  d uxr j                   j                  k\  }|rd}	n|rd}	nd}	t#        j                   j                   z         }
j                  |	|
       y 7 # t        j
                  $ r Y nw xY wg }|D ]  }|d}  nt        |t              r5|j                  d|j                   d       dj                  |        y t        |t              r$|j                  j                  |             |j                  j                  |	              |rdj                  |       | s# t$        t        j&                  f$ r j)                           w xY ww)N	assistant)roler~   FTzdata: {"error": "z"}

 )r~   reasoning_content)r~   rP   schemafunction_tool_call_name	argumentsr   r   )indexrQ   idr   )r~   
tool_callsr   lengthstopprompt_tokenscompletion_tokenstotal_tokens)finish_reasonr~   usage)_build_chunk_sserT   append
get_nowaitasyncio
QueueEmptyrS   r   rW   joinr   r   generated_token_ids	enumerater   max_new_tokensr   r   GeneratorExitCancelledErrorcancel)donetextbatch	sse_partshas_tool_callsparseditchit_maxr   r   r   	input_lenr}   rK   queuerH   r|   streamerrf   s              r/   sse_genz1ChatCompletionHandler._streaming.<locals>.sse_gen   s    J++J[PX+YY!&,D!FE"!LL)9)9);< #2 "'-i9U9UWbckWlmF)-%.v%6 EAr"&"7"7 *&.$7./-7.8\QC+H:<V*SUVaSb1c	%&," #8 #  %334?vHDYDY]g]v]vDv!$0M$,M$*M'"+&.&;&;!*X-B-B!B
 ++"/"	 ,  w -
 #--  ,.I % n<#'D!%dL9%,,/A$((7-ST"$'')"44"%dM:%,,T-B-B:U]qu-B-vw%,,T-B-B:U]gk-B-lmn ! ggi003 D "7#9#9:  !	s^   I	.H D8H !D; CH 7I	8H ;EH EAH %I	&A1H +II	ztext/event-stream
media_type)generate_streamingrS   rU   lenshaper   r)   r	   )r|   rH   r~   rK   r}   r   r   re   rf   rg   rc   r   r   r   r   s   `` `` ` `   @@@r/   rz   z ChatCompletionHandler._streaming   s     &88!#- 9 
x ;'	&0D&AC	NyWYGZ	K	~c4i8 K	 K	Z !7JKKr.   c
                   K   |j                  |||||       d{   \  }
}}|j                  duxr t        |      |j                  k\  }t        |      }t        ||||z         }d}|It	        |||d         }|r7t        |      D cg c]!  \  }}t        | d| d|d   |d   d	
      # }}}d}|	t        |||
|	      \  }
}|d}n|rd}nd}t        | j                  ||
|||||      d      S 7 c c}}w w)z)Run generation and return a JSONResponse.)rH   Nr   r   r   r   r   r   r   )r   rQ   r   r   r   r   )r   r   r   r   zapplication/jsonr   )
generate_non_streamingr   r   r   r   r   r   r   r   _build_completion)r|   rH   r~   rK   r}   r   r   re   rf   rg   rP   r   generated_idsr   r   r   r   r   r   r   r   r   s                         r/   r{   z$ChatCompletionHandler._non_streaming(  sw     3>2T2T9fjZ 3U 3
 -
)M ++47kC<NR\RkRk<k.#/"%66
 
"%iH@UVF "+6!2 2 2(\QC8'*,V*2k?!S
  !')8MSZ\l)m&G&!(M$M"M""+%"3 #  *
 	
I-
 s#   DC:A0D&C<3AD<Dmodel_generation_configrM   c                    t         |   |||      }|j                  d      t        |d         |_        |j                  d      dt        |d         z   |_        |j                  d      6|d   j                         D ci c]  \  }}t        |      f| c}}|_        |j                  d      
|d   |_	        |S c c}}w )zApply Chat Completions params (``max_tokens``, ``frequency_penalty``, ``logit_bias``,
        ``stop``) on top of the base generation config.rL   
max_tokensfrequency_penaltyg      ?
logit_biasr   )
superrx   rT   r+   r   floatrepetition_penaltyitemssequence_biasstop_strings)r|   rG   r   rM   r#   kv	__class__s          r/   rx   z.ChatCompletionHandler._build_generation_configh  s     "G<TCZci<j88L!-/243E/F,88'(436tDW?X9Y3Y088L!-HL\HZH`H`Hb.c1Ay!|.c+88F'-1&\*  	 /ds   CrP   r   r   r   r   c                     t        j                  |d||d      }t        |t        t	        j                               d|t        d||      g|      }	|	j                  d      S )	a  Build a non-streaming ChatCompletion response dict.

        Args:
            request_id (`str`): Unique request identifier.
            content (`str`): The generated text.
            model_id (`str`): Model ID to include in the response.
            finish_reason (`str`): Why generation stopped (``"stop"``, ``"length"``, ``"tool_calls"``).
            usage (`CompletionUsage`, *optional*): Token usage statistics.
            tool_calls (`list[dict]`, *optional*): Parsed tool calls, if any.
            reasoning_content (`str`, *optional*): Chain-of-thought content extracted from the response.

        Returns:
            `dict`: Serialized ``ChatCompletion`` ready for JSON response.
        r   rP   r   r   r   zchat.completionr   )r   messager   )r   createdobjectr~   choicesr   T)exclude_none)r   model_validater
   r+   timer   
model_dump)
r|   rH   rP   r}   r   r   r   r   r   results
             r/   r   z'ChatCompletionHandler._build_completionz  sl    4 (66Jevw
  		$$!WMRS
   d 33r.   r   c	           
          t        j                  ||||d      }	t        |t        t	        j                               |t        |	d|      g|dd      }
| j                  |
      S )a  Build a streaming ``ChatCompletionChunk`` and format it as an SSE ``data:`` line.

        Args:
            request_id (`str`): Unique request identifier.
            content (`str`, *optional*): Text content delta.
            model (`str`, *optional*): Model ID.
            role (`str`, *optional*): Role (only sent in the first chunk).
            finish_reason (`str`, *optional*): Set on the final chunk.
            tool_calls (`list`, *optional*): Tool call deltas.
            usage (`CompletionUsage`, *optional*): Token usage (sent with the final chunk).
            reasoning_content (`str`, *optional*): Reasoning/thinking delta (OpenAI-compatible extension).

        Returns:
            `str`: A formatted SSE event string.
        r   r   )deltar   r   r   zchat.completion.chunk)r   r   r~   r   r   system_fingerprintr   )r   r   r   r+   r   ChoiceChunkchunk_to_sse)r|   rH   rP   r~   r   r   r   r   r   r   chunks              r/   r   z&ChatCompletionHandler._build_chunk_sse  sl    8 **Z^op
 $		$ uA]ST!*
   ''r.   )NN)F)NNN)r   NNNNNNN)r&   r'   r(   __doc__r"   _valid_params_classUNUSED_CHAT_COMPLETION_FIELDS_unused_fieldsr,   r)   r	   r   r   r   rz   r{   boolrx   r   rU   r   r   __classcell__)r   s   @r/   rF   rF   d   sc   
 F2NL L3 LCTWcCc Lr $((,gLgL !gL >	gL
 gL gL 'gL )gL D[gL +gL 
gLh $((,<
<
 !<
 >	<

 <
 <
 '<
 )<
 D[<
 +<
 
<
@!T !L^ !hl !0 )-(,(,%4%4 %4 	%4
 %4 %%4 J%%4 :%4 
%4R " $("&(,(,(((( t(( Tz	((
 Dj(( Tz(( 4K(( %(( :(( 
((r.   rF   )1r   r   r   collections.abcr   typingr   utilsr   utils.import_utilsr   fastapi.responsesr   r	   openai.types.chatr
   r   r   !openai.types.chat.chat_completionr   'openai.types.chat.chat_completion_chunkr   r   r   r   *openai.types.chat.completion_create_paramsr   openai.types.completion_usager   r   r   r   r   r   r   r   r   r   transformersr   r   r   r    r"   r   
get_loggerr&   rn   rF   r-   r.   r/   <module>r      s      *    4 A 
 9 
 NZ=
 
 
 gg2QY^ ! 0 
		H	%e(K e(r.   