
    ^jD                        d dl Z d dlmZ d dlmZmZ d dlmZ d dlZddl	m
Z
mZ ddlmZ  e
       rd dlZdZ ej                   d	      Zej$                  rJ ej&                         Zej+                   ej,                  d
             ej/                  e       de_        deej2                  eeef   fdZ G d de      Ze G d d             Ze G d d             Z G d d      Zy)    N)deepcopy)	dataclassfield)IntEnum   )is_psutil_availableis_torch_xpu_available)loggingContinuousBatchingLoggerz4%(asctime)s - %(name)s - %(levelname)s - %(message)sFreturnc                     t         j                  j                         rt        j                  d      } t         j                  j	                          t         j                  j                          t         j                  j                  |       \  }}t         j                  j                  |       }||z
  }nt               rt        j                  d      } t         j                  j	                          t         j                  j                          t         j                  j                  |       j                  }t         j                  j                  |       }t         j                  j                  |       }nGt         j                  j                  j                         rt         j                  j                  j                         rpt        j                  d      } t         j                  j!                         }t         j                  j#                         }t         j                  j%                         }nt        j                  d      } t'               rMt)        j*                         j,                  }t)        j.                         j1                         j2                  }|}nt4        j7                  d       d}d}d}| |||fS )NcudaxpumpscpuzCannot get memory breakdown on CPU without psutil: returning 0 for all memory values. Please install psutil to get an actual memory breakdown.r   )torchr   is_availabledeviceempty_cachesynchronizemem_get_infomemory_reservedr	   r   get_device_propertiestotal_memorymemory_allocatedbackendsr   is_builtrecommended_max_memorycurrent_allocated_memorydriver_allocated_memoryr   psutilvirtual_memorytotalProcessmemory_inforssloggererror)r   free_memoryr   reserved_memoryallocated_memorys        /var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/generation/continuous_batching/requests.pyget_device_and_memory_breakdownr.   +   s   zz f%

 

  %*JJ$;$;F$C!\**44V<'+5		!e$				yy66v>KK))33F; 9955f=				(	(	*u~~/A/A/J/J/Le$yy779 99==?));;=e$ !00288L%~~/;;=AA.OLL< LO <2BBB    c                   $    e Zd ZdZdZdZdZdZdZy)RequestStatusz5Status of a generation request through its lifecycle.r         r      N)	__name__
__module____qualname____doc__PENDING
PREFILLINGDECODINGFINISHEDFAILED r/   r-   r1   r1   S   s    ?GJHHFr/   r1   c                   &   e Zd ZU dZeed<    ee      Zee	   ed<    ee      Z
ee	   ed<    ee      Zee   ed<   dZedz  ed<   ej                  Zeed	<    eej$                        Zeed
<   dZeeef   ed<   dZee   dz  ed<   defdZy)GenerationOutputa  Tracks the output of a generation request.

    Attributes:
        request_id (str): The ID of the generation request.
        prompt_ids (list[int]): The IDs of the prompt tokens.
        generated_tokens (list[int]): The generated tokens.
        logprobs (list[float]): The log probabilities of the generated tokens.
        error (Optional[str]): Any error message associated with the request. When None, the request was successful.
        status (RequestStatus): The status of the request.
        created_time (float): The time the request was created.
        lifespan (tuple[float, float]): The time the request was no longer pending and the time the request finished.
    
request_iddefault_factory
prompt_idsgenerated_tokenslogprobsNr)   statuscreated_timer   r   lifespan
timestampsr   c                 <    | j                   t        j                  k(  S N)rG   r1   r<   selfs    r-   is_finishedzGenerationOutput.is_finishedv   s    {{m4444r/   )r5   r6   r7   r8   str__annotations__r   listrD   intrE   rF   floatr)   r1   r9   rG   timeperf_counterrH   rJ   tuplerK   boolrP   r>   r/   r-   r@   r@   ]   s     O!$7JS	7"'"=d3i=!$7Hd5k7E3:)11FM10A0ABL%B$,HeE5L!,%)JUd")5T 5r/   r@   c                      e Zd ZU dZeed<   ee   ed<   dZe	ed<   dZ
e	ed<   dZedz  ed	<   dZeee   z  dz  ed
<   dZeed<    ee      Zeed<    ee      Zee   ed<    ee      Zee   ed<    ee      Zee   ed<   dZeed<   dZeed<   ej0                  Zeed<    ee      Zee   ed<    eej:                        Zeed<   dZedz  ed<   dZ e!eef   ed<    ee      Z"ee   ed<   dZ#eed<   dZ$eed<    ee      Z%ee   ed<   dZ&e	ed<   d  Z'e(d!efd"       Z)e)jT                  d#efd$       Z)e(d!ee   dz  fd%       Z+d& Z,d!efd'Z-d!efd(Z.d)ed*edz  d!e	fd+Z/d, Z0d- Z1d.ed!d fd/Z2d!efd0Z3d2d1Z4y)3RequestStatea  Tracks the state of a generation request through its lifecycle.

    Attributes:
        request_id (str): The ID of the generation request.
        initial_tokens (list[int]): The initial prompt tokens.
        num_children (int): The number of children requests
        full_prompt_ids (list[int] | None): The tokens IDs of the full prompt.
        prompt_ids (list[int] | None): The tokens IDs currently being processed.
        remaining_prompt_ids (list[int]): The initial tokens IDs remaining to be processed.
        static_outputs (list[int]): The generated tokens.
        allocated_blocks (int): The number of blocks allocated to the request.
        position_offset (int): The current position in the sequence for position_ids.
        status (RequestStatus): The status of the request: can be one of PENDING, PREFILLING, PREFILLING_SPLIT,
                                SPLIT_PENDING_REMAINDER, DECODING, FINISHED, FAILED
        max_new_tokens (int | None): The maximum number of new tokens to generate.
        eos_token_id (None | int | list[int]): The ID(s) of the end-of-sequence tokens. Only used in post-init.
        _eos_token_ids (set[int]): The IDs of the end-of-sequence tokens, formatted as a set.
        streaming (bool): Whether to stream tokens as they're generated
        created_time (float): The time the request was created.
        error (Optional[str]): Any error message associated with the request. When None, has had no error yet.
    rA   initial_tokensF	streamingrecord_timestamps   Nmax_new_tokenseos_token_idr   num_childrenrB   logit_processor_kwargstokens_to_processrE   rF   position_offsetallocated_blocks_status_eos_token_idsrH   r)   rI   rJ   _timestamps_true_initial_tokens_new_tokens_limitremaining_prefill_tokensis_cpu_offloadedc                    | j                   dn| j                   | _        | j                  d d  | _        | j                  y t        | j                  t              r6| j                  dk\  r&| j                  j                  | j                         y y | j                  D ]#  }|dk\  s	| j                  j                  |       % y )Nrk   r   )	r`   rl   r\   rm   ra   
isinstancerT   rh   add)rO   token_ids     r-   __post_init__zRequestState.__post_init__   s    /3/B/B/JPTPcPc(,(;(;A(>%$))3/  A%##''(9(9: & !-- 6q=''++H56r/   r   c                     | j                   S rM   )rg   rN   s    r-   rG   zRequestState.status   s    ||r/   valuec                 n   | j                   t        j                  k(  r#t        j                         df| _        || _         y |t        j                  k(  r[| j
                  d   t        j                         f| _        t        j                  t        j                        r| j                          || _         y )Nr   r   )rg   r1   r9   rV   rW   rJ   r<   r(   isEnabledForr
   DEBUGlog_end_of_request)rO   ru   s     r-   rG   zRequestState.status   s    <<=000!..0"5DM
 	 m,,,!]]1-t/@/@/BCDM""7==1'')r/   c                 6    | j                   r| j                  S d S rM   )r^   ri   rN   s    r-   rK   zRequestState.timestamps   s    #'#9#9tCtCr/   c                    t        | j                        }| j                         }| j                  d   | j                  z
  }| j                  d   | j                  z
  }t
        j                  d| j                   d|d|d|d|
       y )Nr   r2   Request z finished: prefill_len = z decode_len = z start_time = z end_time = )lenr\   generated_lenrJ   rH   r(   debugrA   )rO   prefill_len
decode_len
start_timeend_times        r-   ry   zRequestState.log_end_of_request   s    $--.'')
]]1%(9(99
==#d&7&77t''A;2B/J?RaT^Sbbodlcpq	
r/   c                     | j                   S )zCGet the current length of the sequence (prompt + generated tokens).)re   rN   s    r-   current_lenzRequestState.current_len   s    ###r/   c                 ,    t        | j                        S )z*Get the number of tokens generated so far.)r}   rE   rN   s    r-   r~   zRequestState.generated_len   s    4(())r/   rr   logprobc                 4   | j                   t        j                  k7  ry| j                  r-| j                  j                  t        j                                || j                  v }| j                         }|s|| j                  k  rF| j                  j                  |       |g| _        |dz  }|A| j                  j                  |       n%t        j                  d| j                    d|        |s|| j                  k\  rt        j"                  | _         yy)zUpdate the request with a newly generated token (and optional log probability of the token) and check for
        completion. Returns True if the request is now complete, False otherwise.Fr2   r|   z generated a useless token: T)rG   r1   r;   r^   ri   appendrV   rW   rh   r~   rl   rE   rd   rF   r(   warningrA   r<   )rO   rr   r   is_eosr   s        r-   update_and_check_completionz(RequestState.update_and_check_completion   s     ;;-000 !!##D$5$5$78 T000((* kD$:$::!!((2&.ZD"1K"$$W-NNXdoo%66RS[R\]^[D$:$::'00DKr/   c                    d| j                    d| j                   d| j                          dt        | j                         dt        | j
                         d| j                   dt        | j                         d| j                   d	| j                   d
| j                   g
}ddj                  |      z   dz   S )Nzrequest_id=zstatus=zout_tokens=zquery_length=zremaining_tokens=z
kv_length=zfull_prompt_length=zallocated_blocks=zgenerated_tokens=zlogit_processor_kwargs=zRequestState(
	z,
	z
))rA   rg   r~   r}   rd   rm   re   r\   rf   rE   rc   join)rO   msgs     r-   __repr__zRequestState.__repr__  s    $//*+dll^$$,,./0C 6 6789D$A$A BCD--./!#d&9&9":!;< 5 567 5 567%d&A&A%BC
 #W\\#%66>>r/   c                    | j                   r@| j                  | j                   d | j                  z   }| j                  d| j                    }n| j                  dd }| j                  }t        | j                  ||| j
                  dd | j                  | j                  | j                  | j                  | j                  | j                  dd 	      S d	      S )z7Convert the request state to a GenerationOutput object.N)	rA   rD   rE   rF   r)   rG   rH   rJ   rK   )rj   r\   rE   r@   rA   rF   r)   rG   rH   rJ   rK   )rO   rE   rD   s      r-   to_generation_outputz!RequestState.to_generation_output  s    $$#2243L3L3NORVRgRgg,,-Ht/H/HIJ#44Q7,,J!-]]1%**;;**]]-1__-Htq)

 
	
 OS

 
	
r/   new_request_idc                     t        |       }||_        t        j                         |_        |j                  df|_        g |_        | j                  dd |_        |S )ziFork the request into a new request with the same state except for request_id, created_time and lifespan.r   N)r   rA   rV   rW   rH   rJ   ri   rm   )rO   r   new_requests      r-   forkzRequestState.fork/  sX    tn!/#'#4#4#6  + 8 8"="$/3/L/LQ/O,r/   c                     | j                   | j                  | j                  | j                  | j                  t        | j                        dS )zXGet all the fields necessary to create a request that would have the same configuration.)r]   r^   r`   ra   rb   rc   )r]   r^   r`   ra   rb   r   rc   rN   s    r-   get_request_configzRequestState.get_request_config;  sG     !%!7!7"11 -- --&.t/J/J&K
 	
r/   c           
         | j                         }| j                  %| j                  t        | j                        z
  |d<   t	        d| j
                  | j                  | j                  z   | j                  dd | j                  t        | j                        z   d|}| j                  r| j                  |_        |S t        | j                        |_        |S )aT  Creates an equivalent new request by removing the generated tokens and adding them to the initial prompt. The
        created request has THE SAME request_id. Notably, we can retrieve the original request from the created one with
        the _true_initial_tokens attribute. The logprobs of the generated tokens are kept in the new request.Nr`   )rA   r\   rF   rj   r>   )	r   r`   r}   rE   r[   rA   r\   rF   rj   )rO   request_config	new_states      r-   !create_equivalent_initial_requestz.RequestState.create_equivalent_initial_requestF  s    
 002*/3/B/BSI^I^E_/_N+,  
..1F1FF]]1%!%!:!:SATAT=U!U	

 
	 $$-1-F-FI*  .11D1D-EI*r/   )r   r[   )5r5   r6   r7   r8   rQ   rR   rS   rT   r]   rY   r^   r`   ra   rb   r   dictrc   rd   rE   rF   rU   re   rf   r1   r9   rg   setrh   rV   rW   rH   r)   rJ   rX   ri   rj   rl   rm   rn   rs   propertyrG   setterrK   ry   r   r~   r   r   r   r   r   r   r>   r/   r-   r[   r[   z   s7   . OI It#t# "$NC$J#+/L#S	/D(/L##(#>D> $)#>tCy>"'"=d3i=!$7Hd5k7OSc*22G]2$S9NCH9  0A0ABL%BE3:$,HeE5L!,$T:Ke: !#! (s'*/*Ed3iE"d"6$    ]]M   DDK$. D D
$S $*s *
C %$, SW >?
(
3 
> 
	
D 	
r/   r[   c            
       0    e Zd ZdZdZdededededdf
d	Zy)
FutureRequestStatezPTracks the current state of a request and the relevant information to update it.statehas_new_tokencomplete_blocksquery_lengthr   r   r   r   r   Nc                 <    || _         || _        || _        || _        y rM   r   )rO   r   r   r   r   s        r-   __init__zFutureRequestState.__init__f  s!    
*.(r/   )	r5   r6   r7   r8   	__slots__r[   rY   rT   r   r>   r/   r-   r   r   `  s8    Z NI)l )4 )RU )eh )mq )r/   r   ) rV   copyr   dataclassesr   r   enumr   r   utilsr   r	   utils.loggingr
   r"   TMP_TOKEN_ID	getLoggerr(   	propagateStreamHandlerhandlersetFormatter	Formatter
addHandlerrX   r   rT   r.   r1   r@   r[   r   r>   r/   r-   <module>r      s      (   @ $   
		5	6	#g##%G***+abc
gF%Cu||S#s/J)K %CPG  5 5 58 b b bJ
) 
)r/   