Ë
    ÛÓ^jD  ã                   óÖ  — d dl Z d dlmZ d dlmZmZ d dlmZ d dlZddl	m
Z
mZ ddlmZ  e
«       rd dlZdZ ej                   d	«      Zej$                  rJ ej&                  «       Zej+                   ej,                  d
«      «       ej/                  e«       de_        deej2                  eeef   fd„Z G d„ de«      Ze G d„ d«      «       Ze G d„ d«      «       Z G d„ d«      Zy)é    N)Údeepcopy)Ú	dataclassÚfield)ÚIntEnumé   )Úis_psutil_availableÚis_torch_xpu_available)ÚloggingéÿÿÿÿÚContinuousBatchingLoggerz4%(asctime)s - %(name)s - %(levelname)s - %(message)sFÚreturnc                  ó’  — t         j                  j                  «       r™t        j                  d«      } t         j                  j	                  «        t         j                  j                  «        t         j                  j                  | «      \  }}t         j                  j                  | «      }||z
  }�nt        «       rºt        j                  d«      } t         j                  j	                  «        t         j                  j                  «        t         j                  j                  | «      j                  }t         j                  j                  | «      }t         j                  j                  | «      }�nGt         j                  j                  j                  «       r˜t         j                  j                  j                  «       rpt        j                  d«      } t         j                  j!                  «       }t         j                  j#                  «       }t         j                  j%                  «       }n‡t        j                  d«      } t'        «       rMt)        j*                  «       j,                  }t)        j.                  «       j1                  «       j2                  }|}nt4        j7                  d«       d}d}d}| |||fS )NÚcudaÚxpuÚmpsÚcpuzŽCannot get memory breakdown on CPU without psutil: returning 0 for all memory values. Please install psutil to get an actual memory breakdown.r   )Útorchr   Úis_availableÚdeviceÚempty_cacheÚsynchronizeÚmem_get_infoÚmemory_reservedr	   r   Úget_device_propertiesÚtotal_memoryÚmemory_allocatedÚbackendsr   Úis_builtÚrecommended_max_memoryÚcurrent_allocated_memoryÚdriver_allocated_memoryr   ÚpsutilÚvirtual_memoryÚtotalÚProcessÚmemory_infoÚrssÚloggerÚerror)r   Úfree_memoryr   Úreserved_memoryÚallocated_memorys        ú/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/generation/continuous_batching/requests.pyÚget_device_and_memory_breakdownr.   +   sÝ  € Ü‡z�z×ÑÔ Ü—‘˜fÓ%ˆÜ�
‰
×ÑÔ Ü�
‰
×ÑÔ ô %*§J¡J×$;Ñ$;¸FÓ$CÑ!ˆ�\ÜŸ*™*×4Ñ4°VÓ<ˆØ'¨+Ñ5ÒÜ	Ô	!Ü—‘˜eÓ$ˆÜ�	‰	×ÑÔÜ�	‰	×ÑÔÜ—y‘y×6Ñ6°vÓ>×KÑKˆÜŸ)™)×3Ñ3°FÓ;ˆÜ Ÿ9™9×5Ñ5°fÓ=ÒÜ	�‰×	Ñ	×	(Ñ	(Ô	*¬u¯~©~×/AÑ/A×/JÑ/JÔ/LÜ—‘˜eÓ$ˆÜ—y‘y×7Ñ7Ó9ˆÜ Ÿ9™9×=Ñ=Ó?ÐÜŸ)™)×;Ñ;Ó=‰ä—‘˜eÓ$ˆÜÔ Ü!×0Ñ0Ó2×8Ñ8ˆLÜ%Ÿ~™~Ó/×;Ñ;Ó=×AÑAÐØ.‰Oä�L‰Lð<ôð ˆLØˆOØ Ðà�< Ð2BÐBÐBó    c                   ó$   — e Zd ZdZdZdZdZdZdZy)ÚRequestStatusz5Status of a generation request through its lifecycle.r   é   é   r   é   N)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚPENDINGÚ
PREFILLINGÚDECODINGÚFINISHEDÚFAILED© r/   r-   r1   r1   S   s   „ Ù?à€GØ€JØ€HØ€HØ�Fr/   r1   c                   ó&  — e Zd ZU dZeed<    ee¬«      Zee	   ed<    ee¬«      Z
ee	   ed<    ee¬«      Zee   ed<   dZedz  ed<   ej                  Zeed	<    eej$                  ¬«      Zeed
<   dZeeef   ed<   dZee   dz  ed<   defd„Zy)ÚGenerationOutputa«  Tracks the output of a generation request.

    Attributes:
        request_id (str): The ID of the generation request.
        prompt_ids (list[int]): The IDs of the prompt tokens.
        generated_tokens (list[int]): The generated tokens.
        logprobs (list[float]): The log probabilities of the generated tokens.
        error (Optional[str]): Any error message associated with the request. When None, the request was successful.
        status (RequestStatus): The status of the request.
        created_time (float): The time the request was created.
        lifespan (tuple[float, float]): The time the request was no longer pending and the time the request finished.
    Ú
request_id©Údefault_factoryÚ
prompt_idsÚgenerated_tokensÚlogprobsNr)   ÚstatusÚcreated_time©r   r   ÚlifespanÚ
timestampsr   c                 ó<   — | j                   t        j                  k(  S ©N)rG   r1   r<   ©Úselfs    r-   Úis_finishedzGenerationOutput.is_finishedv   s   € Ø�{‰{œm×4Ñ4Ñ4Ð4r/   )r5   r6   r7   r8   ÚstrÚ__annotations__r   ÚlistrD   ÚintrE   rF   Úfloatr)   r1   r9   rG   ÚtimeÚperf_counterrH   rJ   ÚtuplerK   ÚboolrP   r>   r/   r-   r@   r@   ]   s­   … ñð ƒOÙ!°$Ô7€J��S‘	Ó7Ù"'¸Ô"=Ð�d˜3‘iÓ=Ù!°$Ô7€Hˆd�5‰kÓ7Ø€Eˆ3�‰:ÓØ)×1Ñ1€FˆMÓ1Ù°×0AÑ0AÔB€L�%ÓBØ$,€Hˆe�E˜5�LÑ!Ó,Ø%)€J��U‘˜dÑ"Ó)ð5˜Tô 5r/   r@   c                   ó  — e Zd ZU dZeed<   ee   ed<   dZe	ed<   dZ
e	ed<   dZedz  ed	<   dZeee   z  dz  ed
<   dZeed<    ee¬«      Zeed<    ee¬«      Zee   ed<    ee¬«      Zee   ed<    ee¬«      Zee   ed<   dZeed<   dZeed<   ej0                  Zeed<    ee¬«      Zee   ed<    eej:                  ¬«      Zeed<   dZedz  ed<   dZ e!eef   ed<    ee¬«      Z"ee   ed<   dZ#eed<   dZ$eed<    ee¬«      Z%ee   ed<   dZ&e	ed<   d „ Z'e(d!efd"„«       Z)e)jT                  d#efd$„«       Z)e(d!ee   dz  fd%„«       Z+d&„ Z,d!efd'„Z-d!efd(„Z.d)ed*edz  d!e	fd+„Z/d,„ Z0d-„ Z1d.ed!d fd/„Z2d!efd0„Z3d2d1„Z4y)3ÚRequestStateaÃ  Tracks the state of a generation request through its lifecycle.

    Attributes:
        request_id (str): The ID of the generation request.
        initial_tokens (list[int]): The initial prompt tokens.
        num_children (int): The number of children requests
        full_prompt_ids (list[int] | None): The tokens IDs of the full prompt.
        prompt_ids (list[int] | None): The tokens IDs currently being processed.
        remaining_prompt_ids (list[int]): The initial tokens IDs remaining to be processed.
        static_outputs (list[int]): The generated tokens.
        allocated_blocks (int): The number of blocks allocated to the request.
        position_offset (int): The current position in the sequence for position_ids.
        status (RequestStatus): The status of the request: can be one of PENDING, PREFILLING, PREFILLING_SPLIT,
                                SPLIT_PENDING_REMAINDER, DECODING, FINISHED, FAILED
        max_new_tokens (int | None): The maximum number of new tokens to generate.
        eos_token_id (None | int | list[int]): The ID(s) of the end-of-sequence tokens. Only used in post-init.
        _eos_token_ids (set[int]): The IDs of the end-of-sequence tokens, formatted as a set.
        streaming (bool): Whether to stream tokens as they're generated
        created_time (float): The time the request was created.
        error (Optional[str]): Any error message associated with the request. When None, has had no error yet.
    rA   Úinitial_tokensFÚ	streamingÚrecord_timestampsé   NÚmax_new_tokensÚeos_token_idr   Únum_childrenrB   Úlogit_processor_kwargsÚtokens_to_processrE   rF   Úposition_offsetÚallocated_blocksÚ_statusÚ_eos_token_idsrH   r)   rI   rJ   Ú_timestampsÚ_true_initial_tokenséÿÿÿÚ_new_tokens_limitÚremaining_prefill_tokensÚis_cpu_offloadedc                 óˆ  — | j                   €dn| j                   | _        | j                  d d  | _        | j                  €y t        | j                  t        «      r6| j                  dk\  r&| j                  j                  | j                  «       y y | j                  D ]#  }|dk\  sŒ	| j                  j                  |«       Œ% y )Nrk   r   )	r`   rl   r\   rm   ra   Ú
isinstancerT   rh   Úadd)rO   Útoken_ids     r-   Ú__post_init__zRequestState.__post_init__·   s¯   € à/3×/BÑ/BÐ/J¡ÐPT×PcÑPcˆÔà(,×(;Ñ(;¹AÐ(>ˆÔ%à×ÑÐ$Øä˜×)Ñ)¬3Ô/Ø× Ñ  AÒ%Ø×#Ñ#×'Ñ'¨×(9Ñ(9Õ:ð &ð !×-Ñ-ò 6�Ø˜q“=Ø×'Ñ'×+Ñ+¨HÕ5ñ6r/   r   c                 ó   — | j                   S rM   )rg   rN   s    r-   rG   zRequestState.statusÉ   s   € à�|‰|Ðr/   Úvaluec                 ón  — | j                   t        j                  k(  r#t        j                  «       df| _        || _         y |t        j                  k(  r[| j
                  d   t        j                  «       f| _        t        j                  t        j                  «      r| j                  «        || _         y )Nr   r   )rg   r1   r9   rV   rW   rJ   r<   r(   ÚisEnabledForr
   ÚDEBUGÚlog_end_of_request)rO   ru   s     r-   rG   zRequestState.statusÍ   s…   € à�<‰<œ=×0Ñ0Ò0Ü!×.Ñ.Ó0°"Ð5ˆDŒMð
 ˆ�ð	 ”m×,Ñ,Ò,Ø!Ÿ]™]¨1Ñ-¬t×/@Ñ/@Ó/BÐCˆDŒMÜ×"Ñ"¤7§=¡=Ô1Ø×'Ñ'Ô)Øˆ�r/   c                 ó6   — | j                   r| j                  S d S rM   )r^   ri   rN   s    r-   rK   zRequestState.timestamps×   s   € à#'×#9Ò#9ˆt×ÑÐC¸tÐCr/   c                 ó  — t        | j                  «      }| j                  «       }| j                  d   | j                  z
  }| j                  d   | j                  z
  }t
        j                  d| j                  › d|›d|›d|›d|›�
«       y )Nr   r2   úRequest z finished: prefill_len = z decode_len = z start_time = z end_time = )Úlenr\   Úgenerated_lenrJ   rH   r(   ÚdebugrA   )rO   Úprefill_lenÚ
decode_lenÚ
start_timeÚend_times        r-   ry   zRequestState.log_end_of_requestÛ   s�   € Ü˜$×-Ñ-Ó.ˆØ×'Ñ'Ó)ˆ
Ø—]‘] 1Ñ%¨×(9Ñ(9Ñ9ˆ
Ø—=‘= Ñ# d×&7Ñ&7Ñ7ˆÜ�‰Ø�t—‘Ð'Ð'A°;Ð2BÀ/ÀJÀ?ÐRaÐT^ÐSbÐboÐdlÐcpÐqõ	
r/   c                 ó   — | j                   S )zCGet the current length of the sequence (prompt + generated tokens).)re   rN   s    r-   Úcurrent_lenzRequestState.current_lenä   s   € à×#Ñ#Ð#r/   c                 ó,   — t        | j                  «      S )z*Get the number of tokens generated so far.)r}   rE   rN   s    r-   r~   zRequestState.generated_lenè   s   € ä�4×(Ñ(Ó)Ð)r/   rr   Úlogprobc                 ó4  — | j                   t        j                  k7  ry| j                  r-| j                  j                  t        j                  «       «       || j                  v }| j                  «       }|s|| j                  k  rF| j                  j                  |«       |g| _        |dz  }|�A| j                  j                  |«       n%t        j                  d| j                   › d|› �«       |s|| j                  k\  rt        j"                  | _         yy)z»Update the request with a newly generated token (and optional log probability of the token) and check for
        completion. Returns True if the request is now complete, False otherwise.Fr2   r|   z generated a useless token: T)rG   r1   r;   r^   ri   ÚappendrV   rW   rh   r~   rl   rE   rd   rF   r(   ÚwarningrA   r<   )rO   rr   r‡   Úis_eosr…   s        r-   Úupdate_and_check_completionz(RequestState.update_and_check_completioní   s÷   € ð �;‰;œ-×0Ñ0Ò0Øð ×!Ò!Ø×Ñ×#Ñ#¤D×$5Ñ$5Ó$7Ô8ð ˜T×0Ñ0Ð0ˆØ×(Ñ(Ó*ˆñ �k D×$:Ñ$:Ò:Ø×!Ñ!×(Ñ(¨Ô2Ø&. ZˆDÔ"Ø˜1ÑˆKØÐ"Ø—‘×$Ñ$ WÕ-ä�N‰N˜X d§o¡oÐ%6Ð6RÐS[ÐR\Ð]Ô^á�[ D×$:Ñ$:Ò:Ü'×0Ñ0ˆDŒKØØr/   c                 óŠ  — d| j                   › �d| j                  › �d| j                  «       › �dt        | j                  «      › �dt        | j
                  «      › �d| j                  › �dt        | j                  «      › �d| j                  › �d	| j                  › �d
| j                  › �g
}ddj                  |«      z   dz   S )Nzrequest_id=zstatus=zout_tokens=zquery_length=zremaining_tokens=z
kv_length=zfull_prompt_length=zallocated_blocks=zgenerated_tokens=zlogit_processor_kwargs=zRequestState(
	z,
	z
))rA   rg   r~   r}   rd   rm   re   r\   rf   rE   rc   Újoin)rO   Úmsgs     r-   Ú__repr__zRequestState.__repr__  sÛ   € à˜$Ÿ/™/Ð*Ð+Ø�d—l‘l�^Ð$Ø˜$×,Ñ,Ó.Ð/Ð0ØœC × 6Ñ 6Ó7Ð8Ð9Ø¤ D×$AÑ$AÓ BÐCÐDØ˜×-Ñ-Ð.Ð/Ø!¤# d×&9Ñ&9Ó":Ð!;Ð<Ø × 5Ñ 5Ð6Ð7Ø × 5Ñ 5Ð6Ð7Ø% d×&AÑ&AÐ%BÐCð
ˆð # W§\¡\°#Ó%6Ñ6¸Ñ>Ð>r/   c                 ó¶  — | j                   r@| j                  | j                   d | j                  z   }| j                  d| j                    }n| j                  dd }| j                  }t        | j                  ||| j
                  dd | j                  | j                  | j                  | j                  | j                  �| j                  dd ¬«	      S d¬«	      S )z7Convert the request state to a GenerationOutput object.N)	rA   rD   rE   rF   r)   rG   rH   rJ   rK   )rj   r\   rE   r@   rA   rF   r)   rG   rH   rJ   rK   )rO   rE   rD   s      r-   Úto_generation_outputz!RequestState.to_generation_output  sÐ   € à×$Ò$Ø#×2Ñ2°4×3LÑ3LÐ3NÐOÐRV×RgÑRgÑgÐØ×,Ñ,Ð-H¨t×/HÑ/HÐI‰Jà#×4Ñ4±QÐ7ÐØ×,Ñ,ˆJÜØ—‘Ø!Ø-Ø—]‘]¡1Ð%Ø—*‘*Ø—;‘;Ø×*Ñ*Ø—]‘]Ø-1¯_©_Ð-H�t—‘¡qÐ)ô

ð 
	
ð OSô

ð 
	
r/   Únew_request_idc                 ó¸   — t        | «      }||_        t        j                  «       |_        |j                  df|_        g |_        | j                  dd |_        |S )ziFork the request into a new request with the same state except for request_id, created_time and lifespan.r   N)r   rA   rV   rW   rH   rJ   ri   rm   )rO   r“   Únew_requests      r-   ÚforkzRequestState.fork/  sX   € ä˜t“nˆà!/ˆÔÜ#'×#4Ñ#4Ó#6ˆÔ Ø +× 8Ñ 8¸"Ð=ˆÔØ"$ˆÔà/3×/LÑ/LÉQÐ/OˆÔ,ØÐr/   c                 óž   — | j                   | j                  | j                  | j                  | j                  t        | j                  «      dœS )zXGet all the fields necessary to create a request that would have the same configuration.)r]   r^   r`   ra   rb   rc   )r]   r^   r`   ra   rb   r   rc   rN   s    r-   Úget_request_configzRequestState.get_request_config;  sG   € ð Ÿ™Ø!%×!7Ñ!7Ø"×1Ñ1Ø ×-Ñ-Ø ×-Ñ-Ü&.¨t×/JÑ/JÓ&Kñ
ð 	
r/   c           
      ó¶  — | j                  «       }| j                  �%| j                  t        | j                  «      z
  |d<   t	        d| j
                  | j                  | j                  z   | j                  dd | j                  t        | j                  «      z   dœ|¤Ž}| j                  r| j                  |_        |S t        | j                  «      |_        |S )aT  Creates an equivalent new request by removing the generated tokens and adding them to the initial prompt. The
        created request has THE SAME request_id. Notably, we can retrieve the original request from the created one with
        the _true_initial_tokens attribute. The logprobs of the generated tokens are kept in the new request.Nr`   )rA   r\   rF   rj   r>   )	r˜   r`   r}   rE   r[   rA   r\   rF   rj   )rO   Úrequest_configÚ	new_states      r-   Ú!create_equivalent_initial_requestz.RequestState.create_equivalent_initial_requestF  sÔ   € ð
 ×0Ñ0Ó2ˆà×ÑÐ*Ø/3×/BÑ/BÄSÈ×I^ÑI^ÓE_Ñ/_ˆNÐ+Ñ,ä ð 
Ø—‘Ø×.Ñ.°×1FÑ1FÑFØ—]‘]¡1Ð%Ø!%×!:Ñ!:¼SÀ×ATÑATÓ=UÑ!Uñ	
ð
 ñ
ˆ	ð ×$Ò$Ø-1×-FÑ-FˆIÔ*ð Ðô .1°×1DÑ1DÓ-EˆIÔ*ØÐr/   )r   r[   )5r5   r6   r7   r8   rQ   rR   rS   rT   r]   rY   r^   r`   ra   rb   r   Údictrc   rd   rE   rF   rU   re   rf   r1   r9   rg   Úsetrh   rV   rW   rH   r)   rJ   rX   ri   rj   rl   rm   rn   rs   ÚpropertyrG   ÚsetterrK   ry   r…   r~   rŒ   r�   r’   r–   r˜   rœ   r>   r/   r-   r[   r[   z   s7  … ñð. ƒOØ˜‘IÓð €IˆtÓØ#Ð�tÓ#ð "$€N�C˜$‘JÓ#Ø+/€L�#˜˜S™	‘/ DÑ(Ó/Ø€L�#ÓÙ#(¸Ô#>Ð˜DÓ>ñ $)¸Ô#>Ð�t˜C‘yÓ>Ù"'¸Ô"=Ð�d˜3‘iÓ=Ù!°$Ô7€Hˆd�5‰kÓ7Ø€O�SÓØÐ�cÓà*×2Ñ2€Gˆ]Ó2Ù$°SÔ9€N�C˜‘HÓ9ñ  °×0AÑ0AÔB€L�%ÓBØ€Eˆ3�‰:ÓØ$,€Hˆe�E˜5�LÑ!Ó,Ù$°TÔ:€K��e‘Ó:Ø !Ð˜#Ó!ð (Ð�sÓ'Ù*/ÀÔ*EÐ˜d 3™iÓEØ"Ð�dÓ"ò6ð$ ð˜ò ó ðð ‡]�]ð˜Mò ó ðð ðD˜D ™K¨$Ñ.ò Dó ðDò
ð$˜Só $ð*˜só *ð
°Cð À%È$Á,ð ÐSWó ò>?ò
ð(
 3ð 
¨>ó 
ð	
 Dó 	
ôr/   r[   c            
       ó0   — e Zd ZdZdZdededededdf
d	„Zy)
ÚFutureRequestStatezPTracks the current state of a request and the relevant information to update it.©ÚstateÚhas_new_tokenÚcomplete_blocksÚquery_lengthr¤   r¥   r¦   r§   r   Nc                 ó<   — || _         || _        || _        || _        y rM   r£   )rO   r¤   r¥   r¦   r§   s        r-   Ú__init__zFutureRequestState.__init__f  s!   € ØˆŒ
Ø*ˆÔØ.ˆÔØ(ˆÕr/   )	r5   r6   r7   r8   Ú	__slots__r[   rY   rT   r©   r>   r/   r-   r¢   r¢   `  s8   „ ÙZð N€Ið)˜lð )¸4ð )ÐRUð )Ðehð )Ðmqô )r/   r¢   ) rV   Úcopyr   Údataclassesr   r   Úenumr   r   Úutilsr   r	   Úutils.loggingr
   r"   ÚTMP_TOKEN_IDÚ	getLoggerr(   Ú	propagateÚStreamHandlerÚhandlerÚsetFormatterÚ	FormatterÚ
addHandlerrX   r   rT   r.   r1   r@   r[   r¢   r>   r/   r-   ú<module>r¸      sü   ðó Ý ß (Ý ã ç @Ý $ñ ÔÛð €ð 
ˆ×	Ñ	Ð5Ó	6€à	×ÒØ#ˆg×#Ñ#Ó%€GØ×ÑÐ*˜×*Ñ*Ð+aÓbÔcØ
×Ñ�gÔØ€FÔð%C¨¨u¯|©|¸SÀ#ÀsÐ/JÑ)Kó %CôP�Gô ð ÷5ð 5ó ð5ð8 ÷bð bó ðb÷J
)ò 
)r/   