Ë
    ÛÓ^j3"  ã                   ó¦  — d dl Z d dlmZ d dlmZ d dlmZmZ d dlm	Z	 d dl
Z
d dlmZ d dlmZ dd	lmZmZmZ  G d
„ d«      Ze G d„ d«      «       Zdedefd„Zdedededefd„Zd+dedededefd„Zdedededefd„Z	 d,de
j6                  dee   dee   deddf
d „Zd!ed"ed#ed$ed%e	dee   fd&„Zd'e j>                  dee   fd(„Z d)„ Z!ed*„ «       Z"y)-é    N)Úcontextmanager)Ú	dataclass)ÚceilÚlog2)ÚAny)ÚPretrainedConfig)Úis_torch_greater_or_equalé   )ÚFutureRequestStateÚRequestStateÚRequestStatusc                   ó®   — e Zd ZdZdd„Zdd„Zdeedf   dej                  j                  dz  fd„Zdeedf   d	ej                  j                  ddfd
„Zy)ÚCudaGraphBufferz`A dict for CUDA graphs with a special __del__ method to make sure the graphs are properly reset.ÚreturnNc                 ó   — i | _         y ©N©Ú_storage)Úselfs    ú|/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/generation/continuous_batching/utils.pyÚ__init__zCudaGraphBuffer.__init__   s	   € ØEGˆ�ó    c                 ó’   — | j                   r;| j                   j                  «       \  }}|j                  «        | j                   rŒ:y y r   )r   ÚpopitemÚreset)r   Ú_Úgraphs      r   Ú__del__zCudaGraphBuffer.__del__"   s1   € Ø�mŠmØ—}‘}×,Ñ,Ó.‰HˆAˆuØ�K‰KŒMð �m�mr   Úkey.c                 ó8   — | j                   j                  |«      S r   )r   Úget)r   r   s     r   Ú	get_graphzCudaGraphBuffer.get_graph'   s   € Ø�}‰}× Ñ  Ó%Ð%r   r   c                 ó"   — || j                   |<   y r   r   )r   r   r   s      r   Ú	set_graphzCudaGraphBuffer.set_graph*   s   € Ø"ˆ�‰�cÒr   )r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtupleÚintÚtorchÚcudaÚ	CUDAGraphr"   r$   © r   r   r   r      sf   „ ÙjóHóð
&˜U 3¨ 8™_ð &°·±×1EÑ1EÈÑ1Ló &ð#˜U 3¨ 8™_ð #°U·Z±Z×5IÑ5Ið #Èdô #r   r   c                   ó<   — e Zd ZU dZdZeed<   dZeed<   dZeed<   y)ÚWorkloadHintszRA tiny dataclass containing hints to help choose good continuous batching defaultsr   Úmax_prompt_lengthÚmax_generated_lengthÚnum_requestsN)	r%   r&   r'   r(   r1   r*   Ú__annotations__r2   r3   r.   r   r   r0   r0   .   s%   … á\àÐ�sÓØ !Ð˜#Ó!Ø€L�#Ôr   r0   Úconfigr   c                 ó   — | j                   dv S )z:Checks if attention mask is needed for the given (config).)zpaged|eagerz
paged|sdpa)Ú_attn_implementation)r5   s    r   Úattn_mask_is_neededr8   7   s   € à×&Ñ&Ð*GÐGÐGr   ÚsizeÚinterval_sizeÚ	max_valuec                 óX   — |dk  r|S | dkD  rt        | |z  «      |z  n|}t        ||«      S )zQReturn the smallest multiple of (interval_size) >= (size), capped at (max_value).r   )r   Úmin)r9   r:   r;   Úpaddeds       r   Úpad_to_intervalr?   <   s9   € à˜ÒØÐØ;?À!º8ŒT�$˜Ñ&Ó'¨-Ò7È€FÜˆv�yÓ!Ð!r   ÚvalueÚ	min_valuec                 ó†   — t        | t        d|«      «      } dt        t        t        | «      «      «      z  }t	        ||«      S )z‘Return the smallest power of 2 >= (value), capped at (max_value). If a minimum value is provided, the value is at
    least padded to that value.r
   é   )Úmaxr*   r   r   r=   )r@   r;   rA   r>   s       r   Úpad_to_pow2rE   D   s<   € ô �”s˜1˜iÓ(Ó)€EØ”#”dœ4 ›;Ó'Ó(Ñ(€FÜˆv�yÓ!Ð!r   ÚxÚ	divide_byÚalign_toc                 óT   — t        t        | |z  «      «      } | |z  r| || |z  z
  z  } | S r   )r*   r   )rF   rG   rH   s      r   Úaligned_dividerJ   L   s4   € ÜŒD��Y‘ÓÓ €AØˆ8‚|Ø	ˆX˜˜X™Ñ&Ñ&ˆØ€Hr   Úattention_maskÚcumulative_seqlens_qÚcumulative_seqlens_kÚsliding_windowc                 óJ  — t        j                  | j                  «      j                  }t	        t        |«      dz
  «      D ]à  }||dz      ||   z
  }||dz      ||   z
  }||k  r|dk\  r	||z
  dz   }nd}t        ||   ||dz      «      }	t        ||   ||dz      «      }
t        j                  | d|	|
f   j                  || j                  | j                  ¬«      }t        j                  ||¬«      }|dkD  r"||z
  |z
  }|t        j                  ||¬«      z  }|| d|	|
f<   Œâ y)u  Builds an attention mask inplace using the cumulative seqlens of the query and key. If given a sliding window, it
    will also apply a sliding window mask on top. The attention mask is not boolean, it uses zeroes and -inf (or its
    equivalent) so it's more of an attention score bias tensor.
    The attention mask is a block-diagonal matrix, with each block an attention mask for a single query-key pair.
    Each of those block is built from a causal mask and, if there is a sliding window, a sliding window mask.

    An example is represented below, with seqlen_k = 8, seqlen_q = 4 and sliding_window = 6:

    CAUSAL MASK:

           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘ â–‘ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ

    SLIDING WINDOW MASK:
         â”Œâ”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€ seqlen_k - seqlen_q - sliding_window = 8 - 4 - 6 = -2 offset to the left
       <â”€â”´â”€>
     â–‘ â–ˆ | â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ
     â–‘ â–‘ | â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ
     â–‘ â–‘ | â–‘ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ
     â–‘ â–‘ | â–‘ â–‘ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ

    ATTENTION MASK (sum of causal and sliding window masks):

           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘ â–‘ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘ â–‘
           â–‘ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–‘
           â–‘ â–‘ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ

    Another example with seqlen_k = 5, seqlen_q = 3 and sliding_window = 2:

    CAUSAL MASK:

           â–ˆ â–ˆ â–ˆ â–‘ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–‘
           â–ˆ â–ˆ â–ˆ â–ˆ â–ˆ

    SLIDING WINDOW MASK:
         â”Œâ”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€ seqlen_k - seqlen_q - sliding_window = 5 - 3 - 2 = 0 offset to the left
        <â”´>
         | â–‘ â–ˆ â–ˆ â–ˆ â–ˆ
         | â–‘ â–‘ â–ˆ â–ˆ â–ˆ
         | â–‘ â–‘ â–‘ â–ˆ â–ˆ

    ATTENTION MASK (sum of causal and sliding window masks):

           â–‘ â–ˆ â–ˆ â–‘ â–‘
           â–‘ â–‘ â–ˆ â–ˆ â–‘
           â–‘ â–‘ â–‘ â–ˆ â–ˆ

    r
   .)ÚdtypeÚdevice)ÚdiagonalN)r+   ÚfinforP   r=   ÚrangeÚlenÚsliceÚfullÚshaperQ   ÚtriuÚtril)rK   rL   rM   rN   rA   ÚiÚseqlen_qÚseqlen_kÚcausal_diagonalÚquery_rangeÚ	key_rangeÚ	minus_infÚmaskedÚsliding_diagonals                 r   Úbuild_attention_maskrd   S   sS  € ôt —‘˜N×0Ñ0Ó1×5Ñ5€IÜ”3Ð+Ó,¨qÑ0Ó1ò =ˆØ'¨¨A©Ñ.Ð1EÀaÑ1HÑHˆØ'¨¨A©Ñ.Ð1EÀaÑ1HÑHˆØ�hÒ 8¨q¢=Ø&¨Ñ1°AÑ5‰OàˆOÜÐ0°Ñ3Ð5IÈ!ÈaÉ%Ñ5PÓQˆÜÐ.¨qÑ1Ð3GÈÈAÉÑ3NÓOˆ	ä—J‘JØ˜3 ¨YÐ6Ñ7×=Ñ=ØØ ×&Ñ&Ø!×(Ñ(ô	
ˆ	ô —‘˜I°Ô@ˆà˜AÒØ'¨(Ñ2°^ÑCÐØ”e—j‘j Ð5EÔFÑFˆFà6<ˆ�s˜K¨Ð2Ò3ñ-=r   ÚnumÚstatusÚnum_q_tokensÚmax_kv_readÚcachec           	      ó†  — t        | «      D �cg c]  }d|j                  › d|› d�‘Œ }}||z   }t        ||j                  z  «      }g }	|D ]n  }
t	        |
dg|z  d¬«      }||_        dg|z  |_        ||_        |j                  ||j                  d«      }|€|	c S |	j                  t        |dd|¬«      «       Œp |	S c c}w )	zPA utility function to create a list of FutureRequestStates for the warmup of CB.Ú	__warmup_r   Ú__r   r
   )Ú
request_idÚinitial_tokensÚmax_new_tokensT)Úhas_new_tokenÚcomplete_blocksÚquery_length)rT   Únamer   Ú
block_sizer   Ú_statusÚtokens_to_processÚposition_offsetÚallocate_blocksrm   Úappendr   )re   rf   rg   rh   ri   r[   Úrequest_idsÚtotal_tokensÚblocks_neededÚfuture_statesÚreq_idÚstateÚ	allocateds                r   Úcreate_warmup_future_statesr�   §   sç   € ô =BÀ#»JÖG°q�Y˜vŸ{™{˜m¨1¨Q¨C¨rÒ2ÐG€KÐGØ +Ñ-€LÜ˜¨×(8Ñ(8Ñ8Ó9€Mà€MØò 
ˆÜ¨À¸sÀ\Ñ?QÐbcÔdˆØˆŒØ#$ #¨Ñ"4ˆÔØ +ˆÔà×)Ñ)¨-¸×9IÑ9IÈ1ÓMˆ	ØÐØ Ò Ø×ÑÜ˜u°DÈ!ÐZfÔgõ	
ð
ð Ðùò# Hs   ŽB>Úrequest_queuec                 óÈ   — g }| j                  «       s5	 | j                  «       }|�|j                  |«       | j                  «       sŒ5|S # t        j                  $ r Y |S w xY w)z3Drains a queue and returns a list of RequestStates.)ÚemptyÚ
get_nowaitry   ÚqueueÚEmpty)r‚   Ú
new_statesr   s      r   Údrain_queuer‰   Ä   sn   € à%'€JØ×!Ñ!Ô#ð	Ø!×,Ñ,Ó.ˆEØÐ Ø×!Ñ! %Ô(ð	 ×!Ñ!Õ#ð Ðøô �{‰{ò 	ØØÐð	ús   ”#A
 Á
A!Á A!c                  ó¼   — t        d«      r.t        j                  j                  «       } | j                  }| |fS d} t        j                  j                  «       }| |fS )z®Returns a tuple of (mem_pool, graph_pool_id) for CUDA graphs. Since the MemPool object is only available in torch
    2.5+, we only return a graph_pool_id for older versions.z2.5.0N)r	   r+   r,   ÚMemPoolÚidÚgraph_pool_handle)Úmem_poolÚgraph_pool_ids     r   Úget_cuda_poolsr�   Ñ   sT   € ô ! Ô)Ü—:‘:×%Ñ%Ó'ˆØ Ÿ™ˆØ˜Ð&Ð&àˆÜŸ
™
×4Ñ4Ó6ˆØ˜Ð&Ð&r   c              #   óŠ   K  — | �-t         j                  j                  | «      5  d–— ddd«       yd–— y# 1 sw Y   yxY w­w)z’A context manager to use a CUDA mem pool. If the mem pool is None, it is a no-op. No type hint because it would
    make torch 2.4 or below crash.N)r+   r,   Úuse_mem_pool)rŽ   s    r   Úmem_pool_ctxr“   Þ   sA   è ø€ ð ÐÜ�Z‰Z×$Ñ$ XÓ.ñ 	Û÷	ð 	ô 	÷	ð 	üs   ‚"A¤7©A·A ¼A)r   )r
   )#r†   Ú
contextlibr   Údataclassesr   Úmathr   r   Útypingr   r+   Ú transformers.configuration_utilsr   Útransformers.utilsr	   Úrequestsr   r   r   r   r0   Úboolr8   r*   r?   rE   rJ   ÚTensorÚlistrd   r�   ÚQueuer‰   r�   r“   r.   r   r   ú<module>rŸ      s…  ðó Ý %Ý !ß Ý ã å =Ý 8ç EÑ E÷#ñ #ð$ ÷ð ó ððHÐ 0ð H°Tó Hð
"˜#ð "¨cð "¸cð "Àcó "ñ"�sð " sð "°sð "À3ó "ð�cð  cð °Sð ¸Só ð ñ	Q=Ø—L‘LðQ=à˜s™)ðQ=ð ˜s™)ðQ=ð ð	Q=ð
 
óQ=ðhØ	ðàðð ðð ð	ð
 ðð 
Ð
Ñóð:
˜uŸ{™{ð 
¨t°LÑ/Aó 
ò
'ð ñó ñr   