Ë
    ÛÓ^jÁ  ã                   ó&  — d Z ddlmZ ddlZddlZddlmZ ddlmc mZ	 ddl
mZ ddlmZmZmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlm Z m!Z!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7m8Z8m9Z9m:Z:m;Z;  e#jx                  e=«      Z>dKde?fd„Z@	 dLdej‚                  de?de?de?fd„ZBdej‚                  dej†                  de?deDd e?d!ej‚                  fd"„ZEd#dejŒ                  fd$ej‚                  d%e?d&e?d'eGd(e?d)ej�                  d!eIej‚                  ej‚                  f   fd*„ZJd+ej‚                  d,e?dz  d!ej‚                  fd-„ZK G d.„ d/e2«      ZL G d0„ d1e3«      ZM G d2„ d3e-«      ZN G d4„ d5e0«      ZO G d6„ d7e4«      ZP G d8„ d9e1«      ZQe! G d:„ d;e/«      «       ZR G d<„ d=eR«      ZS G d>„ d?eR«      ZT G d@„ dAeR«      ZU G dB„ dCeR«      ZV G dD„ dEeR«      ZW e!dF¬G«       G dH„ dIeRe«      «       ZXg dJ¢ZYy)Mz<Blt modular model, inheriting from Mllama where appropriate.é    )ÚCallableNé   )Úinitialization)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_causal_mask)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)Údynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Úrotate_half)ÚLlamaRotaryEmbedding)ÚMllamaPreTrainedModelÚMllamaSelfAttentionDecoderLayerÚMllamaTextCrossAttentionÚMllamaTextMLPÚMllamaTextRMSNormÚMllamaTextSelfAttentionÚeager_attention_forwardé   )Ú	BltConfigÚBltGlobalTransformerConfigÚBltLocalDecoderConfigÚBltLocalEncoderConfigÚBltPatcherConfigÚprimec                 óü   — t        j                  |t         j                  | j                  ¬«      }t        j                  | j
                  d   | j                  ¬«      }||z  }t        j                  | |z  d¬«      S )aœ  
    A polynomial rolling hash algorithm that converts sequences
    of tokens into hash values. The hash is computed as:
        hash = (token_0 * prime^0 + token_1 * prime^1 + ... + token_n * prime^n)

    The rolling hash allows the model to efficiently
    identify and encode recurring byte-level patterns in the input text.

    Args:
        token_tensor (torch.Tensor): [batch_size, seq_len, group_size] containing token IDs to hash
        prime (int): Prime number used as the base for the polynomial hash.

    Returns:
        torch.Tensor: Hash values of shape [batch_size, seq_len] where each value
                     represents the hash of the corresponding token group

    Example:
        >>> tokens = torch.tensor([[1, 2, 3], [4, 5, 6]])
        >>> hashes = rolling_polynomial_hash(tokens, prime=31)
        >>> # hash[0] = 1*31^0 + 2*31^1 + 3*31^2
        >>> # hash[1] = 4*31^0 + 5*31^1 + 6*31^2
    ©ÚdtypeÚdeviceéÿÿÿÿ©r-   ©Údim)ÚtorchÚtensorÚint64r-   ÚarangeÚshapeÚsum)Útoken_tensorr)   Úprime_tensorÚpowersÚprime_powerss        ún/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/blt/modular_blt.pyÚrolling_polynomial_hashr=   9   sa   € ô. —<‘< ¬U¯[©[À×ATÑATÔU€LÜ�\‰\˜,×,Ñ,¨RÑ0¸×9LÑ9LÔM€FØ Ñ'€LÜ�9‰9�\ LÑ0°bÔ9Ð9ó    Ú	token_idsÚ
group_sizeÚmax_hashc                 óZ  — t        j                  «       5  | j                  \  }}t        j                  ||dz
  t         j                  | j
                  ¬«      }t        j                  || gd¬«      }|j                  d|d«      }t        ||«      }	|	|z  }
ddd«       |
S # 1 sw Y   
S xY w)z1Hash token groups and map to range [0, max_hash].r#   r+   r0   N)	r2   Úno_gradr6   Úzerosr4   r-   ÚcatÚunfoldr=   )r?   r@   r)   rA   Ú
batch_sizeÚseq_lenÚpaddingÚpadded_tokensÚwindowsÚhashesÚhash_valuess              r<   Úbyte_group_hash_functionrN   V   sŸ   € ô 
�‰‹ñ 	(Ø'Ÿo™oÑˆ
�Gä—+‘+˜j¨*°q©.ÄÇÁÐT]×TdÑTdÔeˆÜŸ	™	 7¨IÐ"6¸AÔ>ˆð  ×&Ñ& q¨*°aÓ8ˆÜ(¨°%Ó8ˆØ˜xÑ'ˆ÷	(ð Ð÷	(ð Ðús   •BB Â B*Úlocal_encoder_tokensÚencoder_hash_tok_embeddingÚ$encoder_hash_byte_group_nb_functionsÚencoder_hash_byte_group_sizeÚencoder_hash_byte_group_vocabÚreturnc                 ó  — g d¢}|j                  | «      }d}t        |«      D ]Y  }	||	t        |«      z     }
|D ]A  }t        | ||
|«      }|||z  z   }| ||«      j	                  |j
                  «      z  }|dz  }ŒC Œ[ |S )z=Compute token embeddings enhanced with hash-based embeddings.)éÊš;l   Í21A ißoYl   Ívt l   ß. l   }îg l   �A§u l   í†0 l   ©ÿT l   AK l   ™| r   r#   )Úembed_tokensÚrangeÚlenrN   Útor-   )rO   Úlocal_encoderrP   rQ   rR   rS   ÚprimesÚ
embeddingsÚembedding_idxÚfunc_nbr)   r@   Úhash_idsÚoffset_hash_idss                 r<   Úcompute_hash_embeddingsrb   h   s®   € ò€Fð ×+Ñ+Ð,@ÓA€JØ€MÜÐ=Ó>ò ˆØ�w¤ V£Ñ,Ñ-ˆØ6ò 	ˆJÜ/Ð0DÀjÐRWÐYvÓwˆHà&¨Ð9VÑ)VÑVˆOØÑ4°_ÓE×HÑHÈ×IZÑIZÓ[Ñ[ˆJØ˜QÑ‰Mñ	ðð Ðr>   FÚ	patch_idsÚnum_patchesÚsequence_lengthÚpatches_as_queriesÚcross_attn_kr,   c                 óz  — | j                   \  }}| j                  }|rp||z  }	|}
t        j                  ||¬«      j	                  d«      j	                  d«      j                  |||«      }| j	                  d«      j                  |||«      }no|}	||z  }
| j	                  d«      j                  |||«      }t        j                  ||¬«      j	                  d«      j	                  d«      j                  |||«      }||k(  }|rdnd}|j                  ||¬«      }||	|
f}|j                   |k7  rt        d|j                   › d|› �«      ‚|j	                  d«      }d|j                  |«      z
  }|j                  |j                  t        j                  «      t        j                  |«      j                  «      }|S )	aR  
    Prepare cross-attention mask for patch-based attention, following mllama's robust approach.

    This function creates masks that control which patches can attend to which other patches,
    with support for query/key role swapping and cross-attention multipliers.

    Args:
        patch_ids (torch.Tensor): Tensor of shape [batch_size, seq_len] containing patch ids.
        num_patches (int): Total number of patches.
        sequence_length (int): Length of the sequence.
        patches_as_queries (bool): If True, patches are used as queries, otherwise as keys.
        cross_attn_k (int): Cross-attention multiplier for repeating patches.
        dtype (torch.dtype): Data type for the output mask.

    Returns:
        Tuple[torch.Tensor, torch.Tensor]:
            - cross_attention_mask: 4D tensor [batch_size, 1, q_len, kv_len]
    r/   r   r.   r#   r0   zCross attention mask shape z doesn't match expected g      ð?)r6   r-   r2   r5   Ú	unsqueezeÚexpandÚrepeat_interleaveÚ
ValueErrorrZ   Úmasked_fillÚboolÚfinfoÚmin)rc   rd   re   rf   rg   r,   rG   rH   r-   Úq_lenÚkv_lenÚq_patch_idsÚkv_patch_idsÚcross_attention_maskÚ
repeat_dimÚexpected_shapeÚinverted_cross_attn_masks                    r<   Ú#_prepare_patch_cross_attention_maskry   Ž   sË  € ð4 $Ÿ/™/Ñ€J�Ø×Ñ€Fñ Ø˜lÑ*ˆØ ˆô �L‰L˜¨VÔ4ß‰Y�q‹\ß‰Y�r‹]ß‰V�J ¨WÓ5ð	 	ð !×*Ñ*¨1Ó-×4Ñ4°ZÀÈgÓV‰àˆØ˜|Ñ+ˆà×)Ñ)¨"Ó-×4Ñ4°ZÀÈ+ÓVˆä�L‰L˜¨VÔ4×>Ñ>¸qÓA×KÑKÈAÓN×UÑUÐV`ÐbiÐkvÓwð 	ð '¨,Ñ6Ðñ )‘¨b€JØ/×AÑAÀ,ÐT^ÐAÓ_Ðð ! %¨Ð0€NØ×!Ñ! ^Ò3ÜØ)Ð*>×*DÑ*DÐ)EÐE]Ð^lÐ]mÐnó
ð 	
ð
 0×9Ñ9¸!Ó<Ðð  #Ð%9×%<Ñ%<¸UÓ%CÑCÐØ3×?Ñ?Ø ×#Ñ#¤E§J¡JÓ/´·±¸UÓ1C×1GÑ1GóÐð  Ðr>   Úpatch_lengthsÚmax_patch_lengthc                 ó6  — |€| S | j                  d«      }g }| D ]j  }g }||dkD     D ]J  }|j                  «       }t        ||«      \  }}|j                  |g|z  «       |sŒ:|j	                  |«       ŒL |j	                  |«       Œl t        d„ |D «       «      }	t        j                  ||	f| j                  | j                  ¬«      }
t        |«      D ]D  \  }}|sŒ	t        j                  || j                  | j                  ¬«      |
|dt        |«      …f<   ŒF |
dk7  j                  d¬«      j                  «       |
j                  d   k  rM|
dk7  j                  d¬«      j!                  «       j                  «       j                  «       dz   }|
dd…d|…f   }
|
S )a£  
    Splits patch lengths into smaller segments if they exceed `max_patch_length`.
    Pads the result to uniform length across the batch.

    Args:
        patch_lengths (torch.Tensor): [batch_size, num_patches] tensor of patch lengths.
        max_patch_length (int, optional): Maximum allowed length per patch.

    Returns:
        torch.Tensor: [batch_size, max_len] tensor of split and padded patch lengths.
    Nr   c              3   ó2   K  — | ]  }t        |«      –— Œ y ­w©N)rY   )Ú.0Úsplitss     r<   ú	<genexpr>z(process_patch_lengths.<locals>.<genexpr>ù   s   è ø€ Ò6 &”#�f—+Ñ6ùs   ‚r+   r0   r#   )ÚsizeÚitemÚdivmodÚextendÚappendÚmaxr2   rD   r,   r-   Ú	enumerater3   rY   Úanyr7   r6   Únonzero)rz   r{   rG   Ú	processedÚseqr€   ÚlengthÚfull_chunksÚ	remainderÚmax_lenÚpaddedÚiÚlast_nonzeros                r<   Úprocess_patch_lengthsr”   Ü   s¡  € ð ÐØÐà×#Ñ# AÓ&€JØ€Iàò !ˆØˆØ˜# ™'‘lò 	)ˆFØ—[‘[“]ˆFÜ%+¨FÐ4DÓ%EÑ"ˆK˜Ø�M‰MÐ+Ð,¨{Ñ:Ô;ÚØ—‘˜iÕ(ð	)ð 	×Ñ˜Õ ð!ô Ñ6¨IÔ6Ó6€GÜ�[‰[˜* gÐ.°m×6IÑ6IÐR_×RfÑRfÔg€Fä˜yÓ)ò t‰	ˆˆ6ÚÜ',§|¡|°FÀ-×BUÑBUÐ^k×^rÑ^rÔ'sˆF�1�mœ˜F›�mÐ#Ò$ðtð
 	�!‰×Ñ˜QÐÓ×#Ñ#Ó%¨¯©°Q©Ò7Ø !™×(Ñ(¨QÐ(Ó/×7Ñ7Ó9×=Ñ=Ó?×DÑDÓFÈÑJˆØš˜=˜L˜=Ð(Ñ)ˆà€Mr>   c                   ó   — e Zd Zy)ÚBltMLPN©Ú__name__Ú
__module__Ú__qualname__© r>   r<   r–   r–     ó   „ Ør>   r–   c                   ó   — e Zd Zy)Ú
BltRMSNormNr—   r›   r>   r<   rž   rž     rœ   r>   rž   c                   óD   — e Zd Z ej                  «       ed„ «       «       Zy)ÚBltRotaryEmbeddingc                 ó  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      }|d d …d d d …f   j                  «       }t	        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  |dd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j                  |j                   ¬
«      	j                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   r.   r#   ÚmpsÚcpuF)Údevice_typeÚenabledr   r0   )r,   )Úinv_freqÚfloatrj   r6   Ú
isinstancer-   ÚtypeÚstrr   Ú	transposer2   rk   ÚcosÚattention_scalingÚsinrZ   r,   )
ÚselfÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr¤   ÚfreqsÚembr¬   r®   s
             r<   ÚforwardzBltRotaryEmbedding.forward  s@  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓeÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ×)Ñ)¨%°¸Ô;ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   Â=BFÆF
N)r˜   r™   rš   r2   rC   r   r¶   r›   r>   r<   r    r      s$   „ Ø€U‡]�]ƒ_Øñ<ó ó ñ<r>   r    c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚBltTransformerLayerÚ	layer_idxc                 óþ   •— t         ‰| �  «        t        ||¬«      | _        t	        |«      | _        t        |j                  |j                  ¬«      | _	        t        |j                  |j                  ¬«      | _
        y )N)Úconfigr¹   ©Úeps)ÚsuperÚ__init__ÚBltSelfAttentionÚ	self_attnr–   Úmlprž   Úhidden_sizeÚrms_norm_epsÚinput_layernormÚpost_attention_layernorm©r¯   r»   r¹   Ú	__class__s      €r<   r¿   zBltTransformerLayer.__init__"  s]   ø€ Ü‰ÑÔä)°À9ÔMˆŒÜ˜&“>ˆŒÜ)¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(2°6×3EÑ3EÈ6×K^ÑK^Ô(_ˆÕ%r>   )r˜   r™   rš   Úintr¿   Ú__classcell__©rÈ   s   @r<   r¸   r¸   !  s   ø„ ð`¨#÷ `ñ `r>   r¸   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )rÀ   r»   r¹   c                 ó&   •— t         ‰| �  ||«       y r~   )r¾   r¿   rÇ   s      €r<   r¿   zBltSelfAttention.__init__,  s   ø€ Ü‰Ñ˜ Õ+r>   )r˜   r™   rš   r$   rÉ   r¿   rÊ   rË   s   @r<   rÀ   rÀ   +  s   ø„ ð,˜yð ,°S÷ ,ñ ,r>   rÀ   c            
       ó¤   ‡ — e Zd ZdZddedededz  fˆ fd„Z	 	 ddej                  dej                  dz  d	ej                  dz  d
e	e
   fd„Zˆ xZS )ÚBltCrossAttentionz<Cross-attention module for Blt, following transformers styleNr»   r¹   rÃ   c                 óÈ   •— t         ‰| �  «        d| _        t        | j                  |j
                  ¬«      | _        t        | j                  |j
                  ¬«      | _        y )NFr¼   )r¾   r¿   Ú	is_causalrž   rÃ   rÄ   Úq_normÚk_norm)r¯   r»   r¹   rÃ   rÈ   s       €r<   r¿   zBltCrossAttention.__init__3  sI   ø€ Ü‰ÑÔØˆŒÜ  ×!1Ñ!1°v×7JÑ7JÔKˆŒÜ  ×!1Ñ!1°v×7JÑ7JÔKˆ�r>   Úhidden_statesÚcross_attention_statesÚattention_maskÚkwargsc                 ó`  — |j                  «       \  }}}| j                  |«      }| j                  |«      }|j                  ||| j                  | j
                  «      j                  dd«      }| j                  |«      }| j                  |«      }	| j                  |«      }
|	j                  |d| j                  | j
                  «      j                  dd«      }	|
j                  |d| j                  | j
                  «      j                  dd«      }
t        j                  | j                  j                  t        «      } || ||	|
|f| j                   sdn| j"                  | j$                  dœ|¤Ž\  }}|j'                  ||d«      j)                  «       }| j+                  |«      }||z   }||fS )Nr#   r   r.   ç        )ÚdropoutÚscaling)r‚   rÒ   Úq_projÚviewÚ	num_headsÚhead_dimr«   rÓ   Úk_projÚv_projÚnum_key_value_headsr   Úget_interfacer»   Ú_attn_implementationr"   ÚtrainingrÚ   rÛ   ÚreshapeÚ
contiguousÚo_proj)r¯   rÔ   rÕ   rÖ   r×   Úbszrq   Ú_Úquery_statesÚ
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss                 r<   r¶   zBltCrossAttention.forward9  s•  € ð &×*Ñ*Ó,‰ˆˆU�AØ—{‘{ =Ó1ˆØ—{‘{ <Ó0ˆØ#×(Ñ(¨¨e°T·^±^ÀTÇ]Á]ÓS×]Ñ]Ð^_ÐabÓcˆà!%§¡Ð-CÓ!DÐØ—[‘[Ð!7Ó8ˆ
Ø—{‘{Ð#9Ó:ˆØ—_‘_ S¨"¨d×.FÑ.FÈÏÉÓV×`Ñ`ÐabÐdeÓfˆ
Ø#×(Ñ(¨¨b°$×2JÑ2JÈDÏMÉMÓZ×dÑdÐefÐhiÓjˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$·,±,Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð "×)Ñ)¨#¨u°bÓ9×DÑDÓFˆØ—k‘k +Ó.ˆØ! MÑ1ˆØ˜LÐ(Ð(r>   r~   ©NN)r˜   r™   rš   Ú__doc__r$   rÉ   r¿   r2   ÚTensorr   r   r¶   rÊ   rË   s   @r<   rÏ   rÏ   0  sw   ø„ ÙFñL˜yð L°Sð LÀsÈTÁzõ Lð 7;Ø.2ñ	#)à—|‘|ð#)ð !&§¡¨tÑ 3ð#)ð Ÿ™ tÑ+ð	#)ð
 Ð+Ñ,÷#)r>   rÏ   c                   ó‚   — e Zd ZU eed<   dZdZdZdgZ e	e
d¬«       e	ed¬«      dœZ ej                  «       d„ «       Zy	)
ÚBltPreTrainedModelr»   Fr¸   r   )Úindexr#   )rÔ   Ú
attentionsc           	      ó&
  — t        j                  | |«       |j                  j                  }t	        |t
        j                  «      rÆt        | j                  dd«      }|€7t        | j                  d«      r!t        | j                  j                  dd«      }|€|j                  }|dz  }t        j                  |j                  d|d|z  d|z  ¬«       |j                  �,t        j                   |j                  |j                     «       yt	        |t"        t$        f«      s|d	v �rqt        | j                  dd«      }|€t        |d«      r|j&                  }|€>d
D ]9  }t        ||d«      }|€Œt        |d«      sŒ |j                  j(                  d   } n |€y|dz  }dD ]v  }t        ||d«      }|€Œt        |d«      sŒ t        j                  |j                  d|d|z  d|z  ¬«       t        |dd«      €ŒXt        j                   |j*                  «       Œx t        |dt        |dd«      «      }	|	�bt        |	d«      rVt        j                  |	j                  d|d|z  d|z  ¬«       t        |	dd«      �t        j                   |	j*                  «       yt	        |t,        «      s|dk(  �rât        | j                  dd«      }|€7t        | j                  d«      r!t        | j                  j.                  dd«      }|€7t        | j                  d«      r!t        | j                  j                  dd«      }d}
|�|dz  }
t        |dt        |dd«      «      }t        |dd«      }t        |dt        |dd«      «      }||fD ]‰  }|€Œt        |d«      sŒ|
xs |j                  j(                  d   dz  }t        j                  |j                  d|d|z  d|z  ¬«       t        |dd«      €Œkt        j                   |j*                  «       Œ‹ |�€t        |d«      rt|j                  j(                  d   }|dz  }t        j                  |j                  d|d|z  d|z  ¬«       t        |dd«      �t        j                   |j*                  «       yt	        |t
        j0                  «      rg|j2                  }|dz  }t        j                  |j                  d|d|z  d|z  ¬«       |j*                  �t        j                   |j*                  «       yy)a  
        Initialize BLT weights following the original ByteLatentTransformer:

        - Most weights are drawn from a truncated normal.
        - Scale is ~ 1 / sqrt(model_dim) (or 1 / sqrt(hidden_dim) for FFN outputs).
        - Norm layers are set to weight = 1, bias = 0.
        rÃ   NÚencoder_configg      à¿rÙ   éýÿÿÿr   )ÚmeanÚstdÚaÚb)r!   r   )rÜ   rà   rá   rè   ÚdenseÚweightr.   )rÜ   rà   rá   Úbiasrè   rÿ   r   Údecoder_configÚ	gate_projÚfc1Úup_projÚ	down_projÚfc2r#   )r   Ú_init_weightsrÈ   r˜   r¨   ÚnnÚ	EmbeddingÚgetattrr»   Úhasattrrù   Úembedding_dimÚinitÚtrunc_normal_r   Úpadding_idxÚzeros_rÀ   rÏ   rÃ   r6   r  r–   r  ÚLinearÚin_features)r¯   ÚmoduleÚ
class_namerÃ   rü   r1   ÚnameÚprojÚ	proj_namerè   Úin_stdr  r  r  Ú
hidden_dimÚout_stdÚfan_ins                    r<   r  z BltPreTrainedModel._init_weightss  sÊ  € ô 	×%Ñ% d¨FÔ3à×%Ñ%×.Ñ.ˆ
ô �fœbŸl™lÔ+Ü! $§+¡+¨}¸dÓCˆKØÐ"¤w¨t¯{©{Ð<LÔ'MÜ% d§k¡k×&@Ñ&@À-ÐQUÓV�ØÐ"Ø$×2Ñ2�à˜tÑ#ˆCÜ×ÑØ—‘ØØØ�s‘(Ø�c‘'õð ×!Ñ!Ð-Ü—‘˜FŸM™M¨&×*<Ñ*<Ñ=Ô>Øô �fÔ/Ô1BÐCÔDÈ
ð W
ò I
ô ˜$Ÿ+™+ }°dÓ;ˆCØˆ{œw v¨}Ô=Ø×(Ñ(�Øˆ{ØMò �DÜ" 6¨4°Ó6�DØÑ'¬G°D¸(Õ,CØ"Ÿk™k×/Ñ/°Ñ3˜Ùð	ð
 ˆ{Øà�t‘)ˆCð <ò /�	Ü˜v y°$Ó7�ØÑ#¬°°hÕ(?Ü×&Ñ&ØŸ™Ø ØØ˜s™(Ø˜c™'õô ˜t V¨TÓ2Ñ>ÜŸ™ D§I¡IÕ.ð/ô ˜V X¬w°v¸wÈÓ/MÓNˆFØÐ!¤g¨f°hÔ&?Ü×"Ñ"Ø—M‘MØØØ˜3‘hØ˜#‘gõô ˜6 6¨4Ó0Ð<Ü—K‘K §¡Ô,Øô �fœfÔ%¨°Ó)FÜ! $§+¡+¨}¸dÓCˆKØÐ"¤w¨t¯{©{Ð<LÔ'MÜ% d§k¡k×&@Ñ&@À-ÐQUÓV�ØÐ"¤w¨t¯{©{Ð<LÔ'MÜ% d§k¡k×&@Ñ&@À-ÐQUÓV�ð ˆFØÐ&Ø$ dÑ*�ä ¨´W¸VÀUÈDÓ5QÓRˆIÜ˜f i°Ó6ˆGÜ ¨´W¸VÀUÈDÓ5QÓRˆIð # GÐ,ò /�ØÑ#¬°°hÕ(?Ø ÒB T§[¡[×%6Ñ%6°qÑ%9¸TÑ%A�CÜ×&Ñ&ØŸ™Ø ØØ˜s™(Ø˜c™'õô ˜t V¨TÓ2Ñ>ÜŸ™ D§I¡IÕ.ð/ð Ð$¬°¸HÔ)EØ&×-Ñ-×3Ñ3°AÑ6�
Ø$ dÑ*�Ü×"Ñ"Ø×$Ñ$ØØØ˜7‘lØ˜'‘kõô ˜9 f¨dÓ3Ð?Ü—K‘K 	§¡Ô/Øô �fœbŸi™iÔ(Ø×'Ñ'ˆFØ˜$‘,ˆCÜ×ÑØ—‘ØØØ�s‘(Ø�c‘'õð �{‰{Ð&Ü—‘˜FŸK™KÔ(Øð )r>   N)r˜   r™   rš   r$   Ú__annotations__Ú_supports_attention_backendÚ_supports_flash_attnÚ_supports_flex_attnÚ_no_split_modulesr   r¸   rÀ   Ú_can_record_outputsr2   rC   r  r›   r>   r<   rõ   rõ   _  s_   … àÓØ"'ÐØ ÐØÐØ.Ð/Ðá'Ð(;À1ÔEÙ$Ð%5¸QÔ?ñÐð €U‡]�]ƒ_ñJó ñJr>   rõ   c                   ó\  ‡ — e Zd ZU eed<   d eedd¬«      iZdefˆ fd„Z	 	 	 	 	 	 	 	 	 dde	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  dz  dedz  de	j                  dz  dee   fd„Zd„ Zˆ xZS )ÚBltLocalEncoderr»   Úencoder_attentionsr#   r[   ©rö   Ú
layer_namec           	      óØ  •— t         ‰| �  |«       d| _        || _        t	        j
                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _	        t        |¬«      | _        t	        j                  |j                  |j                  |j                  z  d¬«      | _        t	        j                   |j"                  |j                  «      | _        t	        j
                  «       | _        |j(                  r|j                  nd}t        |«      D ]3  }| j&                  j+                  t-        |||j                  ¬«      «       Œ5 | j/                  «        y c c}w )NF©r»   ©r  Úout_featuresr  r#   ©r»   r¹   rÃ   )r¾   r¿   Úgradient_checkpointingr»   r	  Ú
ModuleListrX   Únum_hidden_layersr¸   Úlayersr    Ú
rotary_embr  rÃ   rg   Úpatch_embedding_projectionr
  Ú
vocab_sizerW   Úcross_attn_layersÚcross_attn_all_layersr†   rÏ   Ú	post_init©r¯   r»   r¹   Úlayers_to_addrÈ   s       €r<   r¿   zBltLocalEncoder.__init__  s!  ø€ Ü‰Ñ˜Ô Ø&+ˆÔ#ØˆŒÜ—m‘mÜEJÈ6×KcÑKcÓEdÖe¸	Ô  ¨Õ3Òeó
ˆŒô -°FÔ;ˆŒÜ*,¯)©)Ø×*Ñ*Ø×+Ñ+¨f×.AÑ.AÑAØô+
ˆÔ'ô
 ŸL™L¨×):Ñ):¸F×<NÑ<NÓOˆÔÜ!#§¡£ˆÔØ4:×4PÒ4P˜×0Ò0ÐVWˆÜ˜}Ó-ò 	ˆIØ×"Ñ"×)Ñ)Ü!¨¸9ÐRX×RdÑRdÔeõð	ð
 	�‰Õùò! fs   ÁE'NÚ	input_idsÚinputs_embedsÚpatch_embedsrÖ   r±   Úpast_key_valuesÚencoder_attention_maskrd   rc   r×   c
                 ó.  — |€| j                  |«      }|j                  d   }t        j                  || j                  j                  | j
                  ¬«      }|€Mt        j                  |j                  d   |j                  ¬«      j                  d«      j                  |d«      }| j                  ||«      }t        j                  || j                  j                  | j
                  ¬«      }t        | j                  «      D �]  \  }} ||f|||dœ|
¤Ž}|t        | j                  «      dz
  k(  s| j                  j                  sŒF| j!                  |||	«      }| j#                  |«      }|j%                  ||j                  d   | j                  j&                  z  | j                  j(                  «      }| j                  j                  r|nd} | j*                  |   d|||dœ|
¤Ž\  }}||j-                  |j                  «      z   }�Œ |}||fS )	Nr   ©Úprå   r#   r/   r.   ©Úposition_embeddingsrÖ   r<  ©rÔ   rÕ   rÖ   r›   )rW   r6   ÚFrÚ   r»   rå   r2   r5   r-   ri   rj   r1  rˆ   r0  rY   r5  Úpatch_reducer2  ræ   rg   rÃ   r4  rZ   )r¯   r9  r:  r;  rÖ   r±   r<  r=  rd   rc   r×   rG   rÔ   rB  ÚidxÚlayerr¹   Úcross_attention_outputrê   Úencoder_cross_statess                       r<   r¶   zBltLocalEncoder.forward  s  € ð Ð Ø ×-Ñ-¨iÓ8ˆMà"×(Ñ(¨Ñ+ˆ
ÜŸ	™	 -°4·;±;×3FÑ3FÐQU×Q^ÑQ^Ô_ˆàÐä—‘˜]×0Ñ0°Ñ3¸M×<PÑ<PÔQ×[Ñ[Ð\]Ó^×eÑeÐfpÐrtÓuð ð #Ÿo™o¨m¸\ÓJÐÜŸ	™	 -°4·;±;×3FÑ3FÐQU×Q^ÑQ^Ô_ˆä# D§K¡KÓ0ó 	]‰JˆC�Ù!Øðà$7Ø-Ø /ñ	ð
 ñˆMð ”c˜$Ÿ+™+Ó&¨Ñ*Ò*¨d¯k©k×.OÓ.OØ#×0Ñ0°ÀÈYÓW�Ø#×>Ñ>¸|ÓL�Ø+×3Ñ3Ø × 2Ñ 2°1Ñ 5¸¿¹×8PÑ8PÑ PÐRV×R]ÑR]×RiÑRió �ð $(§;¡;×#DÒ#D™CÈ!�	Ø,M¨D×,BÑ,BÀ9Ñ,Mð -Ø".Ø+8Ø#9ñ-ð ñ	-Ñ)Ð&¨ð  ,Ð.D×.GÑ.GÈ×H[ÑH[Ó.\Ñ\’ð+	]ð,  ,ÐØÐ2Ð2Ð2r>   c                 óF  — |j                   d   }|j                   d   }|j                  d«      j                  dd|j                   d   «      }t        j                  |||f|j
                  |j                  ¬«      }|j                  |d|dd¬«      }|dd…d|…dd…f   }|S )	a†  
        Reduce variable length patches to single embedding per patch
        Note: this works with variable number of patches for different sequences in the batch
        It handles variable length patches by assuming that patch_lengths will be 0 for any
        extra patches on the *right*. Since there can be a variable number of patches
        this function also return the number of patches for each sequence in the batch.
        Any embeddings on the right that are not allocated to a patch
        (i.e. if the sum(patch_lengths[i]) < seq_len for any i)
        will be sent to a dummy patch, which is trimmed before returning.
        r   r.   r+   r#   ÚamaxF)Úsrcr1   rö   ÚreduceÚinclude_selfN)r6   ri   rj   r2   rD   r,   r-   Úscatter_reduce)r¯   rÔ   Úmax_num_patchesrc   rG   r  Úreduced_embeddingss          r<   rE  zBltLocalEncoder.patch_reduceR  s¿   € ð #×(Ñ(¨Ñ+ˆ
Ø%×+Ñ+¨BÑ/ˆà×'Ñ'¨Ó+×2Ñ2°2°r¸=×;NÑ;NÈrÑ;RÓSˆ	ä"Ÿ[™[Ø˜¨-Ð8À×@SÑ@SÐ\i×\pÑ\pô
Ðð 0×>Ñ>ØØØØØð ?ó 
Ðð 0²Ð3C°OÐ3CÂQÐ0FÑGÐà!Ð!r>   ©	NNNNNNNNN)r˜   r™   rš   r'   r  r   rÀ   r"  r¿   r2   Ú
LongTensorró   r   rÉ   r   r   r¶   rE  rÊ   rË   s   @r<   r$  r$    s	  ø… Ø!Ó!à™nÐ-=ÀQÐSbÔcðÐðÐ4õ ð2 .2Ø-1Ø,0Ø.2Ø04Ø(,Ø6:Ø"&Ø)-ñ23à×#Ñ# dÑ*ð23ð —|‘| dÑ*ð23ð —l‘l TÑ)ð	23ð
 Ÿ™ tÑ+ð23ð ×&Ñ&¨Ñ-ð23ð  ™ð23ð !&§¡¨tÑ 3ð23ð ˜4‘Zð23ð —<‘< $Ñ&ð23ð Ð+Ñ,ó23öh"r>   r$  c                   ó  ‡ — e Zd ZU eed<   defˆ fd„Z	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	dz  d
ej                  dz  de
e   fd„Zˆ xZS )ÚBltLocalDecoderr»   c           	      óÔ  •— t         ‰| �  |«       d| _        || _        d| _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _
        t        |¬«      | _        t        j                  |j                  |j                  |j                   z  d¬«      | _        t%        |j                  |j&                  ¬«      | _        t        j                  «       | _        |j,                  r|j                  nd}t        |«      D ]3  }| j*                  j/                  t1        |||j                  ¬«      «       Œ5 | j3                  «        y c c}w )NFTr)  r*  r¼   r#   r,  )r¾   r¿   r-  r»   Úcross_attn_decoderr	  r.  rX   r/  r¸   r0  r    r1  r  Úhidden_size_globalrÃ   rg   r2  rž   rÄ   Únormr4  r5  r†   rÏ   r6  r7  s       €r<   r¿   zBltLocalDecoder.__init__u  s$  ø€ Ü‰Ñ˜Ô Ø&+ˆÔ#ØˆŒØ"&ˆÔÜ—m‘mÜEJÈ6×KcÑKcÓEdÖe¸	Ô  ¨Õ3Òeó
ˆŒô -°FÔ;ˆŒÜ*,¯)©)Ø×1Ñ1Ø×+Ñ+¨f×.AÑ.AÑAØô+
ˆÔ'ô
 ˜v×1Ñ1°v×7JÑ7JÔKˆŒ	Ü!#§¡£ˆÔØ4:×4PÒ4P˜×0Ò0ÐVWˆÜ˜}Ó-ò 	ˆIØ×"Ñ"×)Ñ)Ü!¨¸9ÐRX×RdÑRdÔeõð	ð
 	�‰Õùò! fs   ÁE%Nr9  r:  r;  rÖ   r±   r<  r=  r×   c                 ó"  — |j                   d   }	|}
| j                  |«      }|j                  |	|j                   d   | j                  j                  z  | j                  j
                  «      }|�| j                  s|
|z   }
|€Mt        j                  |j                   d   |j                  ¬«      j                  d«      j                  |	d«      }| j                  |
|«      }t        j                  |
| j                  j                  | j                  ¬«      }
t!        | j"                  «      D ]N  \  }}|dk(  s| j                  j$                  r! | j&                  |   d|
||dœ|¤Ž\  }}|
|z   }
 ||
f|||dœ|¤Ž}
ŒP | j)                  |
«      }|S )	Nr   r#   r/   r.   r?  rC  rA  r›   )r6   r2  ræ   r»   rg   rÃ   rW  r2   r5   r-   ri   rj   r1  rD  rÚ   rå   rˆ   r0  r5  r4  rY  )r¯   r9  r:  r;  rÖ   r±   r<  r=  r×   rG   rÔ   rB  r’   rG  rH  rê   Úlogitss                    r<   r¶   zBltLocalDecoder.forward�  s¤  € ð #×(Ñ(¨Ñ+ˆ
Ø%ˆØ×6Ñ6°|ÓDˆØ#×+Ñ+Ø˜×*Ñ*¨1Ñ-°·±×0HÑ0HÑHÈ$Ï+É+×JaÑJaó
ˆð Ð#¨D×,CÒ,CØ)¨LÑ8ˆMàÐä—‘˜]×0Ñ0°Ñ3¸M×<PÑ<PÔQ×[Ñ[Ð\]Ó^×eÑeÐfpÐrtÓuð ð #Ÿo™o¨m¸\ÓJÐÜŸ	™	 -°4·;±;×3FÑ3FÐQU×Q^ÑQ^Ô_ˆä! $§+¡+Ó.ò 	‰HˆAˆuØ�AŠv˜Ÿ™×:Ò:Ø,E¨D×,BÑ,BÀ1Ñ,Eð -Ø"/Ø+7Ø#9ñ-ð ñ	-Ñ)Ð&¨ð !.Ð0FÑ F�Ù!Øðà$7Ø-Ø /ñ	ð
 ñ‰Mð	ð  —‘˜=Ó)ˆØˆr>   ©NNNNNNN)r˜   r™   rš   r&   r  r¿   r2   rS  ró   r   r   r   r¶   rÊ   rË   s   @r<   rU  rU  r  sÄ   ø… Ø!Ó!ðÐ4õ ð4 .2Ø-1Ø,0Ø.2Ø04Ø(,Ø6:ñ.à×#Ñ# dÑ*ð.ð —|‘| dÑ*ð.ð —l‘l TÑ)ð	.ð
 Ÿ™ tÑ+ð.ð ×&Ñ&¨Ñ-ð.ð  ™ð.ð !&§¡¨tÑ 3ð.ð Ð+Ñ,÷.r>   rU  c                   óÂ   ‡ — e Zd ZU eed<   d eedd¬«      iZdefˆ fd„Z	 	 	 dde	j                  d	e	j                  dz  d
e	j                  dz  dedz  dee   f
d„Zˆ xZS )ÚBltGlobalTransformerr»   Úglobal_attentionsr#   Úglobal_transformerr&  c                 óÔ  •— t         ‰| �  |«       || _        t        j                  «       | _        t        |j                  «      D ]'  }| j
                  j                  t        ||«      «       Œ) t        |¬«      | _        t        |dd «      �2t        j                  |j                  |j                  d¬«      | _        nt        j"                  «       | _        | j%                  «        y )Nr)  Úencoder_cross_output_sizeF©r  )r¾   r¿   r»   r	  r.  r0  rX   r/  r†   r¸   r    r1  r  r  rb  rÃ   Útoken_embedding_projectionÚIdentityr6  rÇ   s      €r<   r¿   zBltGlobalTransformer.__init__Ä  s¶   ø€ Ü‰Ñ˜Ô ØˆŒÜ—m‘m“oˆŒÜ˜v×7Ñ7Ó8ò 	GˆIØ�K‰K×ÑÔ2°6¸9ÓEÕFð	Gä,°FÔ;ˆŒô �6Ð6¸Ó=ÐIÜ.0¯i©iØ×0Ñ0°&×2DÑ2DÈ5ô/ˆDÕ+ô /1¯k©k«mˆDÔ+à�‰Õr>   Nr:  rÖ   r±   r<  r×   c                 óÊ  — |j                   \  }}}| j                  |«      }	t        j                  |	| j                  j                  | j
                  ¬«      }	|€Mt        j                  |j                   d   |j                  ¬«      j                  d«      j                  |d«      }| j                  |	|«      }
t        | j                  «      D ]  \  }} ||	f|
||dœ|¤Ž}	Œ |	S )Nr?  r#   r/   r   r.   rA  )r6   rd  rD  rÚ   r»   rå   r2   r5   r-   ri   rj   r1  rˆ   r0  )r¯   r:  rÖ   r±   r<  r×   rG   rH   rê   rÔ   rB  r’   rG  s                r<   r¶   zBltGlobalTransformer.forwardÖ  sê   € ð "/×!4Ñ!4Ñˆ
�G˜QØ×7Ñ7¸ÓFˆÜŸ	™	 -°4·;±;×3FÑ3FÐQU×Q^ÑQ^Ô_ˆØÐä—‘˜]×0Ñ0°Ñ3¸M×<PÑ<PÔQ×[Ñ[Ð\]Ó^×eÑeÐfpÐrtÓuð ð #Ÿo™o¨m¸\ÓJÐÜ! $§+¡+Ó.ò 	‰HˆAˆuÙ!Øðà$7Ø-Ø /ñ	ð
 ñ‰Mð	ð Ðr>   )NNN)r˜   r™   rš   r%   r  r   rÀ   r"  r¿   r2   ró   rS  r   r   r   r¶   rÊ   rË   s   @r<   r^  r^  ¾  s�   ø… Ø&Ó&à™^Ð,<ÀAÐRfÔgðÐðÐ9õ ð* /3Ø04Ø(,ñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð Ð+Ñ,÷r>   r^  c                   ó  ‡ — e Zd ZU eed<   defˆ fd„Z	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de	dz  dej                  dz  d	edz  d
edz  dedz  dedz  dee   fd„Ze	 	 dd„«       Zˆ xZS )Ú
BltPatcherr»   c                 óÖ  •— t         ‰| �  |«       t        | j                  ¬«      | _        t        j                  «       | _        t        | j                  j                  «      D ]1  }| j                  j                  t        | j                  |«      «       Œ3 t        j                  | j                  j                  | j                  j                  «      | _        t!        | j                  j                  | j                  j"                  ¬«      | _        t        j&                  | j                  j                  | j                  j                  d¬«      | _        | j+                  «        y )Nr)  r¼   Frc  )r¾   r¿   r    r»   r1  r	  r.  r0  rX   r/  r†   r¸   r
  r3  rÃ   rW   rž   rÄ   rY  r  Úlm_headr6  rÇ   s      €r<   r¿   zBltPatcher.__init__ô  sí   ø€ Ü‰Ñ˜Ô Ü,°D·K±KÔ@ˆŒÜ—m‘m“oˆŒÜ˜tŸ{™{×<Ñ<Ó=ò 	LˆIØ�K‰K×ÑÔ2°4·;±;À	ÓJÕKð	LäŸL™L¨¯©×)?Ñ)?ÀÇÁ×AXÑAXÓYˆÔÜ˜tŸ{™{×6Ñ6¸D¿K¹K×<TÑ<TÔUˆŒ	Ü—y‘yØ�K‰K×#Ñ#Ø�K‰K×"Ñ"Øô
ˆŒð 	�‰Õr>   Nr9  rÖ   r±   r<  r:  Ú	use_cacheÚ
patch_sizeÚ	thresholdr{   r×   c
                 ó6  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        | j                  ||||¬«      }|}| j                  ||«      }| j                  D ]  } ||||¬«      }Œ | j                  | j                  |«      «      }t
        j                  j!                  |¬«      j#                  «       }|j                  d d	 \  }}|�| j%                  ||||¬
«      }n.t        j&                  ||f|j(                  |j                  ¬«      }t+        ||	«      }|||fS )Nú:You must specify exactly one of input_ids or inputs_embedsr)  r   r#   r/   ©r»   r:  rÖ   r<  r±   )rB  rÖ   )r[  r   )Ú	entropiesre   rl  rm  r+   )rl   rW   r   r»   Úget_seq_lengthr2   r5   r6   r-   ri   r
   r1  r0  rj  rY  ÚdistributionsÚCategoricalÚentropyÚpatch_lengths_from_entropiesÚonesr,   r”   )r¯   r9  rÖ   r±   r<  r:  rk  rl  rm  r{   r×   Úpast_seen_tokensÚcausal_maskrÔ   rB  rG  r[  Úprediction_entropiesrG   re   rz   s                        r<   r¶   zBltPatcher.forward  s»  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨m¸\ÓJÐà—[‘[ò 	vˆEÙ! -ÐEXÐitÔu‰Mð	vð —‘˜dŸi™i¨Ó6Ó7ˆÜ$×2Ñ2×>Ñ>ÀfÐ>ÓM×UÑUÓWÐà&3×&9Ñ&9¸"¸1Ð&=Ñ#ˆ
�OØÐ!Ø ×=Ñ=Ø.Ø /Ø%Ø#ð	 >ó ‰Mô "ŸJ™JØ˜_Ð-°]×5HÑ5HÐQ^×QeÑQeôˆMô .¨mÐ=MÓNˆØ# ]°FÐ:Ð:r>   c                 óŠ  — | j                   d   }t        j                  ddgt        j                  | j                  ¬«      j                  d«      j                  |d«      }|j                   d   }| dd…dd…f   } | |kD  }|j                   d   }t        j                  || j                  ¬«      j                  d«      j                  |d«      }	t        j                  |	|«      }
t        j                  |	|
gd¬«      }t        j                  || gd¬«      }||   j                  ||«      }|j                  d¬«      j                  «       }|dd…d|…f   }t        j                  |||z   fd¬«      }t        j                  |dd…dd…f   |dz
  «      }t        j                  |dd…dd…f   dz
  |fd¬«      }||z
  dz   }|S )zÔ
        Computes patch lengths from token entropies.

        Depending on whether a threshold is provided, the function uses either:
        - Thresholding the entropy values (when `threshold` is set).
        r   r#   r+   Nr/   r.   r0   )r6   r2   r3   Úlongr-   ri   Úrepeatr5   rj   Ú	full_likerE   ræ   r7   r‡   )rq  re   rl  rm  rG   Úinit_tokensÚoffsetÚ
patch_maskrH   Útoken_indicesÚsentinelÚpadded_indicesÚpadded_maskÚpatch_startsÚmax_valid_patchesÚpatch_start_idsÚ
last_tokenÚ
patch_endsrz   s                      r<   rv  z'BltPatcher.patch_lengths_from_entropies?  sÆ  € ð —_‘_ QÑ'ˆ
ô �L‰L˜!˜Q˜¤u§z¡z¸)×:JÑ:JÔK×UÑUÐVWÓX×_Ñ_Ð`jÐlmÓnð 	ð ×"Ñ" 1Ñ%ˆð ša ¡˜eÑ$ˆ	ð  Ñ*ˆ
à×"Ñ" 1Ñ%ˆô Ÿ™ W°Y×5EÑ5EÔF×PÑPÐQRÓS×ZÑZÐ[eÐgiÓjˆÜ—?‘? =°'Ó:ˆÜŸ™ M°8Ð#<À!ÔDˆô —i‘i ¨j¨[Ð 9¸qÔAˆð & kÑ2×:Ñ:¸:ÀwÓOˆØ&ŸN™N¨q˜NÓ1×5Ñ5Ó7ÐØ#¢AÐ'9Ð(9Ð'9Ð$9Ñ:ˆô  Ÿ)™) [°,ÀÑ2GÐ$HÈaÔPˆô —_‘_ _²Q¸¸¸°UÑ%;¸_ÈqÑ=PÓQˆ
Ü—Y‘Y ²°1±2°Ñ 6¸Ñ :¸JÐGÈQÔOˆ
à" _Ñ4°qÑ8ˆàÐr>   rR  rñ   )r˜   r™   rš   r(   r  r¿   r2   rS  ró   r   ÚFloatTensorrn   rÉ   r§   r   r   r¶   Ústaticmethodrv  rÊ   rË   s   @r<   rh  rh  ñ  sû   ø… ØÓðÐ/õ ð$ .2Ø.2Ø04Ø(,Ø26Ø!%Ø!%Ø"&Ø'+ñ9;à×#Ñ# dÑ*ð9;ð Ÿ™ tÑ+ð9;ð ×&Ñ&¨Ñ-ð	9;ð
  ™ð9;ð ×(Ñ(¨4Ñ/ð9;ð ˜$‘;ð9;ð ˜$‘Jð9;ð ˜4‘<ð9;ð  ™*ð9;ð Ð+Ñ,ó9;ðv ð Øò	3ó ô3r>   rh  c                   óV  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  de
dz  d	ej                  dz  d
edz  dee   deez  fd„«       «       Zd„ Zd„ Zdej                  dedej                  fd„Zˆ xZS )ÚBltModelr»   c                 ó¬  •— t         ‰| �  |«       d| _        || _        t	        |j
                  «      | _        t        |j                  «      | _	        t        |j                  «      | _        |j                  t        |j                  «      z  }|j                   |z  }t#        j$                  ||j
                  j&                  «      | _        | j                  j*                  r[t-        |j.                  «      | _        | j0                  j3                  «        | j0                  j5                  «       D ]	  }d|_        Œ nd | _        | j9                  «        y )NF)r¾   r¿   r-  r»   r$  rù   r[   r^  Úglobal_configr`  rU  r  Úlocal_decoderrQ   rY   rR   rS   r	  r
  rÃ   rP   Úpatch_in_forwardrh  Úpatcher_configÚpatcherÚevalÚ
parametersÚrequires_gradr6  )r¯   r»   Únum_embeddingsÚtotal_vocab_sizeÚparamrÈ   s        €r<   r¿   zBltModel.__init__w  s
  ø€ Ü‰Ñ˜Ô Ø&+ˆÔ#àˆŒÜ,¨V×-BÑ-BÓCˆÔÜ"6°v×7KÑ7KÓ"LˆÔÜ,¨V×-BÑ-BÓCˆÔØ×DÑDÄsÈ6×KnÑKnÓGoÑoˆØ!×?Ñ?À.ÑPÐÜ*,¯,©,Ð7GÈ×I^ÑI^×IjÑIjÓ*kˆÔ'Ø�;‰;×'Ò'Ü% f×&;Ñ&;Ó<ˆDŒLØ�L‰L×ÑÔØŸ™×0Ñ0Ó2ò ,�Ø&+�Õ#ñ,ð  ˆDŒLØ�‰Õr>   Nr9  rz   rÖ   r±   r<  r:  rk  r×   rT   c                 ó~  — |d u |d uz  rt        d«      ‚|rg|€5t        t        | j                  ¬«      t        | j                  ¬«      «      }n0t	        |t        «      s t        |t        | j                  ¬«      «      }|�|}	|j
                  \  }
}}no|j
                  \  }
}t        || j                  | j                  | j                  j                  | j                  j                  | j                  j                  «      }	|�€| j                  j                  dk(  r�| j                  �ƒ|€t        d«      ‚| j                  || j                  j                  | j                  j                  | j                  j                   | j                  j"                  |j$                  ¬«      \  }}}no|�|j$                  n|j$                  }|�|j&                  n|j&                  }t)        t+        j,                  |
|dz   f||¬«      | j                  j                   «      }| j/                  ||«      }|€V|�|j1                  «       nd}t+        j2                  |	j
                  d   |	j$                  ¬	«      |z   }|j5                  d«      }t7        | j                  |	||�|j8                  nd |¬
«      }t;        ||j
                  d   |d| j                  j<                  |	j&                  ¬«      } | j                  d||	||||j
                  d   ||�|j8                  nd dœ|¤Ž\  }}|j?                  |
|j
                  d   d«      }t+        j2                  d|j
                  d   |j$                  ¬	«      }|j5                  d«      }t7        | j                  |d d d ¬
«      } | j@                  d|||dœ|¤Ž}| j/                  |d d …dd …f   |«      }t;        ||j
                  d   |d| j                  j<                  |	j&                  ¬«      } | jB                  d||||||�|jD                  nd |dœ|¤Ž}tG        ||¬«      S )Nro  r)  ru  z0input_ids is required for entropy-based patching)rl  rm  r{   Úpatching_batch_sizer-   r#   r+   r   r/   rp  T)rc   rd   re   rf   rg   r,   )r9  r:  rÖ   r±   r=  rd   rc   r<  r.   )r:  rÖ   r±   F)r9  r:  r;  rÖ   r±   r<  r=  )Úlast_hidden_stater<  r›   )$rl   r   r   r»   r¨   r6   rb   r[   rP   rQ   rR   rS   Úpatching_moder”  rl  Úpatching_thresholdr{   rœ  r-   r,   r”   r2   rw  Ú_patch_ids_from_lengthsrr  r5   ri   r
   Úself_attention_cachery   rg   rÝ   r`  r‘  Úcross_attention_cacher   )r¯   r9  rz   rÖ   r±   r<  r:  rk  r×   Úencoder_embedsrG   re   rê   r-   r,   rc   rx  ry  Úcross_attn_mask_encÚencoder_hidden_statesrI  Úglobal_position_idsÚglobal_causal_maskÚglobal_hidden_statesÚdecoder_patch_idsÚcross_attn_mask_decÚoutputs                              r<   r¶   zBltModel.forward‹  sZ  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZáØÐ&Ü"5Ü ¨¯©Ô4´lÈ$Ï+É+Ô6Vó#‘ô   Ô1DÔEô #6°oÄ|Ð[_×[fÑ[fÔGgÓ"h�ð Ð$Ø*ˆNØ-:×-@Ñ-@Ñ*ˆJ˜©à*3¯/©/Ñ'ˆJ˜Ü4ØØ×"Ñ"Ø×/Ñ/Ø—‘×@Ñ@Ø—‘×8Ñ8Ø—‘×9Ñ9óˆNð Ñ Ø�{‰{×(Ñ(¨IÒ5¸$¿,¹,Ð:RØÐ$Ü$Ð%WÓXÐXØ&*§l¡lØØ#Ÿ{™{×5Ñ5Ø"Ÿk™k×<Ñ<Ø%)§[¡[×%AÑ%AØ(,¯©×(GÑ(GØ$×+Ñ+ð '3ó 'Ñ#��=¡!ð .7Ð-B˜×)Ò)È×H\ÑH\�Ø+4Ð+@˜	ŸšÀm×FYÑFY�Ü 5Ü—J‘J 
¨O¸aÑ,?Ð@ÈÐV\Ô]Ø—K‘K×0Ñ0ó!�ð ×0Ñ0°ÀÓPˆ	àÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(<Ñ(<¸QÑ(?È×H]ÑH]Ô^ÐaqÑqˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø(Ø)ØDSÐD_˜O×@Ò@ÐeiØ%ô
ˆô BØØ%×+Ñ+¨AÑ.Ø+Ø#ØŸ™×1Ñ1Ø ×&Ñ&ô
Ðð 7I°d×6HÑ6Hð 
7
ØØ(Ø&Ø%Ø#6Ø%×+Ñ+¨AÑ.ØØDSÐD_˜O×@Ò@Ðeiñ
7
ð ñ
7
Ñ3ÐÐ3ð  4×8Ñ8¸À]×EXÑEXÐYZÑE[Ð]_Ó`ÐÜ#Ÿl™l¨1Ð.B×.HÑ.HÈÑ.KÐTh×ToÑToÔpÐØ1×;Ñ;¸AÓ>ÐÜ/Ø—;‘;Ø.ØØ Øô
Ðð  7˜t×6Ñ6ð  
Ø.Ø-Ø,ñ 
ð ñ	 
Ðð !×8Ñ8¸ÂqÈ!É"ÀuÑ9MÈÓ_ÐÜAØ'Ø%×+Ñ+¨AÑ.Ø+Ø$ØŸ™×1Ñ1Ø ×&Ñ&ô
Ðð $�×#Ñ#ð 	
ØØ/Ø-Ø&Ø%ØETÐE`˜O×AÒAÐfjØ#6ñ	
ð ñ	
ˆô 'Ø$Ø+ô
ð 	
r>   c                 ó.   — | j                   j                  S r~   ©r[   rW   )r¯   s    r<   Úget_input_embeddingszBltModel.get_input_embeddings  s   € Ø×!Ñ!×.Ñ.Ð.r>   c                 ó&   — || j                   _        y r~   r­  )r¯   Úvalues     r<   Úset_input_embeddingszBltModel.set_input_embeddings  s   € Ø*/ˆ×ÑÕ'r>   rH   c                 ó¦  — |j                   d   }t        j                  t        j                  |d|j                  |j
                  ¬«      |j                  d¬«      d d …d d…f   gd¬«      }t        j                  ||j
                  ¬«      }|j                  d«      |j                  d«      j                  d«      k  j                  d¬«      dz
  S )Nr   r#   r+   r.   r0   r/   )
r6   r2   rE   rD   r,   r-   Úcumsumr5   ri   r7   )r¯   rz   rH   rG   r†  Útoken_positionss         r<   r   z BltModel._patch_ids_from_lengths  sÀ   € Ø"×(Ñ(¨Ñ+ˆ
Ü—y‘yä—‘˜J¨°×1DÑ1DÈ]×MaÑMaÔbØ×$Ñ$¨Ð$Ó,ªQ°°°¨VÑ4ðð ô
ˆô  Ÿ,™, w°}×7KÑ7KÔLˆØ×&Ñ& qÓ)¨_×-FÑ-FÀqÓ-I×-SÑ-SÐTVÓ-WÑW×\Ñ\ÐacÐ\ÓdÐghÑhÐhr>   r\  )r˜   r™   rš   r$   r¿   r   r   r2   rS  ró   r   r‹  rn   r   r   Útupler   r¶   r®  r±  rÉ   r   rÊ   rË   s   @r<   rŽ  rŽ  v  s   ø„ ð˜yõ ð(  Øð .2Ø-1Ø.2Ø04Ø(,Ø26Ø!%ñC
à×#Ñ# dÑ*ðC
ð —|‘| dÑ*ðC
ð Ÿ™ tÑ+ð	C
ð
 ×&Ñ&¨Ñ-ðC
ð  ™ðC
ð ×(Ñ(¨4Ñ/ðC
ð ˜$‘;ðC
ð Ð+Ñ,ðC
ð 
Ð(Ñ	(òC
ó ó  ðC
òJ/ò0ð
i°U·\±\ð 
iÈCð 
iÐTY×T`ÑT`÷ 
ir>   rŽ  zB
    The Blt Text Model with a language modeling head on top.
    )Úcustom_introc                   óÈ  ‡ — e Zd ZU eed<   dZdZddiZdefˆ fd„Ze	e
	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  deej                  ej                  f   dz  dedz  dej                   dz  dej                  dz  dedz  deej                  z  dee   deez  fd„«       «       Zˆ xZS )ÚBltForCausalLMr»   FÚmodelz'model.local_encoder.embed_tokens.weightzlm_head.weightc                 ó&  •— t         ‰| �  |«       |j                  «       | _        |j                  | _        t        |«      | _        t        j                  |j                  j                  |j                  d¬«      | _        | j                  «        y )NFrc  )r¾   r¿   Úget_text_configÚtext_configr3  rŽ  r¹  r	  r  r  rÃ   rj  r6  )r¯   r»   rÈ   s     €r<   r¿   zBltForCausalLM.__init__0  sk   ø€ Ü‰Ñ˜Ô Ø!×1Ñ1Ó3ˆÔØ ×+Ñ+ˆŒÜ˜fÓ%ˆŒ
Ü—y‘y ×!6Ñ!6×!BÑ!BÀF×DUÑDUÐ\aÔbˆŒà�‰Õr>   Nr9  rÖ   r±   rÕ   ru   Úfull_text_row_masked_out_maskr<  r:  Úlabelsrk  Úlogits_to_keepr×   rT   c                 ó‚  —  | j                   d||||||||
dœ|¤Ž}|j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      j                  «       }d}|	� | j                  ||	| j                  fi |¤Ž}t        |||j                  |j                  |j                  ¬«      S )aý
  
        cross_attention_states (`torch.FloatTensor`, *optional*):
            Output of the vision model, used for cross-attention. This tensor contains the processed image features that
            the language model will attend to.
        cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*):
            Cross-attention mask to control the interaction between text tokens and image tiles.
            This 4D tensor defines which image tiles each text token should attend to.

            For each text token (in seq_length):
            - 1 indicates the token **should attend** to the corresponding image tile
            - 0 indicates the token **should not attend** to the corresponding image tile
        full_text_row_masked_out_mask (`tuple[torch.Tensor, torch.Tensor]`, *optional*):
            A tuple containing two tensors that mask out rows in the cross-attention mechanism:
            - The first tensor has shape `(batch_size, 1, seq_length, 1)` and contains values of 0 or 1.
              A value of 0 indicates that the corresponding text token's entire row in the cross-attention
              matrix should be masked out (all image tokens ignored).
            - The second tensor has the same shape and is used internally to apply the masking during
              the forward pass of cross-attention layers.
            This mask is derived from the cross_attention_mask and is used to handle cases where a text token
            should not attend to any image token.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, BltForCausalLM

        >>> model = BltForCausalLM.from_pretrained("itazap/blt-1b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("itazap/blt-1b-hf")

        >>> prompt = "If I had to write a haiku, it would be:"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=40, do_sample=True, temperature=0.6)
        >>> result = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        >>> print(result)
        If I had to write a haiku, it would be: "Snowflakes gently fall" - simple, yet peaceful.
        I love the idea of snowflakes gently falling, each one
        ```
        )r9  rÖ   r±   ru   r½  r<  r:  rk  N)Úlossr[  r<  rÔ   r÷   r›   )r¹  r�  r¨   rÉ   Úslicerj  r§   Úloss_functionr3  r   r<  rÔ   r÷   )r¯   r9  rÖ   r±   rÕ   ru   r½  r<  r:  r¾  rk  r¿  r×   ÚoutputsrÔ   Úslice_indicesr[  rÁ  s                     r<   r¶   zBltForCausalLM.forward9  sß   € ð| �$—*‘*ð 

ØØ)Ø%Ø!5Ø*GØ+Ø'Øñ

ð ñ

ˆð  ×1Ñ1ˆÜ8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓA×GÑGÓIˆàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r>   )NNNNNNNNNNr   )r˜   r™   rš   r$   r  Ú_can_compile_fullgraphÚbase_model_prefixÚ_tied_weights_keysr¿   r   r   r2   rS  ró   rµ  r   r‹  rn   rÉ   r   r   r   r¶   rÊ   rË   s   @r<   r¸  r¸  %  s|  ø… ð ÓØ"ÐØÐØCÐEUÐVÐð˜yõ ð Øð .2Ø.2Ø04Ø:>Ø8<ØRVØ(,Ø26Ø*.Ø!%Ø-.ñV
à×#Ñ# dÑ*ðV
ð Ÿ™ tÑ+ðV
ð ×&Ñ&¨Ñ-ð	V
ð
 !&× 0Ñ 0°4Ñ 7ðV
ð $×.Ñ.°Ñ5ðV
ð (-¨U¯\©\¸5¿<¹<Ð-GÑ'HÈ4Ñ'OðV
ð  ™ðV
ð ×(Ñ(¨4Ñ/ðV
ð × Ñ  4Ñ'ðV
ð ˜$‘;ðV
ð ˜eŸl™lÑ*ðV
ð Ð+Ñ,ðV
ð 
Ð'Ñ	'òV
ó ó ôV
r>   r¸  )rõ   rŽ  rh  r¸  )rV   )r   rV   i0u  )Zrò   Úcollections.abcr   r2   Útorch.distributionsÚtorch.nnr	  Útorch.nn.functionalÚ
functionalrD  Ú r   r  Úcache_utilsr   r   r   Ú
generationr	   Úmasking_utilsr
   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úutils.output_capturingr   r   Úcohere2.modeling_cohere2r   Úllama.modeling_llamar   Úmllama.modeling_mllamar   r   r   r   r    r!   r"   Úconfiguration_bltr$   r%   r&   r'   r(   Ú
get_loggerr˜   ÚloggerrÉ   r=   ró   rN   r
  Úlistrb   Úfloat32rn   r,   rµ  ry   r”   r–   rž   r    r¸   rÀ   rÏ   rõ   r$  rU  r^  rh  rŽ  r¸  Ú__all__r›   r>   r<   ú<module>râ     s•  ðñ Cå $ã Û Ý ß Ð å &ß CÑ CÝ )Ý /ß OÝ 6ß FÝ &ß RÓ Rß Gß EÝ 2Ý 7÷÷ ñ ÷õ ð 
ˆ×	Ñ	˜HÓ	%€ñ:°ó :ð< \añØ�|‰|ðØ),ðØ9<ðØUXóð$#ØŸ,™,ð#ð !#§¡ð#ð +.ð	#ð
 #'ð#ð $'ð#ð ‡\�\ó#ðT  %ØØŸ™ñK Ø�|‰|ðK àðK ð ðK ð ð	K ð
 ðK ð �;‰;ðK ð ˆ5�<‰<˜Ÿ™Ð%Ñ&óK ð\)¨¯©ð )ÈÈtÉð )ÐX]×XdÑXdó )ôX	ˆ]ô 	ô	Ð"ô 	ô<Ð-ô <ô"`Ð9ô `ô,Ð.ô ,ô
,)Ð0ô ,)ð^ ô^Ð.ó ^ó ð^ôBn"Ð(ô n"ôbIÐ(ô IôX0Ð-ô 0ôfBÐ#ô BôJliÐ!ô liñ^ ðôô
g
Ð'¨ó g
óð
g
òT�r>   