Ë
    ÜÓ^je  ã                   óv   — d dl mZ ddlmZmZ ddlmZ ddlmZ  ed¬«      e G d„ d	e«      «       «       Z	d	gZ
y
)é    )Ústricté   )ÚPreTrainedConfigÚremap_legacy_layer_types)ÚRopeParameters)Úauto_docstringzZyphra/Zamba2-2.7B)Ú
checkpointc                   ó°  ‡ — e Zd ZU dZdZdddœZdgZdZee	d<   d	Z
ee	d
<   dZee	d<   dZee	d<   dZee   dz  e	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee   eedf   z  dz  e	d<   d Zee	d!<   d"Zee	d#<   d"Zee	d$<   d%Zee	d&<   d'Zee	d(<   d'Z ee	d)<   dZ!edz  e	d*<   d+Z"ee	d,<   d-Z#ee	d.<   dZ$edz  e	d/<   d0Z%eez  e	d1<   dZ&ee	d2<   d'Z'ee	d3<   d4Z(ee	d5<   d'Z)ee	d6<   dZ*e+e,z  dz  e	d7<   d8Z-ee	d9<   d:Z.ee	d;<   d"Z/ee	d<<   dZ0ee	d=<   d>Z1edz  e	d?<   dZ2edz  e	d@<   dZ3eee   z  dz  e	dA<   d'Z4ee	dB<   d"Z5ee	dC<   ˆ fdD„Z6ˆ xZ7S )EÚZamba2Configað	  
    mamba_ngroups (`int`, *optional*, defaults to 1):
        Number of groups for the evolution matrices of mamba 2.
    n_mamba_heads (`int`, *optional*, defaults to 8):
        Number of heads for the evolution matrices of mamba 2.
    use_mamba_kernels (`bool`, *optional*, defaults to `True`):
        Flag indicating whether or not to use the fast mamba kernels.
    use_conv_bias (`bool`, *optional*, defaults to `True`):
        Whether or not to use bias in the convolution layer of the mixer block.
    chunk_size (`int`, *optional*, defaults to 256):
        Size of the chunks that will comprise the sequence.
    use_mem_eff_path (`bool`, *optional*, defaults to `False`):
        Whether or not to use the fused conv1d and scan in mamba2 layers.
    add_bias_linear (`bool`, *optional*, defaults to `False`):
        Flag indicating whether or not to use bias in various layers
    num_mem_blocks (`int`, *optional*, defaults to 1):
        Number of unshared transformer blocks.
    use_shared_attention_adapter (`bool`, *optional*, defaults to `False`):
        If True, unshared adapters (formally the same as LoRA but used in the base model) will be added to the q, k, v projectors in the shared attention layers.
    adapter_rank (`int`, *optional*, defaults to 128):
        Rank of the adapter in the shared MLP and shared attention layers.
    use_mem_rope (`bool`, *optional*, defaults to `False`):
        If True, includes RoPE in the shared attention layers.
    num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
        Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
        integer value, only last `num_logits_to_keep` logits will be calculated. Default is 1 because only the
        logits of the last prompt token are needed for generation. For long sequences, the logits for the entire
        sequence may use a lot of memory so, setting `num_logits_to_keep=1` will reduce memory footprint
        significantly.
    use_long_context (`bool`, *optional*, defaults to `False`):
        Activates the context-extended version of Zamba by modifying RoPE.

    Example:
    ```python
    >>> from transformers import Zamba2Model, Zamba2Config
    >>> # Initializing a Zamba2-2.7B style configuration
    >>> configuration = Zamba2Config()
    >>> # Initializing a model from the Zamba2-2.7B style configuration
    >>> model = Zamba2Model(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úzamba2Úlayers_block_typeÚattention_head_dim)Úlayer_typesÚhead_dimÚpast_key_valuesi }  Ú
vocab_sizei   Úmax_position_embeddingsi 
  Úhidden_sizeé6   Únum_hidden_layersNé@   Úmamba_d_stateé   Úmamba_d_convé   Úmamba_expandé   Úmamba_ngroupsgü©ñÒMbP?Útime_step_mingš™™™™™¹?Útime_step_maxg-Cëâ6?Útime_step_floor.Útime_step_limité   Ún_mamba_headsTÚuse_mamba_kernelsÚuse_conv_biasé   Ú
chunk_sizeFÚuse_mem_eff_pathÚadd_bias_linearÚintermediate_sizeÚgeluÚ
hidden_acté    Únum_attention_headsÚnum_key_value_headsg        Úattention_dropoutÚnum_mem_blocksÚuse_shared_attention_adapteré€   Úadapter_rankÚuse_mem_ropeÚrope_parametersg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsÚ	use_cacheÚnum_logits_to_keepr   Úpad_token_idÚbos_token_idÚeos_token_idÚuse_long_contextÚtie_word_embeddingsc                 ó  •— | j                   xs d| j                  z  | _         d| j                  z  | _        d| j                  z  | j                  z  | _        t        | j                  | j                  z  «      | j                  z  | _        | j                  rd| _
        | j                  €| j                  | _        | j                  | j                  z  | _        | j                  | _        | j                  €4dgdgdz  dgz   dz  z   dgdz  z   dgz   dgdz  z   dgz   dgdz  z   | _        nt        | j                  «      | _        t!        | j                  «      D ��cg c]  \  }}|dk(  sŒ|‘Œ c}}| _        t%        ‰| �L  d	i |¤Ž y c c}}w )
Nr   r   i @  Úlinear_attentioné   Úhybridé   r   © )r+   r   Úattention_hidden_sizer/   r   Úintr   r$   Úmamba_headdimr?   r   r0   Úkv_channelsÚnum_query_groupsr   r   Ú	enumerateÚhybrid_layer_idsÚsuperÚ__post_init__)ÚselfÚkwargsÚindexÚtypeÚ	__class__s       €úz/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/zamba2/configuration_zamba2.pyrO   zZamba2Config.__post_init__q   s§  ø€ Ø!%×!7Ñ!7Ò!O¸1¸t×?OÑ?OÑ;OˆÔØ%&¨×)9Ñ)9Ñ%9ˆÔ"Ø"# d×&6Ñ&6Ñ"6¸$×:RÑ:RÑ"RˆÔÜ  ×!2Ñ!2°T×5EÑ5EÑ!EÓFÈ$×J\ÑJ\Ñ\ˆÔØ× Ò Ø+0ˆDÔ(à×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×+Ñ+¨t×/GÑ/GÑGˆÔØ $× 8Ñ 8ˆÔð ×!Ñ!Ð)à#Ð$Ø&Ð'¨!Ñ+¨x¨jÑ8¸AÑ=ñ>à%Ð&¨Ñ*ñ+ð �*ñð &Ð&¨Ñ*ñ	+ð
 �*ñð &Ð&¨Ñ*ñ+ð Õ"ô &>¸d×>TÑ>TÓ%UˆDÔ"Ü:CÀD×DZÑDZÓ:[× p©;¨5°$Ð_cÐgoÓ_o¢Ó pˆÔÜ‰ÑÑ' Ó'ùó !qs   ÅF	Å,F	)8Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚattribute_mapÚkeys_to_ignore_at_inferencer   rH   Ú__annotations__r   r   r   r   ÚlistÚstrr   r   r   r   r   Úfloatr    r!   r"   Útupler$   r%   Úboolr&   r(   r)   r*   r+   r-   r/   r0   r1   r2   r3   r5   r6   r7   r   Údictr8   r9   r:   r;   r<   r=   r>   r?   r@   rO   Ú__classcell__)rT   s   @rU   r   r      s  ø… ñ)ðV €JØ$7ÐEYÑZ€MØ#4Ð"5Ðà€J�ÓØ#'Ð˜SÓ'Ø€K�ÓØÐ�sÓØ*.Ð�t˜C‘y 4Ñ'Ó.Ø€M�3ÓØ€L�#ÓØ€L�#ÓØ€M�3ÓØ €M�5Ó Ø€M�5ÓØ!€O�UÓ!Ø>B€O�T˜%‘[ 5¨°¨Ñ#4Ñ4°tÑ;ÓBØ€M�3ÓØ"Ð�tÓ"Ø€M�4ÓØ€J�ÓØ"Ð�dÓ"Ø!€O�TÓ!Ø$(Ð�s˜T‘zÓ(Ø€J�ÓØ!Ð˜Ó!Ø&*Ð˜˜t™Ó*Ø%(Ð�u˜s‘{Ó(Ø€N�CÓØ).Ð  $Ó.Ø€L�#ÓØ€L�$ÓØ48€O�^ dÑ*¨TÑ1Ó8Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØÐ˜ÓØ €L�#˜‘*Ó Ø €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø"Ð�dÓ"Ø $Ð˜Ó$÷(ð (ó    r   N)Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   r   Úmodeling_rope_utilsr   Úutilsr   r   Ú__all__rF   re   rU   ú<module>rk      sK   ðõ" /ç MÝ 1Ý #ñ Ð/Ô0Øôt(Ð#ó t(ó ó 1ðt(ðn Ð
�re   