Ë
    ÛÓ^j—  ã                   ór   — d Z ddlZddlmZ ddlmZ ddlmZ  ed¬«      e G d	„ d
e«      «       «       Zd
gZ	y)zJamba model configurationé    N)Ústricté   )ÚPreTrainedConfig)Úauto_docstringzai21labs/Jamba-v0.1)Ú
checkpointc                   óH  ‡ — e Zd ZU dZdZdgZddiZdZee	d<   dZ
ee	d	<   d
Zee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d <   d!Zedz  e	d"<   d#Zeee   z  dz  e	d$<   d%Zee	d&<   d'Zeez  e	d(<   d#Zee	d)<   d*Z ee	d<   d#Z!ee	d+<   d!Z"ee	d,<   dZ#ee	d-<   d.Z$ee	d/<   dZ%ee	d0<   d*Z&ee	d1<   d.Z'ee	d2<   d#Z(ee	d3<   d4Z)eez  e	d5<   dZ*ee	d6<   dZ+ee	d7<   ˆ fd8„Z,e-d9„ «       Z.e-d:„ «       Z/e-d;„ «       Z0d<„ Z1ˆ xZ2S )=ÚJambaConfiga   
    expert_layer_period (`int`, *optional*, defaults to 2):
        Once in this many layers, we will have an expert layer
    expert_layer_offset (`int`, *optional*, defaults to 1):
        The first layer index that contains an expert mlp layer
    attn_layer_period (`int`, *optional*, defaults to 8):
        Once in this many layers, we will have a vanilla attention layer
    attn_layer_offset (`int`, *optional*, defaults to 4):
        The first layer index that contains a vanilla attention mlp layer
    use_mamba_kernels (`bool`, *optional*, defaults to `True`):
        Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
        `causal-conv1d` are installed, and the mamba modules are running on a CUDA device. Raises ValueError if
        `True` and kernels are not available
    mamba_dt_rank (`Union[int,str]`, *optional*, defaults to `"auto"`):
        Rank of the mamba discretization projection matrix. `"auto"` means that it will default to `math.ceil(self.hidden_size / 16)`
    ÚjambaÚpast_key_valuesÚnum_local_expertsÚnum_expertsi   Ú
vocab_sizeFÚtie_word_embeddingsi   Úhidden_sizei 8  Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   Únum_key_value_headsÚsiluÚ
hidden_actg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefr   NÚpad_token_idé   Úbos_token_idé   Úeos_token_idi   Úmax_position_embeddingsg        Úattention_dropoutÚnum_experts_per_toké   Úexpert_layer_periodÚexpert_layer_offsetÚattn_layer_periodé   Úattn_layer_offsetÚuse_mamba_kernelsÚmamba_d_stateÚmamba_d_convÚmamba_expandÚautoÚmamba_dt_rankÚmamba_conv_biasÚmamba_proj_biasc                 óâ   •— | j                   €| j                  | _         | j                  dk(  r"t        j                  | j
                  dz  «      n| j                  | _        t        ‰| �  di |¤Ž y )Nr0   r&   © )r   r   r1   ÚmathÚceilr   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úx/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/jamba/configuration_jamba.pyr9   zJambaConfig.__post_init__R   s`   ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$àAE×ASÑASÐW]ÒA]œTŸY™Y t×'7Ñ'7¸"Ñ'<Ô=Ðcg×cuÑcuˆÔÜ‰ÑÑ' Ó'ó    c                 óŽ   — t        | j                  «      D �cg c]"  }|| j                  z  | j                  k(  rdnd‘Œ$ c}S c c}w )NÚ	attentionÚmamba)Úranger   r)   r+   ©r:   Úis     r=   Úlayers_block_typezJambaConfig.layers_block_typeY   sM   € ô ˜4×1Ñ1Ó2ö
àð ˜t×5Ñ5Ñ5¸×9OÑ9OÒO‰KÐU\Ñ\ò
ð 	
ùò 
s   ˜'Ac                 óN   — | j                   D �cg c]  }|dk(  rdnd‘Œ c}S c c}w )NrA   Úlinear_attentionÚfull_attention)rE   )r:   Úxs     r=   Úlayer_typeszJambaConfig.layer_types`   s-   € ð SW×RhÑRhÖiÈQ a¨7¢lÑ"Ð8HÑHÒiÐiùÒis   �"c                 ó¢   — t        | j                  «      D �cg c],  }|| j                  z  | j                  k(  r| j                  nd‘Œ. c}S c c}w )Nr   )rB   r   r'   r(   r   rC   s     r=   Úlayers_num_expertszJambaConfig.layers_num_expertse   sS   € ô ˜4×1Ñ1Ó2ö
àð !" D×$<Ñ$<Ñ <À×@XÑ@XÒ XˆD×ÒÐ^_Ñ_ò
ð 	
ùò 
s   ˜1Ac                 ó   — | j                   | j                  k\  r&t        d| j                   › d| j                  › d�«      ‚| j                  | j                  k\  r&t        d| j                  › d| j                  › d�«      ‚y)zOPart of `@strict`-powered validation. Validates the architecture of the config.zattention layer offset (z/) must be smaller than attention layer period (ú)zexpert layer offset (z,) must be smaller than expert layer period (N)r+   r)   Ú
ValueErrorr(   r'   )r:   s    r=   Úvalidate_architecturez!JambaConfig.validate_architecturel   s¸   € à×!Ñ! T×%;Ñ%;Ò;ÜØ*¨4×+AÑ+AÐ*BÐBqÐrv÷  sIñ  sIð  rJð  JKð  Lóð ð ×#Ñ# t×'?Ñ'?Ò?ÜØ'¨×(@Ñ(@Ð'AÐAmÐnr÷  oGñ  oGð  nHð  HIð  Jóð ð @r>   )3Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚattribute_mapr   ÚintÚ__annotations__r   Úboolr   r   r   r   r   r   Ústrr   Úfloatr   r   r   r   r   r    r"   Úlistr#   r$   r%   r   r'   r(   r)   r+   r,   r-   r.   r/   r1   r2   r3   r9   ÚpropertyrE   rJ   rL   rP   Ú__classcell__)r<   s   @r=   r	   r	      sÑ  ø… ñð" €JØ#4Ð"5Ðà˜]ð€Mð €J�ÓØ %Ð˜Ó%Ø€K�ÓØ"Ð�sÓ"ØÐ�sÓØ!Ð˜Ó!Ø Ð˜Ó Ø€J�ÓØ#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ!&Ð˜$Ó&Ø"'Ð˜%Ó'Ø €L�#˜‘*Ó Ø €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø#)Ð˜SÓ)Ø%(Ð�u˜s‘{Ó(Ø Ð˜Ó Ø€K�ÓØ Ð˜Ó Ø Ð˜Ó ØÐ�sÓØÐ�sÓØ"Ð�tÓ"Ø€M�3ÓØ€L�#ÓØ€L�#ÓØ%€M�3˜‘9Ó%Ø €O�TÓ Ø!€O�TÓ!ô(ð ñ
ó ð
ð ñjó ðjð ñ
ó ð
ö
r>   r	   )
rT   r6   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úutilsr   r	   Ú__all__r5   r>   r=   ú<module>rd      sM   ðñ  ã å .å 3Ý #ñ Ð0Ô1Øô\Ð"ó \ó ó 2ð\ð~ ˆ/�r>   