Ë
    ÛÓ^jå  ã                   óx   — d Z ddlmZ ddlmZ ddlmZ ddlmZ e edd¬	«       G d
„ de«      «       «       Z	dgZ
y)zBamba model configurationé    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringz´
    The BambaModel is a hybrid [mamba2](https://github.com/state-spaces/mamba) architecture with SwiGLU.
    The checkpoints are  jointly trained by IBM, Princeton, and UIUC.
    zibm-fms/Bamba-9.8b-2.2T-hf)Úcustom_introÚ
checkpointc                   óð  ‡ — e Zd ZU dZdZddiZdgZdZee	d<   dZ
ee	d	<   d
Zee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d <   d!Zeee   z  dz  e	d"<   d#Zee	d$<   d%Zeez  dz  e	d&<   dZee   dz  e	d'<   d(Zedz  e	d)<   d*Z eez  dz  e	d+<   dZ!edz  e	d,<   d-Z"edz  e	d.<   d/Z#edz  e	d0<   d!Z$edz  e	d1<   d-Z%edz  e	d2<   dZ&edz  e	d3<   dZ'edz  e	d4<   d5Z(edz  e	d6<   d7Z)edz  e	d8<   d% ed9«      fZ*ee   e+eef   z  dz  e	d:<   d%Z,edz  e	d;<   dZ-e.e/z  dz  e	d<<   dZ0ee	d=<   dZ1ee	d><   ˆ fd?„Z2e3d@„ «       Z4dA„ Z5ˆ xZ6S )BÚBambaConfigaN  
    num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
        Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
        integer value, only last `num_logits_to_keep` logits will be calculated. Default is 1 because only the
        logits of the last prompt token are needed for generation. For long sequences, the logits for the entire
        sequence may use a lot of memory so, setting `num_logits_to_keep=1` will reduce memory footprint
        significantly.
    attn_layer_indices (`list`, *optional*):
        Specifies the layer indices that will have full attention. Must contain values at most num_hidden_layers.
    z_loss_coefficient (`float`, *optional*, defaults to 0.0):
        Coefficient for auxiliary z-loss used to control logit growth during training
    ÚbambaÚlayer_typesÚlayers_block_typeÚpast_key_valuesi ô Ú
vocab_sizeFÚtie_word_embeddingsi   Úhidden_sizei 8  Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   NÚnum_key_value_headsÚsiluÚ
hidden_actg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacheé   Únum_logits_to_keepr   Úpad_token_idÚbos_token_idé   Úeos_token_idi   Úmax_position_embeddingsg        Úattention_dropoutÚattn_layer_indicesé€   Úmamba_n_headsÚautoÚmamba_d_headÚmamba_n_groupsé   Úmamba_d_stateé   Úmamba_d_convÚmamba_expandÚmamba_chunk_sizeÚmamba_conv_biasÚmamba_proj_biasgü©ñÒMbP?Útime_step_mingš™™™™™¹?Útime_step_maxÚinfÚtime_step_limitÚz_loss_coefficientÚrope_parametersÚattention_biasÚmlp_biasc                 ó,  •— | j                   €| j                  | _         | j                  dk(  r+| j                  | j                  z  | j
                  z  | _        | j                  �t        | j                  «      nd | _        d|d<   t        ‰| �$  di |¤Ž y )Nr)   g      à?Úpartial_rotary_factor© )
r   r   r*   r0   r   r(   r7   ÚtupleÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úx/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/bamba/configuration_bamba.pyrA   zBambaConfig.__post_init__T   s�   ø€ à×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$ð ×Ñ Ò&Ø $× 1Ñ 1°D×4DÑ4DÑ DÈ×HZÑHZÑ ZˆDÔà>B×>RÑ>RÐ>^œu T×%9Ñ%9Ô:ÐdhˆÔØ*-ˆÐ&Ñ'ä‰ÑÑ' Ó'ó    c                 óŠ   — t        | j                  «      D �cg c]   }| j                  r|| j                  v rdnd‘Œ" c}S c c}w )NÚfull_attentionÚlinear_attention)Úranger   r&   )rB   Úis     rE   r   zBambaConfig.layers_block_typeb   sK   € ô ˜4×1Ñ1Ó2ö
àð "&×!8Ò!8¸QÀ$×BYÑBYÑ=YÑÐ`rÑrò
ð 	
ùò 
s   ˜%A c                 ó¾   — | j                   | j                  z  }|| j                  z  dk7  rt        d«      ‚| j                  | j                  z  |k7  rt        d«      ‚y)zOPart of `@strict`-powered validation. Validates the architecture of the config.r   z4mamba_n_heads must divide mamba_expand * hidden_sizezPThe dimensions for the Mamba head state do not match the model intermediate_sizeN)r0   r   r(   Ú
ValueErrorr*   )rB   Úmamba_intermediates     rE   Úvalidate_architecturez!BambaConfig.validate_architecturei   sd   € à!×.Ñ.°×1AÑ1AÑAÐØ × 2Ñ 2Ñ2°aÒ7ÜÐSÓTÐTà×Ñ˜t×1Ñ1Ñ1Ð5GÒGÜÐoÓpÐpð HrF   )7Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚattribute_mapÚkeys_to_ignore_at_inferencer   ÚintÚ__annotations__r   Úboolr   r   r   r   r   r   Ústrr   Úfloatr   r   r   r    r!   r#   Úlistr$   r%   r&   r(   r*   r+   r-   r/   r0   r1   r2   r3   r4   r5   r7   r?   r8   r9   r   Údictr:   r;   rA   Úpropertyr   rO   Ú__classcell__)rD   s   @rE   r   r      s0  ø… ñð €JØ"Ð$7Ð8€MØ#4Ð"5Ðà€J�ÓØ %Ð˜Ó%Ø€K�ÓØ"Ð�sÓ"ØÐ�sÓØ!Ð˜Ó!Ø&'Ð˜˜t™Ó'Ø€J�ÓØ#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ%&Ð˜˜d™
Ó&Ø €L�#˜‘*Ó Ø €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø#)Ð˜SÓ)Ø,/Ð�u˜s‘{ TÑ)Ó/Ø+/Ð˜˜S™	 DÑ(Ó/Ø #€M�3˜‘:Ó#Ø%+€L�#˜‘)˜dÑ"Ó+Ø!"€N�C˜$‘JÓ"Ø #€M�3˜‘:Ó#Ø €L�#˜‘*Ó Ø €L�#˜‘*Ó Ø#&Ð�c˜D‘jÓ&Ø#'€O�T˜D‘[Ó'Ø#(€O�T˜D‘[Ó(Ø"'€M�5˜4‘<Ó'Ø"%€M�5˜4‘<Ó%ØADÁeÈEÃlÐ@S€O�T˜%‘[ 5¨°¨Ñ#6Ñ6¸Ñ=ÓSØ'*Ð˜ ™Ó*Ø48€O�^ dÑ*¨TÑ1Ó8Ø €N�DÓ Ø€HˆdÓô(ð ñ
ó ð
öqrF   r   N)rS   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r   Ú__all__r>   rF   rE   ú<module>re      sY   ðñ  å .å 3Ý 1Ý #ð Ùðð ,ôôQqÐ"ó Qqóó ðQqðh ˆ/�rF   