Ë
    ÜÓ^jÍ  ã                   óö  — d dl Z d dlmZ d dlmZ d dlZd dlmZ ddlmZ	 ddl
mZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9 ddl:m;Z; dZ< e"jz                  e>«      Z? G d„ dej                  j€                  «      ZA G d„ de8«      ZB G d„ de*«      ZC G d „ d!e1«      ZD G d"„ d#ej€                  «      ZE G d$„ d%ej€                  «      ZF G d&„ d'e2«      ZG G d(„ d)e6«      ZH G d*„ d+e5«      ZIe G d,„ d-e«      «       ZJ G d.„ d/e7eJ«      ZK G d0„ d1e3«      ZL G d2„ d3e4«      ZMg d4¢ZNy)5é    N)ÚCallable)Úcycle)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)Úforce_accelerate_hooks)Úlazy_load_kernel)Úcreate_causal_mask)ÚBaseModelOutputWithPastÚ SequenceClassifierOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Úmerge_with_config_defaults)Úresolve_internal_import)Úcapture_outputsé   )ÚLlamaRotaryEmbeddingÚapply_rotary_pos_emb)Úpad_tensor_by_sizeÚreshape_into_chunksÚsegment_sum)	ÚZambaAttentionÚZambaAttentionDecoderLayerÚZambaForCausalLMÚZambaForSequenceClassificationÚZambaHybridLayerÚZambaMambaDecoderLayerÚ
ZambaModelÚZambaRMSNormÚeager_attention_forwardé   )ÚZamba2ConfigzZyphra/Zamba2-2.7Bc                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚZamba2RMSNormGatedc                 ó˜   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        || _        y ©N)	ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilonÚ
group_size)ÚselfÚhidden_sizer7   ÚepsÚ	__class__s       €út/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/zamba2/modular_zamba2.pyr1   zZamba2RMSNormGated.__init__9   s6   ø€ Ü‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÔØ$ˆ�ó    c                 ób  — |j                   }|j                  t        j                  «      }|�?|t        j
                  j                  |j                  t        j                  «      «      z  }|j                  �^ }}|| j                  z  } |j                  g |¢|‘| j                  ‘­Ž }|j                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  } |j                  g |¢|| j                  z  ‘­Ž }| j                  |j                  |«      z  S )Nr   éÿÿÿÿT)Úkeepdim)ÚdtypeÚtor3   Úfloat32r   Ú
functionalÚsiluÚshaper7   ÚviewÚpowÚmeanÚrsqrtr6   r5   )	r8   Úhidden_statesÚgateÚinput_dtypeÚprefix_dimsÚlast_dimÚgroup_countÚhidden_states_groupÚvariances	            r<   ÚforwardzZamba2RMSNormGated.forward?   s  € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØÐØ)¬B¯M©M×,>Ñ,>¸t¿w¹wÄuÇ}Á}Ó?UÓ,VÑVˆMØ!.×!4Ñ!4Ñˆ�hØ $§/¡/Ñ1ˆØ0˜m×0Ñ0Ð\°+Ð\¸{Ð\ÈDÏOÉOÒ\ÐØ&×*Ñ*¨1Ó-×2Ñ2°2¸tÐ2ÓDˆØ1´E·K±KÀÈ4×K`ÑK`Ñ@`Ó4aÑaÐØ0Ð+×0Ñ0Ð]°+Ð]¸{ÈTÏ_É_Ñ?\Ò]ˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r=   )g�íµ ÷Æ°>r/   )Ú__name__Ú
__module__Ú__qualname__r1   rS   Ú__classcell__©r;   s   @r<   r-   r-   8   s   ø„ õ%÷;r=   r-   c                   ó   — e Zd Zy)ÚZamba2RMSNormN©rT   rU   rV   © r=   r<   rZ   rZ   M   ó   „ Ør=   rZ   c                   ó   — e Zd Zy)ÚZamba2RotaryEmbeddingNr[   r\   r=   r<   r_   r_   Q   r]   r=   r_   c                   óF  ‡ — e Zd ZdZ	 	 	 ddededz  dedz  dedz  fˆ fd„Z	 	 	 ddej                  ded	ej                  dz  d
e	dz  de
ej                  ej                  f   dz  dee   de
ej                  ej                  dz  e
ej                     dz  f   fd„Zˆ xZS )ÚZamba2AttentionaZ  
    Multi-headed attention from 'Attention Is All You Need' paper.

    Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
    The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
    The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
    (see fig. 2 in https://huggingface.co/papers/2405.16712).
    Additionally, replaced
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)
    Finally, this attention layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this
    layer is tied, un-tied adapters (formally the same as LoRA but used in the base model) modules are added to the q, k, v projectors to increase
    expressivity with a small memory overhead (see Fig. 2 of https://huggingface.co/papers/2411.15242).
    NÚconfigÚ	layer_idxÚnum_fwd_mem_blocksÚblock_idc           	      óð  •— t         ‰| �  ||«       || _        |j                  | _        || _        |j                  �r…t        j                  g «      | _	        t        j                  g «      | _
        t        j                  g «      | _        t        | j                  «      D �]  }||j                  z  |k(  �r{t        j                  t        j                  | j                   | j"                  j$                  d¬«      t        j                  | j"                  j$                  | j                   d¬«      «      }t        j                  t        j                  | j                   | j"                  j$                  d¬«      t        j                  | j"                  j$                  | j                   d¬«      «      }t        j                  t        j                  | j                   | j"                  j$                  d¬«      t        j                  | j"                  j$                  | j                   d¬«      «      }n<t        j&                  «       }t        j&                  «       }t        j&                  «       }| j                  j)                  |«       | j                  j)                  |«       | j                  j)                  |«       �Œ! t+        | j                  «      D �	�
ci c]  \  }	}
|
|	“Œ
 c}
}	| _        y c c}
}	w )NF©Úbias)r0   r1   rd   Úhybrid_layer_idsÚlayer_block_mapre   Úuse_shared_attention_adapterr   Ú
ModuleListÚlinear_q_adapter_listÚlinear_k_adapter_listÚlinear_v_adapter_listÚrangeÚnum_mem_blocksÚ
SequentialÚLinearÚattention_hidden_sizerb   Úadapter_rankÚIdentityÚappendÚ	enumerateÚ	layer_dic)r8   rb   rc   rd   re   ÚiÚlinear_q_adapterÚlinear_k_adapterÚlinear_v_adapterÚindexÚvaluer;   s              €r<   r1   zZamba2Attention.__init__e   s#  ø€ ô 	‰Ñ˜ Ô+Ø"4ˆÔØ%×6Ñ6ˆÔØ ˆŒà×.Ó.Ü)+¯©°rÓ):ˆDÔ&Ü)+¯©°rÓ):ˆDÔ&Ü)+¯©°rÓ):ˆDÔ&ä˜4×2Ñ2Ó3ó D�Ø�v×,Ñ,Ñ,°Ó8Ü')§}¡}ÜŸ	™	 $×"<Ñ"<¸d¿k¹k×>VÑ>VÐ]bÔcÜŸ	™	 $§+¡+×":Ñ":¸D×<VÑ<VÐ]bÔcó(Ð$ô (*§}¡}ÜŸ	™	 $×"<Ñ"<¸d¿k¹k×>VÑ>VÐ]bÔcÜŸ	™	 $§+¡+×":Ñ":¸D×<VÑ<VÐ]bÔcó(Ð$ô (*§}¡}ÜŸ	™	 $×"<Ñ"<¸d¿k¹k×>VÑ>VÐ]bÔcÜŸ	™	 $§+¡+×":Ñ":¸D×<VÑ<VÐ]bÔcó(Ñ$ô
 (*§{¡{£}Ð$Ü')§{¡{£}Ð$Ü')§{¡{£}Ð$Ø×*Ñ*×1Ñ1Ð2BÔCØ×*Ñ*×1Ñ1Ð2BÔCØ×*Ñ*×1Ñ1Ð2BÖCð)Dô, <EÀT×EYÑEYÓ;Z×[©<¨5°%˜% ™,Ó[ˆ�ùÓ[s   ËK2rK   Úattention_maskÚpast_key_valuesÚposition_embeddingsÚkwargsÚreturnc                 ó  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      }	| j                  |«      }
| j	                  |«      }| j
                  j                  rW| j                  |   }|	 | j                  |   |«      z   }	|
 | j                  |   |«      z   }
| | j                  |   |«      z   }|	j                  |«      j                  dd«      }	|
j                  |«      j                  dd«      }
|j                  |«      j                  dd«      }| j
                  j                  r|\  }}t        |	|
||«      \  }	}
|�|j                  |
||«      \  }
}t!        j"                  | j
                  j$                  t&        «      } || |	|
||f| j(                  sdn| j*                  | j,                  dœ|¤Ž\  }} |j.                  g |¢d‘­Ž j1                  «       }| j3                  |«      }||fS )Nr?   r*   r   g        )ÚdropoutÚscaling)rF   Úhead_dimÚq_projÚk_projÚv_projrb   rk   ry   rm   rn   ro   rG   Ú	transposeÚuse_mem_roper   Úupdater   Úget_interfaceÚ_attn_implementationr)   ÚtrainingÚattention_dropoutr‡   ÚreshapeÚ
contiguousÚo_proj)r8   rK   rc   r€   r�   r‚   rƒ   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚadapter_layer_idxÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                     r<   rS   zZamba2Attention.forwardŽ   s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆØ�;‰;×3Ò3Ø $§¡¨yÑ 9ÐØ'Ð*W¨$×*DÑ*DÐEVÑ*WÐXeÓ*fÑfˆLØ#Ð&S d×&@Ñ&@ÐARÑ&SÐTaÓ&bÑbˆJØ'Ð*W¨$×*DÑ*DÐEVÑ*WÐXeÓ*fÑfˆLà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà�;‰;×#Ò#Ø*‰HˆC�Ü';¸LÈ*ÐVYÐ[^Ó'_Ñ$ˆL˜*àÐ&Ø'6×'=Ñ'=¸jÈ,ÐXaÓ'bÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r=   ©NNN)rT   rU   rV   Ú__doc__r+   Úintr1   r3   ÚTensorr	   Útupler   r   rS   rW   rX   s   @r<   ra   ra   U   sù   ø„ ñð$ !%Ø)-Ø#ñ'\àð'\ð ˜‘:ð'\ð   $™Jð	'\ð
 ˜‘*õ'\ðZ /3Ø(,ØHLñ1)à—|‘|ð1)ð ð1)ð Ÿ™ tÑ+ð	1)ð
  ™ð1)ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð1)ð Ð+Ñ,ð1)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷1)r=   ra   c                   ó   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej                  de	dz  dej                  dz  fd	„Z
dde	dz  dej                  dz  fd
„Z ed«      	 	 dde	dz  dej                  dz  fd„«       Zˆ xZS )ÚZamba2MambaMixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    Nrb   rc   c           	      ó&  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        |j                  | _        t        |j                  | j                  z  «      | _
        || _        |j                  | _        d| _        t        j                  «       | _        |j"                  | _        |j$                  | _        |j(                  | _        | j                  j,                  | _        |j0                  | _        |j2                  | _        |j4                  | _        |j6                  | _        | j                  d| j&                  z  | j
                  z  z   | _        t        j:                  | j8                  | j8                  d|j                  | j8                  |j                  dz
  ¬«      | _        | j                  | j8                  z   | j.                  z   }t        j>                  | j                  ||j@                  ¬«      | _!        t        jD                  tG        jH                  | j.                  «      «      | _%        tG        jL                  d| j.                  dz   «      }t        jD                  tG        jN                  |«      «      | _(        tS        | j                  | j                  | j&                  z  d¬«      | _*        t        jD                  tG        jH                  | j.                  «      «      | _+        t        j>                  | j                  | j                  |j@                  ¬«      | _,        |jZ                  r|t]        d	«      }t_        |d
d «      a0t_        |dd «      a1t]        d«      }te        |d¬«      a3te        |d¬«      a4te        |d¬«      a5tm        tf        th        tj        tb        t`        f«      a7nd a0d a1d a3d a4d a5da7t_        |dd«      rtn        stp        js                  d«       |jt                  |   | _;        y )NrE   r   Tr*   )Úin_channelsÚout_channelsrh   Úkernel_sizeÚgroupsÚpaddingrg   gñhãˆµøä>)r7   r:   zcausal-conv1dÚcausal_conv1d_updateÚcausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)Úchained_pathz1ops.triton.ssd_combined.mamba_chunk_scan_combinedz8ops.triton.ssd_combined.mamba_split_conv1d_scan_combinedFÚuse_mamba_kernelsa  The fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d)<r0   r1   rb   r9   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizer£   Úmamba_expandÚintermediate_sizerc   Úuse_conv_biasÚ
activationr   ÚSiLUÚactÚuse_mem_eff_pathÚmamba_ngroupsÚn_groupsÚmamba_headdimrˆ   Ún_mamba_headsÚ	num_headsÚ
chunk_sizeÚtime_step_limitÚtime_step_minÚtime_step_maxÚconv_dimÚConv1dÚconv1drs   Úadd_bias_linearÚin_projr2   r3   r4   Údt_biasÚarangeÚlogÚA_logr-   ÚnormÚDÚout_projr±   r   Úgetattrr®   r¯   r   Úselective_state_updateÚmamba_chunk_scan_combinedÚ mamba_split_conv1d_scan_combinedÚallÚis_fast_path_availableÚloggerÚwarning_onceÚlayer_typesÚ
layer_type)r8   rb   rc   Úprojection_sizeÚAÚcausal_conv1dÚ	mamba_ssmr;   s          €r<   r1   zZamba2MambaMixer.__init__Ê   sW  ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ$×2Ñ2ˆÔØ &× 3Ñ 3ˆÔÜ!$ V×%8Ñ%8¸4×;KÑ;KÑ%KÓ!LˆÔØ"ˆŒØ#×1Ñ1ˆÔØ ˆŒÜ—7‘7“9ˆŒØ &× 7Ñ 7ˆÔà×,Ñ,ˆŒØ×,Ñ,ˆŒØŸ™×2Ñ2ˆŒØ ×+Ñ+ˆŒà%×5Ñ5ˆÔØ#×1Ñ1ˆÔØ#×1Ñ1ˆÔà×.Ñ.°°T·]±]Ñ1BÀT×EXÑEXÑ1XÑXˆŒÜ—i‘iØŸ™ØŸ™ØØ×+Ñ+Ø—=‘=Ø×'Ñ'¨!Ñ+ô
ˆŒð ×0Ñ0°4·=±=Ñ@À4Ç>Á>ÑQˆÜ—y‘yØ×ÑØØ×'Ñ'ô
ˆŒô —|‘|¤E§J¡J¨t¯~©~Ó$>Ó?ˆŒô �L‰L˜˜DŸN™N¨QÑ.Ó/ˆÜ—\‘\¤%§)¡)¨A£,Ó/ˆŒ
Ü&Ø×"Ñ"¨t×/EÑ/EÈÏÉÑ/VÐ\`ô
ˆŒ	ô —‘œeŸj™j¨¯©Ó8Ó9ˆŒäŸ	™	 $×"8Ñ"8¸$×:JÑ:JÐQW×QgÑQgÔhˆŒð ×#Ò#Ü,¨_Ó=ˆMÜ#*¨=Ð:PÐRVÓ#WÐ Ü& }Ð6HÈ$ÓOÐä(¨Ó5ˆIÜ%<ØÐ(bô&Ð"ô )@ØÐ([ô)Ð%ô 0GØÐ(bô0Ð,ô &)ä*Ü-Ü4Ü$Ü(ðó&Ñ"ð $(Ð Ø#ÐØ%)Ð"Ø(,Ð%Ø/3Ð,Ø%*Ð"ä�6Ð.°Ô5Õ>TÜ×Ñð>ôð !×,Ñ,¨YÑ7ˆ�r=   rK   Úcache_paramsr€   c                 ó„  — |j                   \  }}}| j                  | j                  z  }d| j                  z  d| j                  z  | j                  z  z   | j                  z   }|d uxr |j                  | j                  «      }	|	rL|j                  | j                     j                  d   }
|j                  | j                     j                  d   }|	�r,|dk(  �r&| j                  |j                  d«      «      }|j                   d   |z
  dz  }||| j                  | j                  | j                  g}t        j                  ||d¬«      \  }}}}}t        |
| j                   j"                  j                  d«      | j                   j$                  | j&                  «      }t        j                  || j                  ||gd¬«      \  }}}t        j(                  | j*                  j-                  «       «       }|d d …d df   d d …d d …d f   j/                  d| j0                  | j                  «      j3                  t        j4                  ¬«      }|d d …d d …d f   j/                  dd| j0                  «      }| j6                  d d …d df   j/                  d| j0                  «      }| j8                  d d …d df   j/                  d| j0                  «      }|j;                  || j                  |j                   d   | j                  z  «      }|j;                  || j                  |j                   d   | j                  z  «      }|j;                  || j                  | j0                  «      }t=        ||||||d |d¬	«
      }|j;                  || j                  | j0                  z  «      }| j?                  ||«      }| jA                  |j3                  | j@                  j"                  jB                  «      «      d d …d df   }|S |�Bt        jD                  |dk(  «      s*|jB                  }||d d …d d …d f   z  j3                  |«      }| j                  |«      }t        j(                  | j*                  j-                  «       «       }| jF                  €i nd
| jF                  i}|�t        jD                  |dk(  «      }nd}| jH                  rõ| jJ                  ré|€ç|råtM        || j                   j"                  j                  d«      | j                   j$                  | j6                  |f| j8                  | jN                  d | j&                  | j>                  j"                  | j>                  jP                  | j@                  j"                  | j@                  j$                  | j0                  | j                  dddœ|¤Ž\  }}|S t        j                  || j                  | j                  | j                  gd¬«      \  }}}|jS                  dd«      }|	rt        jT                  
|gd¬«      }|�XtV        jX                  j[                  || j\                  |j                   d   z
  df«      }|j_                  || j                  «       t`        �| j&                  dvr5| jc                  | j!                  |«      dd |j                   d   …f   «      }nPta        || j                   j"                  j                  d«      | j                   j$                  | j&                  ¬«      }|	r|d d …d d …| d …f   }|jS                  dd«      }t        j                  || j                  ||gd¬«      \  }}}|�Bt        jD                  |dk(  «      s*|jB                  }||d d …d d …d f   z  j3                  |«      }te        |j;                  ||d| j0                  «      |||j;                  ||| j                  d«      |j;                  ||| j                  d«      f| jN                  | j8                  d d d| j6                  d|	rnd dœ|¤Ž\  } }|�|�|jg                  || j                  «       | j;                  ||d«      } | j?                  | |«      } | jA                  | j3                  | j@                  j"                  jB                  «      «      }|S )Nr   r   r*   r?   ©Údim.©rA   T)ÚzrË   Údt_softplusÚdt_limitF)rÐ   rÂ   Úseq_idxr¹   Úrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gateÚreturn_final_states)rE   Úswish)Úxr5   rh   r¹   )rÂ   rÐ   rå   rè   rð   rË   ræ   Úinitial_states)4rF   r¾   r³   r·   rÁ   Úhas_previous_staterc   ÚlayersÚconv_statesÚrecurrent_statesrÊ   ÚsqueezerÆ   r3   Úsplitr®   rÈ   r5   rh   r¹   ÚexprÎ   ÚfloatÚexpandrˆ   rB   rC   rË   rÐ   rG   rÓ   rÏ   rÑ   rA   rÖ   rÃ   r¼   r‘   rÕ   rÂ   r6   rŒ   Úcatr   rD   Úpadrµ   Úupdate_conv_stater¯   r»   rÔ   Úupdate_recurrent_state)!r8   rK   rà   r€   Ú
batch_sizeÚseq_lenÚ_Úgroups_time_state_sizeÚd_to_removeÚuse_precomputed_statesÚ
conv_stateÚrecurrent_stateÚin_projected_statesÚd_mlpÚsplit_projection_dimrL   Úhidden_states_B_CÚdtÚBÚCrÝ   rË   rÐ   Úhidden_states_reshapedÚoutrA   Úprojected_statesÚdt_limit_kwargsÚinput_not_maskedÚ	ssm_stateÚ	time_stepÚnew_conv_stateÚscan_outputs!                                    r<   Úcuda_kernels_forwardz%Zamba2MambaMixer.cuda_kernels_forward0  sñ  € ð "/×!4Ñ!4Ñˆ
�G˜QØ!%§¡°×1DÑ1DÑ!DÐØ˜$×0Ñ0Ñ0°1°t·}±}Ñ3DÀt×GZÑGZÑ3ZÑZÐ]a×]kÑ]kÑkˆà!-°TÐ!9Ò!m¸l×>]Ñ>]Ð^b×^lÑ^lÓ>mÐÙ!Ø%×,Ñ,¨T¯^©^Ñ<×HÑHÈÑKˆJØ*×1Ñ1°$·.±.ÑA×RÑRÐSTÑUˆOò " g°£lØ"&§,¡,¨}×/DÑ/DÀQÓ/GÓ"HÐØ(×.Ñ.¨rÑ2°[Ñ@ÀQÑFˆEØ$)¨5°$×2HÑ2HÈ$Ï-É-ÐY]×YgÑYgÐ#hÐ Ü05·±Ð<OÐQeÐkmÔ0nÑ-ˆAˆq�$Ð)¨2ä 4Ø!ØØ—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ Ø—‘ó!Ðô #(§+¡+Ø!Ø×'Ñ'Ð)?ÐAWÐXØô#ÑˆM˜1˜aô
 —‘˜4Ÿ:™:×+Ñ+Ó-Ó.Ð.ˆAà’!�T˜3�,‘¢¢1 d 
Ñ+×2Ñ2°2°t·}±}Àd×FYÑFYÓZ×]Ñ]Ôdi×dqÑdqÐ]ÓrˆAØ’A’q˜$�J‘×&Ñ& r¨2¨t¯}©}Ó=ˆBØ—l‘l¢1 d¨C <Ñ0×7Ñ7¸¸D¿M¹MÓJˆGØ—‘’q˜$ �|Ñ$×+Ñ+¨B°·±Ó>ˆAØ—‘�z 4§=¡=°!·'±'¸!±*ÀÇÁÑ2MÓNˆAØ—‘�z 4§=¡=°!·'±'¸!±*ÀÇÁÑ2MÓNˆAØ%2×%7Ñ%7¸
ÀDÇNÁNÐTX×TaÑTaÓ%bÐ"Ü2ØØ&ØØØØØØØØ ôˆMð *×.Ñ.¨z¸4¿>¹>ÈDÏMÉMÑ;YÓZˆMØ ŸI™I m°TÓ:ˆMð —-‘- × 0Ñ 0°·±×1EÑ1E×1KÑ1KÓ LÓMÊaÐQUÐWZÈlÑ[ˆCðL ˆ
ðG Ð)´%·)±)¸NÈaÑ<OÔ2Pà%×+Ñ+�Ø!.°ÂÂ1ÀdÀ
Ñ1KÑ!K× OÑ OÐPUÓ V�à#Ÿ|™|¨MÓ:ÐÜ—‘˜4Ÿ:™:×+Ñ+Ó-Ó.Ð.ˆAØ$(×$8Ñ$8Ð$@™bÀzÐSW×SgÑSgÐFhˆOØÐ)Ü#(§9¡9¨^¸qÑ-@Ó#AÑ à#'Ð à×$Ò$¨¯ª¸<Ð;OÑTdÜ!AØ$Ø—K‘K×&Ñ&×.Ñ.¨qÓ1Ø—K‘K×$Ñ$Ø—L‘LØð"ð —f‘fØ#Ÿ™Ø Ø#Ÿ™Ø#'§9¡9×#3Ñ#3Ø $§	¡	× :Ñ :Ø#'§=¡=×#7Ñ#7Ø!%§¡×!3Ñ!3Ø ŸM™MØ ŸM™MØ%*Ø(,ñ#"ð$ &ñ%"‘��Yðj ˆ
ô 6;·[±[Ø$Ø×+Ñ+¨T¯]©]¸D¿N¹NÐKØô6Ñ2�Ð'¨ð %6×$?Ñ$?ÀÀ1Ó$EÐ!Ù)ô ).¯	©	°:Ð?PÐ2QÐWYÔ(ZÐ%ØÐ+Ü%'§]¡]×%6Ñ%6Ø)¨D×,AÑ,AÐDU×D[ÑD[Ð\^ÑD_Ñ,_ÐabÐ+có&�Nð !×2Ñ2°>À4Ç>Á>ÔRÜ#Ð+¨t¯©ÐFWÑ/WØ(,¯©°·±Ð=NÓ1OÐPSÐUrÐWh×WnÑWnÐoqÑWrÐUrÐPrÑ1sÓ(tÑ%ä(8Ø+Ø#Ÿ{™{×1Ñ1×9Ñ9¸!Ó<Ø!Ÿ[™[×-Ñ-Ø#'§?¡?ô	)Ð%ñ *Ø(9º!ºQÀÀÁ	¸/Ñ(JÐ%Ø$5×$?Ñ$?ÀÀ1Ó$EÐ!Ü&+§k¡kØ%Ø×+Ñ+Ð-CÐE[Ð\Øô'Ñ#�˜q !ð
 "Ð-´e·i±iÀÐRSÑ@SÔ6Tà)×/Ñ/�EØ%2°^ÂAÂqÈ$ÀJÑ5OÑ%O×$SÑ$SÐTYÓ$Z�MÜ)BØ!×&Ñ& z°7¸BÀÇÁÓNØØØ—F‘F˜: w°·±¸rÓBØ—F‘F˜: w°·±¸rÓBð*ð  $Ÿ™Ø—f‘fØØ Ø(,Ø ŸL™LØ $Ù6L¡?ÐRVñ*ð &ñ*Ñ&�˜Yð  Ð(¨\Ð-EØ ×7Ñ7¸	À4Ç>Á>ÔRØ)×.Ñ.¨z¸7ÀBÓG�à"Ÿi™i¨°TÓ:�ð —m‘m K§N¡N°4·=±=×3GÑ3G×3MÑ3MÓ$NÓO�Øˆ
r=   c                 ó4  — |j                   \  }}}|j                  }|�-|j                  | j                  «      r| j	                  |«      }n1|�||d d …d d …d f   z  j                  |«      }| j	                  |«      }|j                   d   d| j                  z  z
  d| j                  z  | j                  z  z
  | j                  z
  dz  }	|j                  |	|	| j                  | j                  | j                  gd¬«      \  }}}
}}|j                  dd«      }|d uxr |j                  | j                  «      }|r&|j                  | j                     j                  d   }|rº|dk(  rµ|j                  || j                  «      d| j                    d …f   }t#        j$                  || j&                  j(                  d d …dd d …f   z  d¬«      }| j*                  r|| j&                  j,                  z  }| j/                  |«      j                  |«      d d …d df   }�n|rt#        j0                  |gd¬«      }|�jt2        j4                  j7                  || j                   |j                   d   z
  df«      }|j                  || j                  «      d| j                    d …f   }| j/                  | j'                  |«      dd |j                   d   …f   j                  dd«      «      }|r|d d …| d …d d …f   }|�*|j                  }||d d …d d …d f   z  j                  |«      }t#        j                  || j                  | j                  | j                  z  | j                  | j                  z  gd¬«      \  }}}t#        j8                  | j:                  j=                  «       «       }|�ri|dk(  �rc|j>                  dk(  r
|d d …d df   n|d d …dd d …f   d d …d df   }|j                  dd«      jA                  ||j                   d   | jB                  «      }| jD                  d   jA                  | jD                  j                   d   | jB                  «      }t"        j2                  j4                  jG                  ||j                  |j                  «      z   «      }t#        jH                  || jJ                  «      }|d   jA                  | j                  | jB                  | j                  «      j                  t"        jL                  ¬	«      }t#        j8                  |d   |z  «      }|jO                  || j                  d«      dd d d …f   }|jA                  || j                  | j                  | j                  z  |j                   d   «      jQ                  «       }|jO                  |d|j                   d   «      }|d   |dd d d …f   z  }|jO                  |d| jB                  «      }||d   z  }|j                  | j                     jR                  d   jU                  «       }||z  |z   }|jW                  || j                  «      }|jO                  || j                  d«      dd d d …f   }|jA                  || j                  | j                  | j                  z  |j                   d   «      jQ                  «       }|jO                  |d|j                   d   «      }|j                  |j                  «      }|jY                  || j                  z  | jB                  | j                  «      }|jY                  || j                  z  | j                  d«      }t#        jZ                  ||«      }|jY                  || j                  | jB                  «      }| j\                  d   jA                  | j\                  j                   d   | jB                  «      }|||z  z   j                  |j                  «      }|jO                  |d«      d d …d df   }�nLt2        j4                  jG                  || jD                  z   «      }t#        jH                  || jJ                  «      }|jO                  ||d| jB                  «      j=                  «       }|jO                  ||d| j                  «      j=                  «       }|jO                  ||d| j                  «      j=                  «       }|j_                  | j                  | j                  z  d| j                  ¬
«      }|j_                  | j                  | j                  z  d| j                  ¬
«      }| j`                  || j`                  z  z
  | j`                  z  }| j\                  d   tc        ||«      z  }||d   z  }|j                  |j                  «      |z  }||||fD �cg c]  }te        ||| j`                  «      ‘Œ c}\  }}}}|jg                  dddd«      }t#        jh                  |d¬«      }t#        j8                  tk        |«      «      } |d d …d d …d d …d d d …d d …f   |d d …d d …d d d …d d …d d …f   z  }!|!j%                  d¬«      }"|"d   | jg                  ddddd«      d   z  }#|#j%                  d¬«      }$|$d   |d d …d d …d f   z  j%                  d«      }%t#        j8                  |d d …d d …d d …dd …f   |z
  «      }&||&jg                  dddd«      d   z  }'|'jg                  ddddd«      d   |jg                  ddddd«      dd d d …f   z  j%                  d¬«      jg                  ddddd«      }(|rR|j                  | j                     jR                  d   d d …d f   j                  |(j                  |(jl                  ¬«      nt#        jn                  |(d d …d d…f   «      })t#        j0                  |)|(gd¬«      }(t#        j8                  tk        t2        j4                  j7                  |d d …d d …d d …df   d«      «      «      }*|(jg                  ddddd«      }+|*d   |+d d …d d …d df   z  j%                  d¬«      },|,jg                  ddddd«      }-|-d d …d d…f   |-d d …df   }.}(t#        j8                  |«      }/|dd d d …f   |(d d …d d …d df   z  }0|/jg                  dddd«      }1|0j%                  d«      |1d   z  }2|%|2z   }|jO                  |d| j                  | jB                  «      }||z   }|dkD  r|d d …d |…d d …d d …f   }|jO                  ||d«      }|.�|�|jW                  |.| j                  «       | jq                  ||
«      }3| js                  |3j                  |«      «      }4|4S c c}w )Nr?   r   râ   r*   r   .).N).NNrä   )rã   Úoutput_sizer   é   )rA   Údevice)r*   r   ):rF   rA   rô   rc   rÊ   rB   r·   r¾   r³   rÁ   rù   rÆ   rŒ   rõ   rö   rÿ   rµ   r3   ÚsumrÈ   r5   r¸   rh   r»   rý   r   rD   rþ   rú   rÎ   rû   Úndimrü   rˆ   rË   ÚsoftplusÚclamprÄ   rC   r“   r”   r÷   Úcloner   rG   ÚbmmrÐ   Úrepeat_interleaverÂ   r   r   ÚpermuteÚcumsumr    r  Ú
zeros_likerÏ   rÑ   )5r8   Úinput_statesrà   r€   r  r  r  rA   r  r
  rL   rK   r  Úuse_precomputed_stater  rö   r  r  rÝ   rË   ÚdAÚdBÚdBxÚ
ssm_statesÚssm_states_reshapedÚ
C_reshapedÚyrÐ   Úpad_sizeÚ
D_residualÚtÚA_cumsumÚLÚG_intermediateÚGÚM_intermediateÚMÚY_diagÚdecay_statesÚB_decay_contractionÚstatesÚprevious_statesÚdecay_chunkÚstates_permutedÚresultÚ
new_statesr  Ústate_decay_outÚC_times_statesÚstate_decay_out_permutedÚY_offr  Úcontextualized_statess5                                                        r<   Útorch_forwardzZamba2MambaMixer.torch_forwardØ  sê  € Ø!-×!3Ñ!3Ñˆ
�G˜QØ×"Ñ"ˆàÐ#¨×(GÑ(GÈÏÉÔ(WØ#Ÿ|™|¨LÓ9ÑàÐ)à ,¨~ºaÂÀD¸jÑ/IÑ I×MÑMÈeÓT�Ø#Ÿ|™|¨LÓ9ÐØ!×'Ñ'¨Ñ+¨a°$×2HÑ2HÑ.HÑHÈ1ÈtÏ}É}ÑK\Ð_c×_rÑ_rÑKrÑrÐtx÷  uCñ  uCñ  Cð  HIñ  IˆØ(8×(>Ñ(>Ø˜˜t×5Ñ5¸¿¹ÀtÇ~Á~ÐVÐ\^ð )?ó )
Ñ%ˆˆ1ˆd�M 2ð &×/Ñ/°°1Ó5ˆà ,°DÐ 8Ò l¸\×=\Ñ=\Ð]a×]kÑ]kÓ=lÐÙ Ø%×,Ñ,¨T¯^©^Ñ<×HÑHÈÑKˆJñ ! W°¢\Ø&×8Ñ8¸ÈÏÉÓWÐX[Ð^b×^sÑ^sÐ]sÑ]tÐXtÑuˆKÜ!ŸI™I k°D·K±K×4FÑ4FÂqÈ!ÊQÀwÑ4OÑ&OÐUWÔXˆMØ×!Ò!Ø §¡×!1Ñ!1Ñ1�Ø ŸH™H ]Ó3×6Ñ6°uÓ=ºaÀÀs¸lÑKŠMá$ä %§	¡	¨:°}Ð*EÈ2Ô N�ØÐ'Ü Ÿm™m×/Ñ/Ø!Ø×*Ñ*¨]×-@Ñ-@ÀÑ-DÑDÀaÐHó�ð +×<Ñ<¸[È$Ï.É.ÓYÐZ]Ð`d×`uÑ`uÐ_uÑ_vÐZvÑw�à ŸH™H T§[¡[°Ó%?ÀÐE]Àm×FYÑFYÐZ\ÑF]ÐE]Ð@]Ñ%^×%hÑ%hÐijÐlmÓ%nÓoˆMÙ$Ø -ªa°'°±ºA¨oÑ >�ØÐ)Ø%×+Ñ+�à!.°ÂÂ1ÀdÀ
Ñ1KÑ!K× OÑ OÐPUÓ V�ä#Ÿk™k¨-¸$×:PÑ:PÐRV×R_ÑR_Ðbf×buÑbuÑRuÐw{÷  xEñ  xEð  HL÷  H[ñ  H[ñ  x[ð  :\ð  bdô  eÑˆ�q˜!Ü�Y‰Y�t—z‘z×'Ñ'Ó)Ó*Ð*ˆÚ  W°£\ð &(§W¡W°¢\�’A�t˜S�LÒ!°rº!¸QÂ¸'±{Â1ÀdÈCÀ<Ñ7PˆBØ—‘˜a Ó#×*Ñ*¨:°r·x±xÀ±|ÀTÇ]Á]ÓSˆBà—l‘l 9Ñ-×4Ñ4°T·\±\×5GÑ5GÈÑ5JÈDÏMÉMÓZˆGä—‘×$Ñ$×-Ñ-¨b°7·:±:¸b¿h¹hÓ3GÑ.GÓHˆBÜ—‘˜R ×!3Ñ!3Ó4ˆBØ�/Ñ"×)Ñ)¨$¯.©.¸$¿-¹-È×I\ÑI\Ó]×`Ñ`Ôgl×gtÑgtÐ`ÓuˆAä—‘˜2˜i™=¨1Ñ,Ó-ˆBð
 —	‘	˜* d§m¡m°RÓ8¸¸dÂA¸ÑFˆAØ—‘˜ T§]¡]°D·N±NÀdÇmÁmÑ4SÐUV×U\ÑU\Ð]_ÑU`Óa×lÑlÓnˆAØ—	‘	˜* b¨!¯'©'°"©+Ó6ˆAà�I‘  3¨ªa <¡Ñ0ˆBð *×1Ñ1°*¸bÀ$Ç-Á-ÓPˆMØ�} YÑ/Ñ/ˆCð &×,Ñ,¨T¯^©^Ñ<×MÑMÈaÑP×VÑVÓXˆJØ# b™¨3Ñ.ˆJØ%×<Ñ<¸ZÈÏÉÓXˆJð —	‘	˜* d§m¡m°RÓ8¸¸dÂA¸ÑFˆAØ—‘˜ T§]¡]°D·N±NÀdÇmÁmÑ4SÐUV×U\ÑU\Ð]_ÑU`Óa×lÑlÓnˆAØ—	‘	˜* b¨!¯'©'°"©+Ó6ˆAð $Ÿ™ q§w¡wÓ/ˆJà",§/¡/°*¸t¿~¹~Ñ2MÈtÏ}É}Ð^b×^qÑ^qÓ"rÐØŸ™ 
¨T¯^©^Ñ ;¸T×=PÑ=PÐRSÓTˆJÜ—	‘	Ð-¨zÓ:ˆAØ—‘�z 4§>¡>°4·=±=ÓAˆAð —‘�yÑ!×(Ñ(¨¯©¯©°a©¸$¿-¹-ÓHˆAØ�] QÑ&Ñ&×*Ñ*¨1¯7©7Ó3ˆAð —	‘	˜* bÓ)ª!¨T°3¨,Ñ7ŠAô —‘×'Ñ'¨¨T¯\©\Ñ(9Ó:ˆBÜ—‘˜R ×!3Ñ!3Ó4ˆBØ)×1Ñ1°*¸gÀrÈ4Ï=É=ÓY×_Ñ_ÓaˆMØ—	‘	˜* g°°D×4GÑ4GÓH×NÑNÓPˆAØ—	‘	˜* g¨r°4×3FÑ3FÓG×MÑMÓOˆAØ×#Ñ# D§N¡N°d·m±mÑ$CÈÐX\×XfÑXfÐ#ÓgˆAØ×#Ñ# D§N¡N°d·m±mÑ$CÈÐX\×XfÑXfÐ#ÓgˆAØŸ™¨'°D·O±OÑ*CÑCÀtÇÁÑVˆHàŸ™ 	Ñ*Ô-?ÀÈxÓ-XÑXˆJð *¨B¨y©MÑ9ˆMØ—‘�]×(Ñ(Ó)¨BÑ.ˆAð cpÐqrÐtuÐwxÐayÖ%zÐ\]Ô&9¸!¸XÀtÇÁÕ&WÒ%zÑ"ˆM˜1˜a ð —	‘	˜!˜Q  1Ó%ˆAÜ—|‘| A¨2Ô.ˆHô —	‘	œ+ a›.Ó)ˆAð šq¢!¢Q¨ªa²Ð2Ñ3°aºº1¸dÂAÂqÊ!Ð8KÑ6LÑLˆNØ×"Ñ" rÐ"Ó*ˆAð ˜y™\¨A¯I©I°a¸¸A¸qÀ!Ó,DÀYÑ,OÑOˆNØ×"Ñ" rÐ"Ó*ˆAð ˜	‘l ]²1²a¸°:Ñ%>Ñ>×CÑCÀAÓFˆFô !Ÿ9™9 Xªa²²A°r±s¨lÑ%;¸hÑ%FÓGˆLØ"# l×&:Ñ&:¸1¸aÀÀAÓ&FÀyÑ&QÑ"QÐà)×1Ñ1°!°Q¸¸1¸aÓ@ÀÑKÈ}×OdÑOdÐefÐhiÐklÐnoÐqrÓOsÐtwÐy}ò  @Að  uAñ  PBñ  B÷  Gñ  Gð  LMð  Gó  N÷  Vñ  Vð  WXð  Z[ð  ]^ð  `að  cdó  eˆFñ )ð ×#Ñ# D§N¡NÑ3×DÑDÀQÑGÊÈ4ÈÑP×SÑSÐZ`×ZfÑZfÐou×o|Ño|ÐSÔ}ä×%Ñ% fªQ°°°¨U¡mÓ4ð ô
 —Y‘Y °Ð8¸aÔ@ˆFÜŸ)™)¤K´·±×0AÑ0AÀ(Ê1ÊaÒQRÐTVÈ;ÑBWÐY_Ó0`Ó$aÓbˆKà$Ÿn™n¨Q°°1°a¸Ó;ˆOØ! /Ñ2°_ÂQÊÈ4ÐQTÀ_Ñ5UÑU×ZÑZÐ_`ÐZÓaˆFØŸ™¨¨1¨a°°AÓ6ˆJØ *ª1¨c¨r¨c¨6Ñ 2°JºqÀ"¸uÑ4E�IˆFô $Ÿi™i¨Ó1ˆOà  Tª1 ™o°²qº!¸TÀ3°Ñ0GÑGˆNØ'6×'>Ñ'>¸qÀ!ÀQÈÓ'JÐ$Ø#×'Ñ'¨Ó+Ð.FÀyÑ.QÑQˆEð ˜‘ˆAà—	‘	˜* b¨$¯.©.¸$¿-¹-ÓHˆAà�J‘ˆAà˜!Š|Ø’a˜˜'˜¢1¢aÐ'Ñ(�Ø—	‘	˜* g¨rÓ2ˆAØÐ$¨Ð)AØ×3Ñ3°I¸t¿~¹~ÔNà—i‘i  4Ó(ˆð
 !%§¡¨k¯n©n¸UÓ.CÓ DÐØ$Ð$ùòK &{s   çvrÈ   c                 óÆ   — t         rId| j                  j                  j                  j                  v rt        «       s| j                  |||«      S | j                  |||«      S )NÚcuda)r×   rÊ   r5   r  Útyper   r  rH  )r8   rK   rà   r€   rƒ   s        r<   rS   zZamba2MambaMixer.forward™  sT   € õ " f°·±×0CÑ0C×0JÑ0J×0OÑ0OÑ&OÔXpÔXrØ×,Ñ,¨]¸LÈ.ÓYÐYà×!Ñ! -°¸~ÓNÐNr=   r/   ©NN)rT   rU   rV   r¢   r+   r£   r1   r3   r¤   r	   r  rH  r   rS   rW   rX   s   @r<   r§   r§   Â   sÐ   ø„ ññd8˜|ð d8¸¸d¹
õ d8ðR &*Ø.2ñ	eà—|‘|ðeð ˜d‘lðeð Ÿ™ tÑ+ó	eñP~%¸À¹ð ~%Ð[`×[gÑ[gÐjnÑ[nó ~%ñB ˜HÓ%ð &*Ø.2ñ	
Oð ˜d‘lð
Oð Ÿ™ tÑ+ò	
Oó &ô
Or=   r§   c                   ó8   ‡ — e Zd Zddededz  fˆ fd„Zdd„Zˆ xZS )Ú	Zamba2MLPNrb   re   c           	      ó   •— t         ‰	| �  «        || _        |j                  | _        |j                  | _        || _        || _        t        j                  | j                  d| j                  z  |j                  ¬«      | _
        t        j                  | j                  | j                  |j                  ¬«      | _        t        |j                     | _        t        j                  g «      | _        t#        | j
                  «      D ]Ï  }||j$                  z  |k(  rŒt        j&                  t        j                  | j                  j                  | j                  j(                  d¬«      t        j                  | j                  j(                  d| j                  z  d¬«      «      }nt        j*                  «       }| j                   j-                  |«       ŒÑ |j.                  }t1        |«      D ��ci c]  \  }}||“Œ
 c}}| _        yc c}}w )aQ  
        This MLP layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this layer
        is tied, un-tied adapter modules (formally same as LoRA, but used in the base model) are added to the up and gate projectors to increase expressivity with a small memory overhead.
        r   rg   FN)r0   r1   rb   r9   r·   rd   re   r   rs   rÉ   Úgate_up_projÚ	down_projr   Ú
hidden_actÚact_fnrl   Úgate_up_proj_adapter_listrp   rq   rr   ru   rv   rw   ri   rx   ry   )
r8   rb   rd   re   rz   Úgate_up_proj_adapterrj   r~   r   r;   s
            €r<   r1   zZamba2MLP.__init__¨  s”  ø€ ô
 	‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔØ"4ˆÔØ ˆŒäŸI™I d×&6Ñ&6¸¸D×<RÑ<RÑ8RÐY_×YoÑYoÔpˆÔÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRX×RhÑRhÔiˆŒÜ˜V×.Ñ.Ñ/ˆŒä)+¯©°rÓ):ˆÔ&Ü�t×.Ñ.Ó/ò 	HˆAØ�6×(Ñ(Ñ(¨HÒ4Ü')§}¡}Ü—I‘I˜dŸk™k×5Ñ5°t·{±{×7OÑ7OÐV[Ô\Ü—I‘I˜dŸk™k×6Ñ6¸¸D×<RÑ<RÑ8RÐY^Ô_ó(Ñ$ô
 (*§{¡{£}Ð$Ø×*Ñ*×1Ñ1Ð2FÕGð	Hð !×1Ñ1ˆÜ;DÀ_Ó;U×V©<¨5°%˜% ™,ÓVˆ�ùÓVs   Ç3H
c                 óü   — | j                  |«      }| j                  |   }| | j                  |   |«      z   }t        j                  |dd¬«      }| j                  |d   «      |d   z  }| j                  |«      }|S )Nr   r?   râ   r   r*   )rP  ry   rT  r3   ÚchunkrS  rQ  )r8   Úhidden_staterc   Úgate_up_stateÚoutputs        r<   rS   zZamba2MLP.forwardÆ  s€   € Ø×)Ñ)¨,Ó7ˆØ—N‘N 9Ñ-ˆ	Ø%Ð(Q¨×(FÑ(FÀyÑ(QÐR^Ó(_Ñ_ˆäŸ™ M°1¸"Ô=ˆØ—{‘{ =°Ñ#3Ó4°}ÀQÑ7GÑGˆØ—‘ Ó-ˆØˆr=   rL  r/   )rT   rU   rV   r+   r£   r1   rS   rW   rX   s   @r<   rN  rN  §  s%   ø„ ñW˜|ð WÐPSÐVZÑPZõ W÷<r=   rN  c                   óì   ‡ — e Zd Zddededz  dedz  fˆ fd„Z	 	 	 ddej                  dej                  dedej                  dz  d	edz  d
ej                  dz  de
e   deej                     fd„Zˆ xZS )ÚZamba2AttentionDecoderLayerNrb   re   rc   c                 ó¬   •— || _         t        |j                  «      }t        ‰| �  ||«       t        |d||¬«      | _        t        |||¬«      | _        y )Nr?   )rc   rd   re   )rd   re   )	re   Úlenri   r0   r1   ra   Ú	self_attnrN  Úfeed_forward)r8   rb   re   rc   Únum_gsr;   s        €r<   r1   z$Zamba2AttentionDecoderLayer.__init__Ò  sO   ø€ Ø ˆŒÜ�V×,Ñ,Ó-ˆÜ‰Ñ˜ Ô+Ü(¨¸2ÐRXÐckÔlˆŒÜ% fÀÐRZÔ[ˆÕr=   rK   Úoriginal_hidden_statesr€   r�   r‚   rƒ   r„   c           	      óÖ   — t        j                  ||gd¬«      }| j                  |«      } | j                  d|||||dœ|¤Ž\  }}| j	                  |«      }| j                  ||«      }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`.
                This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The
                concatenated tensor is then used as input of the pre-attention RMSNorm
                (see fig. 2 in https://huggingface.co/papers/2405.16712).
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        r?   râ   )rK   rc   r€   r�   r‚   r\   )r3   ÚconcatenateÚinput_layernormr_  Úpre_ff_layernormr`  )	r8   rK   rb  rc   r€   r�   r‚   rƒ   r  s	            r<   rS   z#Zamba2AttentionDecoderLayer.forwardÙ  sŠ   € ô6 ×)Ñ)¨=Ð:PÐ*QÐWYÔZˆØ×,Ñ,¨]Ó;ˆØ)˜4Ÿ>™>ð 
Ø'ØØ)Ø+Ø 3ñ
ð ñ
Ñˆ�qð ×-Ñ-¨mÓ<ˆØ×)Ñ)¨-¸ÓCˆàÐr=   rL  r¡   )rT   rU   rV   r+   r£   r1   r3   r¤   r	   Ú
LongTensorr   r   r¥   ÚFloatTensorrS   rW   rX   s   @r<   r\  r\  Ñ  s»   ø„ ñ\˜|ð \°s¸T±zð \ÐUXÐ[_ÑU_õ \ð /3Ø(,Ø7;ñ)à—|‘|ð)ð !&§¡ð)ð ð	)ð
 Ÿ™ tÑ+ð)ð  ™ð)ð #×-Ñ-°Ñ4ð)ð Ð+Ñ,ð)ð 
ˆu× Ñ Ñ	!÷)r=   r\  c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚZamba2MambaDecoderLayerrb   rc   c                 ó–   •— t         ‰| �  ||«       t        ||¬«      | _        t	        |j
                  |j                  ¬«      | _        y )N)rb   rc   ©r:   )r0   r1   r§   ÚmambarZ   r9   Úrms_norm_epsre  )r8   rb   rc   r;   s      €r<   r1   z Zamba2MambaDecoderLayer.__init__  s;   ø€ Ü‰Ñ˜ Ô+Ü%¨V¸yÔIˆŒ
Ü,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÕr=   )rT   rU   rV   r+   r£   r1   rW   rX   s   @r<   rj  rj    s   ø„ ðZ˜|ð Z¸÷ Zñ Zr=   rj  c                   óŠ  ‡ — e Zd Zdedej
                  defˆ fd„Z	 	 	 	 	 	 	 	 ddej                  dej                  dz  de
dz  d	ej                  dz  d
ej                  dz  dedz  dedz  dej                  dz  dej                  dz  dee   deej"                  eej"                  ej"                  f   dz  f   fd„Zˆ xZS )ÚZamba2HybridLayerÚshared_transformerÚlinearrm  c                 ó:   •— t         ‰| �  |||«       | `|| _        y r/   )r0   r1   Úshared_transfrq  )r8   rq  rr  rm  r;   s       €r<   r1   zZamba2HybridLayer.__init__  s%   ø€ ô 	‰ÑÐ+¨V°UÔ;ØÐØ"4ˆÕr=   NrK   rb  rc   r€   Úcausal_maskr�   Ú	use_cacher‚   Úposition_idsrƒ   r„   c
           
      óŽ   —  | j                   |f||||||	dœ|
¤Ž}| j                  |«      } | j                  |f|||||dœ|
¤Ž}|S )ap  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with
            hidden activations to form the input of the shared transformer layer.
            layer_idx (`int`): layer number.
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        )rb  rc   r€   r�   r‚   rw  )Útransformer_hidden_statesr€   r�   rv  r‚   )rq  rr  Úmamba_decoder)r8   rK   rb  rc   r€   ru  r�   rv  r‚   rw  rƒ   ry  s               r<   rS   zZamba2HybridLayer.forward  sˆ   € ð< %< D×$;Ñ$;Øð	%
à#9ØØ&Ø+Ø 3Ø%ñ	%
ð ñ	%
Ð!ð %)§K¡KÐ0IÓ$JÐ!à*˜×*Ñ*Øð
à&?Ø)Ø+ØØ 3ñ
ð ñ
ˆð Ðr=   )NNNNNFNN)rT   rU   rV   r\  r   rs   rj  r1   r3   r¤   r£   r	   Úboolrg  r   r   r¥   rh  rS   rW   rX   s   @r<   rp  rp    s"  ø„ ð5Ø"=ð5ØGIÇyÁyð5ØYpõ5ð 7;Ø $Ø.2Ø+/Ø(,Ø!&Ø7;Ø04ñ4à—|‘|ð4ð !&§¡¨tÑ 3ð4ð ˜‘:ð	4ð
 Ÿ™ tÑ+ð4ð —\‘\ DÑ(ð4ð  ™ð4ð ˜$‘;ð4ð #×-Ñ-°Ñ4ð4ð ×&Ñ&¨Ñ-ð4ð Ð+Ñ,ð4ð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷4r=   rp  c                   ó‚   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZeedœZ ej"                  «       ˆ fd„«       Zˆ xZS )	ÚZamba2PreTrainedModelrb   ÚmodelTrp  rj  r�   )rK   Ú
attentionsc                 óˆ  •— t         ‰| �  |«       t        |t        «      �r¡t	        j
                  t	        j                  | j                  j                  «      t        j                  | j                  j                  «      t        j                  | j                  j                  «      z
  z  t        j                  | j                  j                  «      z   «      j                  | j                  j                  ¬«      }|t	        j                  t	        j                  | «       «      z   }t!        j"                  |j$                  |«       t	        j&                  d|j(                  dz   «      }t!        j"                  |j*                  t	        j                  |«      «       t!        j,                  |j.                  «       y y )N)Úminr*   )r0   Ú_init_weightsÚ
isinstancer§   r3   rú   Úrandrb   rÀ   ÚmathrÍ   rÅ   rÄ   r!  Útime_step_floorÚexpm1ÚinitÚcopy_rË   rÌ   rÁ   rÎ   Úones_rÐ   )r8   Úmoduler  Úinv_dtrÝ   r;   s        €r<   r‚  z#Zamba2PreTrainedModel._init_weights[  s(  ø€ ä‰Ñ˜fÔ%Ü�fÔ.Õ/Ü—‘Ü—
‘
˜4Ÿ;™;×4Ñ4Ó5Ü—8‘8˜DŸK™K×5Ñ5Ó6¼¿¹À$Ç+Á+×B[ÑB[Ó9\Ñ\ñ^ä—(‘(˜4Ÿ;™;×4Ñ4Ó5ñ6ó÷ ‰e˜Ÿ™×3Ñ3ˆeÓ4ð	 ð œ%Ÿ)™)¤U§[¡[°"°Ó%5Ð$5Ó6Ñ6ˆFÜ�J‰J�v—~‘~ vÔ.ä—‘˜Q × 0Ñ 0°1Ñ 4Ó5ˆAÜ�J‰J�v—|‘|¤U§Y¡Y¨q£\Ô2Ü�J‰J�v—x‘xÕ ð 0r=   )rT   rU   rV   r+   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpaÚ_is_statefulrj  ra   Ú_can_record_outputsr3   Úno_gradr‚  rW   rX   s   @r<   r}  r}  K  sj   ø… àÓØÐØ&*Ð#Ø,Ð.GÐHÐØ#4Ð"5ÐØÐØÐØ€NØ€Là0Ø%ñÐð
 €U‡]�]ƒ_ó!ó ô!r=   r}  c                   óö   — e Zd ZdZdefd„Zd„ Zeee		 	 	 	 	 	 dde
j                  dz  de
j                  dz  de
j                  dz  d	edz  d
e
j                  dz  dedz  dee   deez  fd„«       «       «       Zy)ÚZamba2Modelzh
    Model consisting of *config.num_hidden_layers* layers.

    Args:
        config: Zamba2Config
    rb   c                 óX  — t         j                  | |«       || _        |j                  | _        |j
                  | _        t        j                  |j
                  |j                  | j                  «      | _	        |j                  | _
        | j                  «       | _        |j                  | _        t        |j                  |j                  ¬«      | _        |j"                  r1|j$                  rt&        j)                  d«       t+        |«      | _        d| _        | j1                  «        y )Nrl  ze`use_long_context` set to `True`: using rescaled `rope_theta` and extended `max_position_embeddings`.F)r}  r1   rb   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr9   Úembed_tokensÚlayers_block_typeÚ
get_layersrõ   r�   rZ   rn  Úfinal_layernormr�   Úuse_long_contextrØ   rÙ   r_   Ú
rotary_embÚgradient_checkpointingÚ	post_init)r8   rb   s     r<   r1   zZamba2Model.__init__u  sæ   € Ü×&Ñ& t¨VÔ4ØˆŒØ!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔØ!'×!9Ñ!9ˆÔØ—o‘oÓ'ˆŒà$*×$?Ñ$?ˆÔ!Ü,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔØ×ÒØ×&Ò&Ü×#Ñ#Ø{ôô 4°FÓ;ˆDŒOØ&+ˆÔ#ð 	�‰Õr=   c                 ó   — g }i | _         d| _        g }t        | j                  «      D �]O  \  }}t	        | j
                  |¬«      }|dk(  �rd|› d�}t        |t        «      r"t        |«      | j
                  j                  k\  rDt        |t        «      rt        |«      }t        |«      }| j                   j                  ||i«       n|j                  |«       || j
                  j                  z  }t        | j
                  |¬«      }	t        j                   | j
                  j"                  | j
                  j"                  d¬«      }
|j                  t%        |	|
|«      «       �Œ?|j                  |«       �ŒR t        j&                  |«      S )	Nr   )rc   Úhybridzlayers.z.shared_transformer)re   Frg   )Ú_tied_weights_keysÚfirst_transformer_layer_idrx   r   rj  rb   rƒ  Úlistr^  rq   r   ÚnextrŽ   rw   r\  r   rs   r9   rp  rl   )r8   rõ   Úunique_hybrid_blocksÚlayer_idrÛ   Úmamba_layerÚprefix_patternÚtarget_patternre   Ú
attn_blockÚlinear_layers              r<   r¡  zZamba2Model.get_layersŒ  sQ  € ØˆØ"$ˆÔØ*+ˆÔ'Ø!Ðä$-¨d×.DÑ.DÓ$Eó 	+Ñ ˆH�jÜ1°$·+±+ÈÔRˆKØ˜XÓ%Ø#*¨8¨*Ð4GÐ!H�ô #Ð#7¼Ô>ÜÐ/Ó0°D·K±K×4NÑ4NÒNä!Ð"6¼Ô=Ü/4Ð5IÓ/JÐ,Ü%)Ð*>Ó%?�NØ×+Ñ+×2Ñ2°NÀNÐ3SÕTð )×/Ñ/°Ô?à# d§k¡k×&@Ñ&@Ñ@�Ü8¸¿¹ÈxÔX�
Ü!Ÿy™y¨¯©×)@Ñ)@À$Ç+Á+×BYÑBYÐ`eÔf�Ø—‘Ô/°
¸LÈ+ÓVÖWà—‘˜kÖ*ð3	+ô4 �}‰}˜VÓ$Ð$r=   NÚ	input_idsr€   rw  r�   Úinputs_embedsrv  rƒ   r„   c           	      óš  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|}t        j                  |«      }	|r|€t	        | j
                  ¬«      }|€V|�|j                  «       nd}
t        j                  |j                  d   |j                  ¬«      |
z   }|j                  d«      }t        | j
                  ||||¬«      }| j
                  j                  r| j                  ||¬«      }nd }t        | j                  «      D ]  \  }} |||	|||f||||dœ|¤Ž}Œ | j!                  |«      }t#        ||r|¬	«      S d ¬	«      S )
NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either one)rb   r   r*   ©r  )rb   rµ  r€   r�   rw  )rw  )r�   rv  r‚   rw  )Úlast_hidden_stater�   )Ú
ValueErrorrŸ  r3   r"  r
   rb   Úget_seq_lengthrÌ   rF   r  Ú	unsqueezer   r�   r¤  rx   rõ   r¢  r   )r8   r´  r€   rw  r�   rµ  rv  rƒ   rK   rb  Úpast_seen_tokensru  r‚   rc   Úlayers                  r<   rS   zZamba2Model.forward®  sŠ  € ð ˜Ð -°tÐ";Ò<ÜØsóð ð Ð Ø ×-Ñ-¨iÓ8ˆMà%ˆä!&§¡¨]Ó!;Ðñ ˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð �;‰;×#Ò#Ø"&§/¡/°-Èl /Ó"[Ñà"&Ðä )¨$¯+©+Ó 6ò 	ÑˆI�uÙ!ØØ&ØØØðð !0Ø#Ø$7Ø)ñð ñ‰Mð	ð ×,Ñ,¨]Ó;ˆä&Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
r=   )NNNNNN)rT   rU   rV   r¢   r+   r1   r¡  r   r   r   r3   rg  r¤   r	   rh  r{  r   r   r¥   r   rS   r\   r=   r<   r™  r™  m  sç   „ ñð˜|ó ò. %ðD  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ@
à×#Ñ# dÑ*ð@
ð Ÿ™ tÑ+ð@
ð ×&Ñ&¨Ñ-ð	@
ð
  ™ð@
ð ×(Ñ(¨4Ñ/ð@
ð ˜$‘;ð@
ð Ð+Ñ,ð@
ð 
Ð(Ñ	(ò@
ó ó ó  ñ@
r=   r™  c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚZamba2ForCausalLMrb   c                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y r/   ©r0   r1   r™  r~  r¦  ©r8   rb   r;   s     €r<   r1   zZamba2ForCausalLM.__init__õ  ó&   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø�‰Õr=   )rT   rU   rV   r+   r1   rW   rX   s   @r<   r¿  r¿  ô  s   ø„ ð˜|÷ ñ r=   r¿  c                   ó.  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de
dz  dej                  dz  d	ej                  dz  d
edz  deej                  z  dee   deez  fd„«       «       Zˆ xZS )ÚZamba2ForSequenceClassificationrb   c                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y r/   rÁ  rÂ  s     €r<   r1   z(Zamba2ForSequenceClassification.__init__ü  rÃ  r=   Nr´  r€   rw  r�   rµ  Úlabelsrv  Úlogits_to_keeprƒ   r„   c	           	      ó˜  —  | j                   |f|||||dœ|	¤Ž}
|
d   }| j                  |«      }|�|j                  d   }n|j                  d   }| j                  j                  €|dk7  rt        d«      ‚| j                  j                  €d}nÃ|�“|| j                  j                  k7  j                  |j                  t        j                  «      }t        j                  |j                  d   |j                  t        j                  ¬«      }||z  j                  d«      }n.d}t        j                  | j                  j                  › d�«       |t        j                  ||j                  ¬	«      |f   }d}|�! | j                   d|||| j                  d
œ|	¤Ž}t#        |||
j$                  |
j&                  |
j(                  ¬«      S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        )r€   rw  r�   rµ  rv  r   Nr*   z=Cannot handle batch sizes > 1 if no padding token is defined.r?   )r  rA   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r·  )ÚlogitsrÇ  Úpooled_logitsrb   )ÚlossrÊ  r�   rK   r  r\   )r~  ÚscorerF   rb   r›  r¹  rB   r  r3   Úint32rÌ   ÚargmaxrØ   rÙ   r;   rT   Úloss_functionr   r�   rK   r  )r8   r´  r€   rw  r�   rµ  rÇ  rv  rÈ  rƒ   Útransformer_outputsrK   rÊ  r  Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesrË  rÌ  s                      r<   rS   z'Zamba2ForSequenceClassification.forward  sÖ  € ð( 8B°t·z±zØð8
à)Ø%Ø+Ø'Øñ8
ð ñ8
Ðð ,¨AÑ.ˆØ—‘˜MÓ*ˆàÐ Ø"Ÿ™¨Ñ+‰Jà&×,Ñ,¨QÑ/ˆJà�;‰;×#Ñ#Ð+°
¸a²ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ð+Ø!#ÑØÐ"Ø%¨¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸL™L¨¯©¸Ñ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Ñ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|™|¨J¸v¿}¹}ÔMÐOaÐaÑbˆàˆØÐØ%�4×%Ñ%ð Ø$¨VÀ=ÐY]×YdÑYdñØhnñˆDô 0ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r=   )NNNNNNNr   )rT   rU   rV   r+   r1   r   r   r3   rg  r¤   r	   rh  r{  r£   r   r   r¥   r   rS   rW   rX   s   @r<   rÅ  rÅ  û  s  ø„ ð˜|õ ð
 Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ@
à×#Ñ# dÑ*ð@
ð Ÿ™ tÑ+ð@
ð ×&Ñ&¨Ñ-ð	@
ð
  ™ð@
ð ×(Ñ(¨4Ñ/ð@
ð × Ñ  4Ñ'ð@
ð ˜$‘;ð@
ð ˜eŸl™lÑ*ð@
ð Ð+Ñ,ð@
ð 
Ð1Ñ	1ò@
ó ó ô@
r=   rÅ  )r¿  rÅ  r™  r}  )Or…  Úcollections.abcr   Ú	itertoolsr   r3   r   Ú r   rˆ  Úactivationsr   Úcache_utilsr	   r
   Úintegrations.accelerater   Úintegrations.hub_kernelsr   Úmasking_utilsr   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.import_utilsr   Úutils.output_capturingr   Úllama.modeling_llamar   r   Úmamba2.modeling_mamba2r   r   r    Úzamba.modeling_zambar!   r"   r#   r$   r%   r&   r'   r(   r)   Úconfiguration_zamba2r+   Ú_CONFIG_FOR_DOCÚ
get_loggerrT   rØ   ÚModuler-   rZ   r_   ra   r§   rN  r\  rj  rp  r}  r™  r¿  rÅ  Ú__all__r\   r=   r<   ú<module>rì     sU  ðó Ý $Ý ã Ý å &Ý !ß .Ý =Ý 8Ý /ß Yß FÝ &ß lÕ lÝ 7Ý 9Ý 5ß Mß YÑ Y÷
÷ 
õ 
õ /ð '€à	ˆ×	Ñ	˜HÓ	%€ô;˜Ÿ™Ÿ™ô ;ô*	�Lô 	ô	Ð0ô 	ôj)�nô j)ôZbO�r—y‘yô bOôJ'�—	‘	ô 'ôT1Ð"<ô 1ôhZÐ4ô Zô<Ð(ô <ð~ ô!˜Oó !ó ð!ôBD
�*Ð3ô D
ôNÐ(ô ôH
Ð&Dô H
òV�r=   