Ë
    ÛÓ^j=  ã                   óv   — d Z ddlmZ ddlmZ ddlmZ ddlmZ  ed¬«      e G d	„ d
e«      «       «       Z	d
gZ
y)zFalcon configurationé    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringztiiuae/falcon-7b)Ú
checkpointc                   ó  ‡ — e Zd ZU dZdZdgZdZeed<   dZ	eed<   dZ
eed	<   d
Zeed<   dZedz  ed<   dZedz  ed<   dZeed<   dZeed<   dZeez  dz  ed<   dZeez  dz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZeed<   dZeez  dz  ed <   d!Zedz  ed"<   d!Zee e   z  dz  ed#<   dZ!edz  ed$<   dZ"edz  ed%<   d&Z#e$dz  ed'<   dZ%eed(<   ˆ fd)„Z&e'd*„ «       Z(e'd+„ «       Z)ˆ xZ*S ),ÚFalconConfiga  
    num_ln_in_parallel_attn (`int`, *optional*):
        Set to 2 if separate layer norms are to be used for the MLP and the attention output when using parallel
        attention, otherwise, 1.
    alibi (`bool`, *optional*, defaults to `False`):
        Whether to use ALiBi positional biases during self-attention.
    new_decoder_architecture (`bool`, *optional*, defaults to `False`):
        Whether to use the new (Falcon-40B) decoder architecture. If `True`, the `multi_query` and `parallel_attn`
        arguments are ignored, as the new decoder always uses parallel attention.
    multi_query (`bool`, *optional*, defaults to `True`):
        Whether to use multi-query attention in the decoder. Ignored when `new_decoder_architecture` is `True`.
    parallel_attn (`bool`, *optional*, defaults to `True`):
        Whether to compute attention in parallel with the feedforward layer. If False, they are consecutive
        instead, as in the original Transformer architecture. Ignored when `new_decoder_architecture` is `True`.
    bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias on Linear layers.
    ffn_hidden_size (`int`, *optional*):
        The hidden size of the feedforward layer in the Transformer decoder.
        defaults to 4x hidden dim
    activation (`str`, *optional*, defaults to `"gelu"`):
        The activation function used in the feedforward layer.

    Example:

    ```python
    >>> from transformers import FalconModel, FalconConfig

    >>> # Initializing a small (2-layer) Falcon configuration
    >>> configuration = FalconConfig(num_hidden_layers=2)

    >>> # Initializing a model from the small configuration
    >>> model = FalconModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚfalconÚpast_key_valuesi þ  Ú
vocab_sizeiÀ  Úhidden_sizeé    Únum_hidden_layerséG   Únum_attention_headsNÚnum_ln_in_parallel_attngñhãˆµøä>Úlayer_norm_epsilong{®Gáz”?Úinitializer_rangeTÚ	use_cacheg        Úhidden_dropoutÚattention_dropoutÚnum_kv_headsFÚalibiÚnew_decoder_architectureÚmulti_queryÚparallel_attnÚbiasi   Úmax_position_embeddingsÚrope_parametersé   Úbos_token_idÚeos_token_idÚpad_token_idÚffn_hidden_sizeÚgeluÚ
activationÚtie_word_embeddingsc                 ó  •— |j                  dd «      }|€| j                  n|| _        | j                  €| j                  n| j                  | _        | j                  €| j                  dz  | _        t        ‰| �  di |¤Ž y )NÚn_embedé   © )Úpopr   r   r   r%   ÚsuperÚ__post_init__)ÚselfÚkwargsr*   Ú	__class__s      €úz/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/falcon/configuration_falcon.pyr/   zFalconConfig.__post_init__[   sx   ø€ à—*‘*˜Y¨Ó-ˆØ/6¨˜4×+Ò+ÀGˆÔØ8<×8IÑ8IÐ8Q˜D×4Ò4ÐW[×WhÑWhˆÔØ×ÑÐ'Ø#'×#3Ñ#3°aÑ#7ˆDÔ ä‰ÑÑ' Ó'ó    c                 ó4   — | j                   | j                  z  S ©N)r   r   ©r0   s    r3   Úhead_dimzFalconConfig.head_dime   s   € à×Ñ 4×#;Ñ#;Ñ;Ð;r4   c                 ó   — | j                    S r6   )r   r7   s    r3   ÚrotaryzFalconConfig.rotaryi   s   € à—:‘:ˆ~Ðr4   )+Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferencer   ÚintÚ__annotations__r   r   r   r   r   Úfloatr   r   Úboolr   r   r   r   r   r   r   r   r   r    r   Údictr"   r#   Úlistr$   r%   r'   Ústrr(   r/   Úpropertyr8   r:   Ú__classcell__)r2   s   @r3   r
   r
      s˜  ø… ñ#ðJ €JØ#4Ð"5Ðà€J�ÓØ€K�ÓØÐ�sÓØ!Ð˜Ó!Ø*.Ð˜S 4™ZÓ.Ø'+Ð˜ ™Ó+Ø#Ð�uÓ#Ø€IˆtÓØ),€N�E˜C‘K $Ñ&Ó,Ø,/Ð�u˜s‘{ TÑ)Ó/Ø#€L�#˜‘*Ó#Ø€Eˆ4�$‰;ÓØ,1Ð˜d T™kÓ1Ø#€K�˜‘Ó#Ø!%€M�4˜$‘;Ó%Ø€Dˆ$�‰+ÓØ#'Ð˜SÓ'Ø48€O�^ dÑ*¨TÑ1Ó8Ø!€L�#˜‘*Ó!Ø+-€L�#˜˜S™	‘/ DÑ(Ó-Ø#€L�#˜‘*Ó#Ø"&€O�S˜4‘ZÓ&Ø#€J��d‘
Ó#Ø $Ð˜Ó$ô(ð ñ<ó ð<ð ñó ôr4   r
   N)r>   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r
   Ú__all__r,   r4   r3   ú<module>rO      sN   ðñ å .å 3Ý 1Ý #ñ Ð-Ô.ØôRÐ#ó Ró ó /ðRðj Ð
�r4   