Ë
    ÛÓ^jÙ  ã            
       óÚ  — d Z ddlZddlmZ ddlmZ ddlZddlmZ ddlm	Z	m
Z
mZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1  e«       rddlm2Z2  e-jf                  e4«      Z5 G d„ dejl                  «      Z7d„ Z8dAd„Z9 G d„ dejt                  «      Z;dejx                  de=d ej|                  d!ejx                  fd"„Z?d#ejx                  d$ejx                  d%e@d&eAd!ejx                  f
d'„ZB G d(„ d)ejt                  «      ZC G d*„ d+eC«      ZD G d,„ d-ejt                  «      ZEeCeCeDd.œZF G d/„ d0e«      ZGe, G d1„ d2e*«      «       ZHe, G d3„ d4eH«      «       ZI e,d5¬6«       G d7„ d8eHe«      «       ZJ e,d9¬6«       G d:„ d;eH«      «       ZKe, G d<„ d=eH«      «       ZLe, G d>„ d?eH«      «       ZMg d@¢ZNy)BzPyTorch Falcon model.é    N)ÚCallable)ÚOptional)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚ	LayerNormÚMSELoss)Ú
functionalé   )Úinitialization)Úget_activation)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)Ú!flash_attn_supports_top_left_maskÚis_flash_attn_available)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚQuestionAnsweringModelOutputÚ SequenceClassifierOutputWithPastÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚPreTrainedModel)Úauto_docstringÚlogging)Úmaybe_autocasté   )ÚFalconConfig)Ú_flash_attention_forwardc                   óD   — e Zd Zdej                  dej                  fd„Zy)ÚFalconLinearÚinputÚreturnc                 ón   — || j                   j                  z  }| j                  €|S || j                  z   S ©N)ÚweightÚTÚbias)Úselfr%   Úhidden_statess      úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/falcon/modeling_falcon.pyÚforwardzFalconLinear.forward=   s3   € Ø §¡§¡Ñ-ˆØ�9‰9ÐØ Ð Ø˜tŸy™yÑ(Ð(ó    N)Ú__name__Ú
__module__Ú__qualname__ÚtorchÚTensorr/   © r0   r.   r$   r$   <   s   „ ð)˜UŸ\™\ð )¨e¯l©lô )r0   r$   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Néÿÿÿÿé   ©Údim)Úshaper4   Úcat)ÚxÚx1Úx2s      r.   Úrotate_halfrA   E   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r0   c                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerA   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r.   Úapply_rotary_pos_embrK   M   sY   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr0   c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚFalconRotaryEmbeddingÚinv_freqNÚconfigc                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultrN   F)Ú
persistentÚoriginal_inv_freq)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrO   Úrope_parametersrQ   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r,   rO   ÚdeviceÚrope_init_fnrN   Ú	__class__s        €r.   rV   zFalconRotaryEmbedding.__init__j   s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr0   r_   ztorch.deviceÚseq_lenr&   ztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNç      ð?r   r9   ©Údtype©r_   rh   )	rZ   ÚgetattrÚhidden_sizeÚnum_attention_headsr4   ÚarangeÚint64ÚtoÚfloat)rO   r_   rb   Úbaser;   Úattention_factorrN   s          r.   r[   z5FalconRotaryEmbedding.compute_default_rope_parametersz   s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r0   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   r8   r    ÚmpsÚcpuF)Údevice_typeÚenabledr9   r:   rg   )rN   rp   Úexpandr<   ro   r_   Ú
isinstanceÚtypeÚstrr   Ú	transposer4   r=   rF   r\   rG   rh   )
r,   r>   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedrv   ÚfreqsÚembrF   rG   s
             r.   r/   zFalconRotaryEmbedding.forward˜   sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$r(   )NNN)r1   r2   r3   r4   r5   Ú__annotations__r!   rV   Ústaticmethodr   ÚintÚtuplerp   r[   Úno_gradr   r/   Ú__classcell__©ra   s   @r.   rM   rM   g   s˜   ø… Ø�l‰lÓñV˜|õ Vð  à&*Ø+/Ø"ñ*Ø˜tÑ#ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r0   rM   Úattention_maskÚ	num_headsrh   r&   c                 óô  — | j                   \  }}dt        j                  t        j                  |«      «      z  }t	        j
                  ddt        j                  |«      dz
   z   z  | j                  t        j                  ¬«      }t	        j                  dd|z   | j                  t        j                  ¬«      }t	        j                  ||«      }||k7  rÅt	        j
                  ddt        j                  d|z  «      dz
   z   z  | j                  t        j                  ¬«      }	t        |||z
  «      }
t	        j                  ddd|
z  z   d| j                  t        j                  ¬«      }t	        j                  |t	        j                  |	|«      gd¬«      }| j                  d¬«      dz
  | z  d d …d d d …f   }|d   j                  «       |z  }|j                  ||z  d|«      j!                  |«      S )	Nr9   r   ri   r    r   r:   r8   ).N)r<   ÚmathÚfloorÚlog2r4   Útensorr_   Úfloat32rm   Úint32ÚpowÚminr=   ÚcumsumÚbfloat16Úreshapero   )r‰   rŠ   rh   Ú
batch_sizeÚ
seq_lengthÚclosest_power_of_2rq   ÚpowersÚslopesÚ
extra_baseÚnum_remaining_headsÚextra_powersÚarange_tensorÚalibis                 r.   Úbuild_alibi_tensorr¡   ¨   sÁ  € Ø+×1Ñ1Ñ€J�
ØœdŸj™j¬¯©°9Ó)=Ó>Ñ>ÐÜ�<‰<Ø	�”t—y‘yÐ!3Ó4°qÑ8Ð9Ñ9Ð:Ñ;ÀN×DYÑDYÔaf×anÑanô€Dô �\‰\˜!˜QÐ!3Ñ3¸N×<QÑ<QÔY^×YdÑYdÔe€FÜ�Y‰Y�t˜VÓ$€Fà˜YÒ&Ü—\‘\Ø�Aœ4Ÿ9™9 QÐ);Ñ%;Ó<¸qÑ@ÐAÑAÐBÑCÈN×LaÑLaÔin×ivÑivô
ˆ
ô "Ð"4°iÐBTÑ6TÓUÐÜ—|‘| A q¨1Ð/BÑ+BÑ'BÀAÈn×NcÑNcÔkp×kvÑkvÔwˆÜ—‘˜F¤E§I¡I¨j¸,Ó$GÐHÈaÔPˆð %×+Ñ+°Ð+Ó3°aÑ7¸>ÑIÊ1ÈdÒTUÈ:ÑV€MØ�9Ñ×&Ñ&Ó(¨=Ñ8€EØ�=‰=˜ iÑ/°°JÓ?×BÑBÀ5ÓIÐIr0   r>   ÚresidualÚprobÚtrainingc                 ó@   — t        j                  | ||¬«      }||z   }|S )a
  
    Dropout add function

    Args:
        x (`torch.tensor`):
            input tensor
        residual (`torch.tensor`):
            residual tensor
        prob (`float`):
            dropout probability
        training (`bool`):
            training mode
    )Úpr¤   )ÚFÚdropout)r>   r¢   r£   r¤   Úouts        r.   Údropout_addrª   Å   s$   € ô �)‰)�A˜¨Ô
1€CØ
�S‰.€CØ€Jr0   c                   ó˜  ‡ — e Zd Zddefˆ fd„Zdej                  deej                  ej                  ej                  f   fd„Zdej                  dej                  fd„Z		 	 	 	 	 dd	ej                  d
ej                  dz  dej                  dej                  dz  dedz  dededeej                  ej                  f   dz  fd„Zˆ xZS )ÚFalconAttentionNrO   c                 ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        | j                  | j
                  z  | _        | j                  | _        |j                  | _        |j                  | _	        d| _
        || _        |€-t        j                  d| j                  j                  › d�«       | j                  | j
                  z  | j                  k7  r&t!        d| j                  › d| j
                  › d�«      ‚dt#        j$                  | j                  «      z  | _        | j&                  | _        |j*                  r*|j,                  dz  |j                  z   | j                  z  }n8|j.                  r| j                  d| j                  z  z   }nd	| j                  z  }t1        | j                  ||j2                  ¬
«      | _        |j*                  | _        |j.                  | _        t1        | j                  | j                  |j2                  ¬
«      | _        t9        j:                  |j<                  «      | _        | j*                  s| j.                  s|j,                  | _        y d| _        y )NTzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.zA`hidden_size` must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).rf   r9   r   ©r+   r    )rU   rV   rO   rk   rl   rŠ   re   Ú
split_sizeÚhidden_dropoutrW   Ú	is_causalÚ	layer_idxÚloggerÚwarning_oncera   r1   Ú
ValueErrorrŒ   ÚsqrtÚinv_norm_factorÚbetaÚnew_decoder_architectureÚnum_kv_headsÚmulti_queryr$   r+   Úquery_key_valueÚdenser   ÚDropoutÚattention_dropout)r,   rO   r²   Úqkv_out_dimra   s       €r.   rV   zFalconAttention.__init__Ù   s   ø€ Ü‰ÑÔàˆŒØ!×-Ñ-ˆÔØ×3Ñ3ˆŒØ×(Ñ(¨D¯N©NÑ:ˆŒØ×*Ñ*ˆŒØ$×3Ñ3ˆÔØ'-×'EÑ'EˆÔ$àˆŒØ"ˆŒØÐÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôð �=‰=˜4Ÿ>™>Ñ)¨T×-=Ñ-=Ò=ÜØSÐTX×TdÑTdÐSeð fØ—N‘NÐ# 2ð'óð ð  #¤T§Y¡Y¨t¯}©}Ó%=Ñ=ˆÔØ×(Ñ(ˆŒ	Ø×*Ò*Ø!×.Ñ.°Ñ2°V×5OÑ5OÑOÐSW×S`ÑS`Ñ`‰KØ×ÒØ×*Ñ*¨Q°·±Ñ->Ñ>‰Kà˜d×.Ñ.Ñ.ˆKÜ+¨D×,<Ñ,<¸kÐPV×P[ÑP[Ô\ˆÔØ(.×(GÑ(GˆÔ%Ø!×-Ñ-ˆÔÜ! $×"2Ñ"2°D×4DÑ4DÈ6Ï;É;ÔWˆŒ
Ü!#§¡¨F×,DÑ,DÓ!EˆÔØ48×4QÒ4QÐY]×YiÒYi˜F×/Ñ/ˆÕÐpqˆÕr0   Ú	fused_qkvr&   c                 ól  — | j                   ræ|j                  \  }}}|j                  ||d| j                  | j                  z  dz   | j
                  «      }|dd…dd…dd…dd…f   }|dd…dd…dd…dgf   }|dd…dd…dd…dgf   }t        j                  ||j                  «      }t        j                  ||j                  «      }|||fD �	cg c]  }	|	j                  dd«      ‘Œ c}	\  }}}|||fS | j                  sV|j                  \  }
}}|j                  |
|| j                  d| j
                  «      }|dddd…f   |dddd…f   |dddd…f   fS |j                  \  }
}}|j                  |
|| j                  dz   | j
                  «      }|ddd…dd…f   |ddgdd…f   |ddgdd…f   fS c c}	w )	a¨  
        Split the last dimension into (num_heads, head_dim), results share same memory storage as `fused_qkv`

        Args:
            fused_qkv (`torch.tensor`): [batch_size, seq_length, num_heads * 3 * head_dim]

        Returns:
            query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim]
            value: [batch_size, seq_length, num_heads, head_dim]
        r8   r9   Néþÿÿÿr   .r   r    )
r¹   r<   ÚviewrŠ   rº   re   r4   Úbroadcast_toÚflattenr»   )r,   rÁ   Úbatchrb   Ú_ÚqkvÚqueryÚkeyÚvaluer>   r—   r˜   Úthree_times_hidden_sizes                r.   Ú_split_headszFalconAttention._split_heads  sÍ  € ð ×(Ò(Ø )§¡ÑˆE�7˜AØ—.‘. ¨°°T·^±^Àt×GXÑGXÑ5XÐ[\Ñ5\Ð^b×^kÑ^kÓlˆCØšš1ša  " ˜Ñ%ˆEØ’aššA ˜t�mÑ$ˆCØšš1ša " ˜Ñ&ˆEÜ×$Ñ$ S¨%¯+©+Ó6ˆCÜ×&Ñ& u¨e¯k©kÓ:ˆEà;@À#ÀuÐ:MÖ N°Q §¡¨1¨a¥Ò NÑˆE�3˜Ø˜#˜uÐ$Ð$Ø×!Ò!Ø>G¿o¹oÑ;ˆJ˜
Ð$;Ø!Ÿ™ z°:¸t¿~¹~ÈqÐRV×R_ÑR_Ó`ˆIØ˜S !¢Q˜YÑ'¨°3¸º1°9Ñ)=¸yÈÈaÒQRÈÑ?SÐSÐSà>G¿o¹oÑ;ˆJ˜
Ð$;Ø!Ÿ™ z°:¸t¿~¹~ÐPQÑ?QÐSW×S`ÑS`ÓaˆIØ˜S # 2 #¢q˜[Ñ)¨9°S¸2¸$Â°\Ñ+BÀIÈcÐTVÐSWÒYZÈlÑD[Ð[Ð[ùò !Os   ÃF1r>   c                 ó  — |j                   \  }}}|| j                  z  }|j                  || j                  || j                  «      }|j	                  dddd«      }|j                  ||| j                  | j                  z  «      S )z÷
        Merge heads together over the last dimension

        Args:
            x (`torch.tensor`): [batch_size * num_heads, seq_length, head_dim]

        Returns:
            torch.tensor: [batch_size, seq_length, num_heads * head_dim]
        r   r9   r    r   )r<   rŠ   rÄ   re   Úpermuter–   )r,   r>   Úbatch_size_and_num_headsr˜   rÈ   r—   s         r.   Ú_merge_headszFalconAttention._merge_heads#  sy   € ð 34·'±'Ñ/Ð  *¨aØ-°·±Ñ?ˆ
ð �F‰F�:˜tŸ~™~¨z¸4¿=¹=ÓIˆð �I‰I�a˜˜A˜qÓ!ˆð �y‰y˜ Z°·±À$Ç-Á-Ñ1OÓPÐPr0   r-   r    r‰   r}   Ú
layer_pastÚ	use_cacheÚoutput_attentionsÚposition_embeddingsc	                 ó2	  — | j                  |«      }
| j                  r| j                  n| j                  }| j	                  |
«      \  }}}|j
                  \  }}}}|j                  dd«      j                  || j                  || j                  «      }|j                  dd«      j                  |||| j                  «      }|j                  dd«      j                  |||| j                  «      }|€|\  }}t        ||||«      \  }}|� |j                  ||| j                  «      \  }}|j
                  d   }|�€B| j                  j                  dk(  rM|sK| j                  xr |d u xr |dkD  }t        j                   j"                  j%                  ||||d|¬«      }d }na||j                  dd«      z  }|t'        j(                  | j                  «      z  }t+        j,                  ||z   d|j.                  ¬«      }||z  }|j1                  || j                  || j                  «      }|j3                  d	ddd
«      }|j                  ||| j                  | j                  z  «      }| j5                  |«      }||fS | j                  j                  dk(  r¿|s½| j                  xr |d u xr |dkD  }t        j                   j"                  j%                  ||||| j6                  r| j8                  j:                  nd|¬«      }d }|j                  dd«      }|j                  ||| j                  | j                  z  «      }| j5                  |«      }||fS ||j                  dd«      z  }|j1                  || j                  ||«      }|j.                  }|t        j<                  k(  s|t        j>                  k(  r|jA                  t        jB                  «      }||j1                  || j                  dd«      z   }|| jD                  z  }t+        j,                  ||z   d|j.                  ¬«      }| j9                  |«      }|j1                  || j                  ||«      }||z  jG                  d	d«      }| jI                  |«      }| j5                  |«      }||fS )Nr    r9   rÃ   Úsdpaç        )Ú	attn_maskÚ	dropout_pr±   r8   )r;   rh   r   r   )%r¼   r¹   rŠ   rº   rÎ   r<   r|   r–   re   rK   Úupdater²   rO   Ú_attn_implementationr±   r4   r   r
   Úscaled_dot_product_attentionrŒ   r¶   r§   Úsoftmaxrh   rÄ   rÐ   r½   r¤   r¿   r¦   Úfloat16r•   ro   r�   r·   rÆ   rÒ   )r,   r-   r    r‰   r}   rÓ   rÔ   rÕ   rÖ   ÚkwargsrÁ   rº   Úquery_layerÚ	key_layerÚvalue_layerr—   Úquery_lengthrÈ   rF   rG   Ú	kv_lengthr±   Úattn_outputÚattention_scoresÚattention_probsÚmatmul_resultÚinput_dtypeÚattention_logitsÚattention_probs_reshapeds                                r.   r/   zFalconAttention.forward<  s‚  € ð ×(Ñ(¨Ó7ˆ	Ø)-×)FÒ)F�t—~’~ÈD×L]ÑL]ˆà04×0AÑ0AÀ)Ó0LÑ-ˆ�i à)4×):Ñ):Ñ&ˆ
�L ! Qà!×+Ñ+¨A¨qÓ1×9Ñ9¸*ÀdÇnÁnÐVbÐdh×dqÑdqÓrˆØ×'Ñ'¨¨1Ó-×5Ñ5°jÀ,ÐP\Ð^b×^kÑ^kÓlˆ	Ø!×+Ñ+¨A¨qÓ1×9Ñ9¸*ÀlÐT`Ðbf×boÑboÓpˆàˆ=Ø*‰HˆC�Ü%9¸+ÀyÐRUÐWZÓ%[Ñ"ˆK˜àÐ!Ø%/×%6Ñ%6°yÀ+ÈtÏ~É~Ó%^Ñ"ˆI�{à—O‘O BÑ'ˆ	à‰=Ø�{‰{×/Ñ/°6Ò9ÑBSð
 !ŸN™NÒZ¨~ÀÐ/EÒZÈ,ÐYZÑJZ�	Ü#Ÿh™h×1Ñ1×NÑNØØØØ,Ø!Ø'ð Oó �ð $(Ñ à#.°×1DÑ1DÀRÈÓ1LÑ#LÐ Ø ¤D§I¡I¨d¯m©mÓ$<Ñ<Ð ä#$§9¡9Ð-=ÀÑ-NÐTVÐ^k×^qÑ^qÔ#rÐ à.°Ñ<�à%×*Ñ*¨:°t·~±~À|ÐUY×UbÑUbÓcˆKØ%×-Ñ-¨a°°A°qÓ9ˆKØ%×-Ñ-¨j¸,ÈÏÉÐY]×YfÑYfÑHfÓgˆKàŸ*™* [Ó1ˆKàÐ 0Ð0Ð0ð �{‰{×/Ñ/°6Ò9ÑBSð !ŸN™NÒZ¨~ÀÐ/EÒZÈ,ÐYZÑJZ�	Ü#Ÿh™h×1Ñ1×NÑNØØØØ,Ø:>¿-º-˜d×4Ñ4×6Ò6ÈSØ'ð Oó �ð #'�Ø)×3Ñ3°A°qÓ9�Ø)×1Ñ1°*¸lÈDÏNÉNÐ]a×]jÑ]jÑLjÓk�à"Ÿj™j¨Ó5�ð<  Ð/Ð/ð9 !,¨i×.AÑ.AÀ"ÀbÓ.IÑ I�ð $1×#5Ñ#5°jÀ$Ç.Á.ÐR^Ð`iÓ#jÐ ð /×4Ñ4�à¤%§-¡-Ò/°;Ä%Ç.Á.Ò3PØ'7×':Ñ':¼5¿=¹=Ó'IÐ$à#3°e·j±jÀÈTÏ^É^Ð]^Ð`bÓ6cÑ#cÐ Ø  D×$8Ñ$8Ñ8Ð Ü"#§)¡)Ð,<¸~Ñ,MÐSUÐ]j×]pÑ]pÔ"q�à"&×"8Ñ"8¸Ó"I�ð ,;×+?Ñ+?À
ÈDÏNÉNÐ\hÐjsÓ+tÐ(ð  8¸+ÑE×NÑNÈqÐRSÓT�ð #×/Ñ/°Ó<�à"Ÿj™j¨Ó5�à Ð/Ð/r0   r(   ©NNFFN)r1   r2   r3   r!   rV   r4   r5   r…   rÎ   rÒ   Ú
LongTensorr   Úboolr/   r‡   rˆ   s   @r.   r¬   r¬   Ø   s
  ø„ ñ(r˜|õ (rðT\ e§l¡lð \°u¸U¿\¹\È5Ï<É<ÐY^×YeÑYeÐ=eÑ7fó \ð@Q˜eŸl™lð Q¨u¯|©|ó Qð< 15Ø#'ØØ"'ØHLño0à—|‘|ðo0ð �|‰|˜dÑ"ðo0ð Ÿ™ð	o0ð
 ×&Ñ&¨Ñ-ðo0ð ˜D‘Lðo0ð ðo0ð  ðo0ð # 5§<¡<°·±Ð#=Ñ>ÀÑE÷o0r0   r¬   c                   óð   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dej                  dz  dedz  d	e	d
e	de
ej                  ej                  f   dz  fd„Zˆ xZS )ÚFalconFlashAttention2aH  
    Falcon flash attention module. This module inherits from `FalconAttention` as the weights of the module stays
    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
    flash attention and deal with padding tokens in case the input contains any of them.
    c                 óB   •— t        ‰| �  |i |¤Ž t        «       | _        y r(   )rU   rV   r   Ú_flash_attn_uses_top_left_mask)r,   Úargsrá   ra   s      €r.   rV   zFalconFlashAttention2.__init__µ  s#   ø€ Ü‰Ñ˜$Ð) &Ò)ô
 /PÓ.QˆÕ+r0   Nr-   r    r‰   r}   rÓ   rÔ   rÕ   rÖ   c	                 ó   — | j                  |«      }
| j                  r| j                  n| j                  }| j	                  |
«      \  }}}|j
                  \  }}}}|j                  dd«      j                  || j                  || j                  «      }|j                  dd«      j                  |||| j                  «      }|j                  dd«      j                  |||| j                  «      }|€|\  }}t        ||||«      \  }}|� |j                  ||| j                  «      \  }}|j                  dd«      }|j                  dd«      }|j                  dd«      }|�t        d«      ‚| j                  r| j                  j                  nd}|j                   }|j"                  j$                  dk7  r|j"                  j$                  nd}|t&        j(                  k(  rÄt'        j*                  |«      rt'        j,                  |«      }nMt/        | j                  d«      r| j                  j                   }n | j                   j0                  j                   }t2        j5                  d|› d	�«       |j7                  |«      }|j7                  |«      }|j7                  |«      }t9        |||||||| j:                  | j<                  ¬
«	      }|j                  ||| j                  | j                  z  «      }| j?                  |«      }|sd }||fS )Nr    r9   z6`alibi` is not supported when `use_flash_attn` is TruerÙ   rt   ru   Ú_is_quantizedz¾The input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in ú.)r}   r¨   r±   Úuse_top_left_mask) r¼   r¹   rŠ   rº   rÎ   r<   r|   r–   re   rK   rÜ   r²   rµ   r¤   rO   r¿   rh   r_   rz   r4   r�   Úis_autocast_enabledÚget_autocast_dtypeÚhasattrr)   r³   r´   ro   r"   r±   rô   r½   )r,   r-   r    r‰   r}   rÓ   rÔ   rÕ   rÖ   rá   rÁ   rº   râ   rã   rä   r—   rå   rÈ   rF   rG   Úattn_dropoutrë   rv   Útarget_dtyperç   Úattn_weightss                             r.   r/   zFalconFlashAttention2.forward½  sÙ  € ð ×(Ñ(¨Ó7ˆ	Ø)-×)FÒ)F�t—~’~ÈD×L]ÑL]ˆà04×0AÑ0AÀ)Ó0LÑ-ˆ�i à)4×):Ñ):Ñ&ˆ
�L ! Qà!×+Ñ+¨A¨qÓ1×9Ñ9¸*ÀdÇnÁnÐVbÐdh×dqÑdqÓrˆØ×'Ñ'¨¨1Ó-×5Ñ5°jÀ,ÐP\Ð^b×^kÑ^kÓlˆ	Ø!×+Ñ+¨A¨qÓ1×9Ñ9¸*ÀlÐT`Ðbf×boÑboÓpˆàˆ=Ø*‰HˆC�Ü%9¸+ÀyÐRUÐWZÓ%[Ñ"ˆK˜àÐ!Ø%/×%6Ñ%6°yÀ+ÈtÏ~É~Ó%^Ñ"ˆI�{ð "×+Ñ+¨A¨qÓ1ˆØ×'Ñ'¨¨1Ó-ˆ	Ø!×+Ñ+¨A¨qÓ1ˆàÐÜÐUÓVÐVà8<¿º�t—{‘{×4Ò4È3ˆð
 "×'Ñ'ˆØ1<×1CÑ1C×1HÑ1HÈEÒ1Q�k×(Ñ(×-Ò-ÐW\ˆØœ%Ÿ-™-Ò'Ü×(Ñ(¨Ô5Ü$×7Ñ7¸ÓD‘ä˜Ÿ™ oÔ6Ø#Ÿ{™{×0Ñ0‘à#×3Ñ3×:Ñ:×@Ñ@�ä×Ñðà �> ð$ôð &Ÿ.™.¨Ó6ˆKØ!Ÿ™ \Ó2ˆIØ%Ÿ.™.¨Ó6ˆKä.ØØØØØØ%Ø Ø—n‘nØ"×AÑAô

ˆð #×*Ñ*¨:°|ÀTÇ^Á^ÐVZ×VcÑVcÑEcÓdˆØ—j‘j Ó.ˆá ØˆLà˜LÐ(Ð(r0   rî   )r1   r2   r3   Ú__doc__rV   r4   r5   rï   r   rð   r…   r/   r‡   rˆ   s   @r.   rò   rò   ®  s²   ø„ ñôRð 15Ø#'ØØ"'ØHLñS)à—|‘|ðS)ð �|‰|˜dÑ"ðS)ð Ÿ™ð	S)ð
 ×&Ñ&¨Ñ-ðS)ð ˜D‘LðS)ð ðS)ð  ðS)ð # 5§<¡<°·±Ð#=Ñ>ÀÑE÷S)r0   rò   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )Ú	FalconMLPrO   c                 ó,  •— t         ‰| �  «        |j                  }t        ||j                  |j
                  ¬«      | _        t        |j                  «      | _	        t        |j                  ||j
                  ¬«      | _
        |j                  | _        y )Nr®   )rU   rV   rk   r$   Úffn_hidden_sizer+   Údense_h_to_4hr   Ú
activationÚactÚdense_4h_to_hr°   )r,   rO   rk   ra   s      €r.   rV   zFalconMLP.__init__  su   ø€ Ü‰ÑÔØ×(Ñ(ˆä)¨+°v×7MÑ7MÐTZ×T_ÑT_Ô`ˆÔÜ! &×"3Ñ"3Ó4ˆŒÜ)¨&×*@Ñ*@À+ÐTZ×T_ÑT_Ô`ˆÔØ$×3Ñ3ˆÕr0   r>   r&   c                 óh   — | j                  | j                  |«      «      }| j                  |«      }|S r(   )r  r  r  )r,   r>   s     r.   r/   zFalconMLP.forward  s0   € Ø�H‰H�T×'Ñ'¨Ó*Ó+ˆØ×Ñ˜qÓ!ˆØˆr0   )	r1   r2   r3   r!   rV   r4   r5   r/   r‡   rˆ   s   @r.   r  r    s*   ø„ ð4˜|õ 4ð˜Ÿ™ð ¨%¯,©,÷ r0   r  )ÚeagerrØ   Úflash_attention_2c                   ó,  ‡ — e Zd Zddefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dej                  dz  dee	ej                  ej                  f   z  dz  d	e
d
e
de	ej                  ej                  f   dz  fd„Zˆ xZS )ÚFalconDecoderLayerNrO   c                 óx  •— t         ‰| �  «        |j                  }|j                  | _        t        |j                     ||«      | _        t        |«      | _	        |j                  | _
        || _        |j                  €|j                  rd|_        |j                  s9t        ||j                   ¬«      | _        t        ||j                   ¬«      | _        y |j                  dk(  r9t        ||j                   ¬«      | _        t        ||j                   ¬«      | _        y t        ||j                   ¬«      | _        y )Nr9   ©Úeps)rU   rV   rk   rl   rŠ   ÚFALCON_ATTENTION_CLASSESrÝ   Úself_attentionr  Úmlpr°   rO   Únum_ln_in_parallel_attnr¹   Úparallel_attnr   Úlayer_norm_epsilonÚpost_attention_layernormÚinput_layernormÚln_attnÚln_mlp)r,   rO   r²   rk   ra   s       €r.   rV   zFalconDecoderLayer.__init__+  sý   ø€ Ü‰ÑÔØ×(Ñ(ˆØ×3Ñ3ˆŒä6°v×7RÑ7RÑSÐTZÐ\eÓfˆÔÜ˜VÓ$ˆŒØ$×3Ñ3ˆÔØˆŒà×)Ñ)Ð1°f×6UÒ6UØ-.ˆFÔ*à×#Ò#Ü,5°kÀv×G`ÑG`Ô,aˆDÔ)Ü#,¨[¸f×>WÑ>WÔ#XˆDÕ à×-Ñ-°Ò2ä(¨¸&×:SÑ:SÔT�”ä'¨¸×9RÑ9RÔS�•ä'0°À&×B[ÑB[Ô'\�Õ$r0   r-   r    r‰   r}   rÓ   rÔ   rÕ   rÖ   c	           
      óR  — |}
| j                   j                  r<| j                   j                  dk(  r#| j                  |«      }| j	                  |«      }n| j                  |«      }| j                  ||||||||¬«      \  }}| j                   j                  sW| j                   j                  r|}n>t        ||
| j                   j                  | j                  ¬«      }
| j                  |
«      }| j                   j                  r1| j                   j                  r| j                   j                  dk(  r|}| j                  «      }| j                   j                  s| j                   j                  r||z  }t        ||
| j                   j                  | j                  ¬«      }||fS )Nr9   )rÓ   r‰   r}   r    rÔ   rÕ   rÖ   )r¤   r    )rO   r¹   r  r  r  r  r  r  rª   r¿   r¤   r  r  r°   )r,   r-   r    r‰   r}   rÓ   rÔ   rÕ   rÖ   rá   r¢   Úattention_layernorm_outÚmlp_layernorm_outÚattention_outputrÿ   Ú
mlp_outputÚoutputs                    r.   r/   zFalconDecoderLayer.forwardD  sz  € ð !ˆà�;‰;×/Ò/°D·K±K×4WÑ4WÐ[\Ò4\Ø&*§l¡l°=Ó&AÐ#Ø $§¡¨MÓ :Ñà&*×&:Ñ&:¸=Ó&IÐ#ð *.×)<Ñ)<Ø#Ø!Ø)Ø%ØØØ/Ø 3ð *=ó 	*
Ñ&Ð˜,ð �{‰{×3Ò3Ø�{‰{×(Ò(Ø$;Ñ!ä&Ø$ h°·±×0MÑ0MÐX\×XeÑXeô�ð %)×$AÑ$AÀ(Ó$KÐ!ð �K‰K×0Ò0Ø—‘×)Ò)Ø—‘×3Ñ3°qÒ8à 7Ðð —X‘XÐ/Ó0ˆ
à�;‰;×/Ò/°4·;±;×3LÒ3LØÐ*Ñ*ˆJä˜Z¨°4·;±;×3MÑ3MÐX\×XeÑXeÔfˆà�|Ð#Ð#r0   r(   rî   )r1   r2   r3   r!   rV   r4   r5   rï   r   r…   rð   r/   r‡   rˆ   s   @r.   r  r  *  sÄ   ø„ ñ]˜|õ ]ð< 15ØGKØØ"'ØHLñ8$à—|‘|ð8$ð �|‰|˜dÑ"ð8$ð Ÿ™ð	8$ð
 ×&Ñ&¨Ñ-ð8$ð ˜E %§,¡,°·±Ð"<Ñ=Ñ=ÀÑDð8$ð ð8$ð  ð8$ð # 5§<¡<°·±Ð#=Ñ>ÀÑE÷8$r0   r  c                   óž   ‡ — e Zd ZU eed<   dZdZdgZdZdZ	dZ
 ej                  «       dej                  fˆ fd„«       Zed	defd„«       Zˆ xZS )
ÚFalconPreTrainedModelrO   ÚtransformerTr  Úmodulec                 ó
  •— t         ‰| �  |«       t        |t        «      rct	        j
                  |j                  d| j                  j                  ¬«       |j                  � t	        j                  |j                  «       yyy)zInitialize the weights.rÙ   )ÚmeanÚstdN)rU   Ú_init_weightsry   r$   ÚinitÚnormal_r)   rO   Úinitializer_ranger+   Úzeros_)r,   r$  ra   s     €r.   r(  z#FalconPreTrainedModel._init_weights‰  s^   ø€ ô 	‰Ñ˜fÔ%Ü�fœlÔ+Ü�L‰L˜Ÿ™¨S°d·k±k×6SÑ6SÕTØ�{‰{Ð&Ü—‘˜FŸK™KÕ(ð 'ð ,r0   Úhard_check_onlyc                 ó:   — t        | dd«      }|r|S |sd|_        |S )NÚuse_bettertransformerFrØ   )rj   rÝ   )ÚclsrO   r-  Ú_is_bettertransformers       r.   Ú_check_and_enable_sdpaz,FalconPreTrainedModel._check_and_enable_sdpa“  s*   € ä '¨Ð-DÀeÓ LÐÙ ØˆMáØ*0ˆFÔ'Øˆr0   )F)r1   r2   r3   r!   r‚   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphr4   r†   r   ÚModuler(  Úclassmethodrð   r2  r‡   rˆ   s   @r.   r"  r"    so   ø… àÓØ%ÐØ&*Ð#Ø-Ð.ÐØÐØ€NØ!Ðà€U‡]�]ƒ_ð) B§I¡Iô )ó ð)ð ñ¸Tò ó ôr0   r"  c                   óB  ‡ — e Zd Zdefˆ fd„Zd„ Zdej                  fd„Ze		 	 	 	 	 	 	 	 	 ddej                  dz  dedz  d	ej                  dz  d
ej                  dz  dej                  dz  dedz  dedz  dedz  dedz  deej                  df   ez  fd„«       Zˆ xZS )ÚFalconModelrO   c           	      ó  •— t         ‰| �  |«       |j                  | _        |j                  | _        |j                  | _        t        j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}«      | _        t#        | j                  |j$                  ¬«      | _        d| _        t+        |¬«      | _        | j/                  «        y c c}w )N)r²   r  F©rO   )rU   rV   rk   Ú	embed_dimrl   rŠ   r    Ú	use_alibir   Ú	EmbeddingÚ
vocab_sizeÚword_embeddingsÚ
ModuleListÚrangeÚnum_hidden_layersr  Úhr   r  Úln_fÚgradient_checkpointingrM   Ú
rotary_embÚ	post_init)r,   rO   Úira   s      €r.   rV   zFalconModel.__init__   sÆ   ø€ Ü‰Ñ˜Ô à×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™ˆŒô  "Ÿ|™|¨F×,=Ñ,=¸t¿~¹~ÓNˆÔô —‘ÔQVÐW]×WoÑWoÓQpÖqÈAÔ 2°6ÀQÖ GÒqÓrˆŒô ˜dŸn™n°&×2KÑ2KÔLˆŒ	Ø&+ˆÔ#Ü/°vÔ>ˆŒð 	�‰Õùò  rs   ÂDc                 ó   — | j                   S r(   ©rC  )r,   s    r.   Úget_input_embeddingsz FalconModel.get_input_embeddingsµ  s   € Ø×#Ñ#Ð#r0   Únew_embeddingsc                 ó   — || _         y r(   rN  ©r,   rP  s     r.   Úset_input_embeddingsz FalconModel.set_input_embeddings¸  s
   € Ø-ˆÕr0   NÚ	input_idsÚpast_key_valuesr‰   r}   Úinputs_embedsrÔ   rÕ   Úoutput_hidden_statesÚreturn_dictr&   .c
                 ó   — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|	�|	n| j                   j                  }	|du |duz  rt        d«      ‚| j                  r%| j                  r|rt        j                  d«       d}|€| j                  |«      }|r|€t        | j                   ¬«      }d}|�|j                  «       nd}|j                  \  }}}| j                  r[|€5t        j                   |||z   f|j"                  t        j$                  ¬«      n|}t'        || j(                  |j*                  ¬«      }|€V|�|j                  «       nd}t        j,                  |j                  d	   |j"                  ¬
«      |z   }|j/                  d«      }t1        | j                   |||d„ ¬«      }|��|��|j2                  dk(  rùt        j4                  |j*                  «      j6                  }|j*                  t        j8                  k(  rAt        j:                  |t        j<                  d|j"                  |j*                  ¬«      |«      } |j>                  |dg|j                  d	d ¢­Ž }t        j@                  |tC        jD                  | j                   jF                  | j(                  z  «      z  |dk  |«      }|}| jI                  ||¬«      }|rdnd}|rdnd}tK        | jL                  «      D ].  \  }}|r||fz   } |||||||||¬«      }|d   }|sŒ&||d	   fz   }Œ0 | jO                  |«      }|r||fz   }|	stQ        d„ ||||fD «       «      S tS        ||||¬«      S )á²  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
            (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr>  r   ri   rg   r    ©r_   c                  óL   — t        j                  dt         j                  ¬«      S )NTrg   )r4   r�   rð   )rõ   s    r.   ú<lambda>z%FalconModel.forward.<locals>.<lambda>  s   € ¬E¯L©L¸ÄUÇZÁZÔ,P€ r0   )rO   rV  r‰   rU  Úand_mask_functioné   rÙ   r8   )r}   r6   )rÓ   r‰   r}   rÔ   rÕ   r    rÖ   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr(   r6   )Ú.0Úvs     r.   ú	<genexpr>z&FalconModel.forward.<locals>.<genexpr>9  s   è ø€ ò ØÐghÑgt”ñùs   ‚Š)Úlast_hidden_staterU  r-   Ú
attentions)*rO   rÕ   rW  rÔ   rX  rµ   rI  r¤   r³   r´   rC  r   Úget_seq_lengthr<   r@  r4   Úonesr_   Úlongr¡   rŠ   rh   rm   rC   r   ÚndimÚfinfor“   rð   Úwherer�   r–   Úmasked_fillrŒ   r¶   rk   rJ  Ú	enumeraterG  rH  r…   r   )r,   rT  rU  r‰   r}   rV  rÔ   rÕ   rW  rX  rá   r    Úpast_key_values_lengthr—   r˜   rÈ   ÚmaskÚpast_seen_tokensÚcausal_maskÚ	min_dtyper-   rÖ   Úall_self_attentionsÚall_hidden_statesrL  ÚblockÚoutputss                              r.   r/   zFalconModel.forward»  sº  € ð6 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×BYÑBYˆà˜Ð -°tÐ";Ò<ÜÐYÓZÐZà×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øpôð "�	àÐ Ø ×0Ñ0°Ó;ˆMá˜Ð0Ü*°$·+±+Ô>ˆOð ˆØETÐE` ×!?Ñ!?Ô!AÐfgÐØ$1×$7Ñ$7Ñ!ˆ
�J Ø�>Š>ð
 "Ð)ô —
‘
Ø Ð.DÑ!DÐEÈm×NbÑNbÔjo×jtÑjtõð $ð ô ' t¨T¯^©^À=×CVÑCVÔWˆEàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+áPô
ˆð Ñ Ñ!8¸[×=MÑ=MÐQRÒ=RÜŸ™ M×$7Ñ$7Ó8×<Ñ<ˆIð × Ñ ¤E§J¡JÒ.Ü#Ÿk™kØ¤§¡¨c¸+×:LÑ:LÐTa×TgÑTgÔ!hÐjsó�ð
 "�E—M‘M *¨bÐC°5·;±;¸q¸r°?ÒCˆEÜ×+Ñ+ØœŸ	™	 $§+¡+×"9Ñ"9¸T¿^¹^Ñ"KÓLÑLØ˜bÑ ØóˆKð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐá$5™b¸4ÐÙ"6™B¸DÐä! $§&¡&Ó)ò 	J‰HˆAˆuÙ#Ø$5¸Ð8HÑ$HÐ!áØØ*Ø*Ø)Ø#Ø"3ØØ$7ô	ˆGð $ A™JˆMÚ Ø&9¸WÀQ¹Z¸MÑ&IÑ#ð#	Jð( Ÿ	™	 -Ó0ˆáØ 1°]Ð4DÑ DÐáÜñ Ø)¨?Ð<MÐObÐcôó ð ô 9Ø+Ø+Ø+Ø*ô	
ð 	
r0   ©	NNNNNNNNN)r1   r2   r3   r!   rV   rO  r4   r5   rS  r   rï   r   rð   r…   r   r/   r‡   rˆ   s   @r.   r<  r<  ž  s  ø„ ð˜|õ ò*$ð.°5·<±<ó .ð ð .2Ø(,Ø.2Ø04Ø15Ø!%Ø)-Ø,0Ø#'ñF
à×#Ñ# dÑ*ðF
ð  ™ðF
ð Ÿ™ tÑ+ð	F
ð
 ×&Ñ&¨Ñ-ðF
ð ×'Ñ'¨$Ñ.ðF
ð ˜$‘;ðF
ð   $™;ðF
ð # T™kðF
ð ˜D‘[ðF
ð 
ˆu�|‰|˜SÐ Ñ	!Ð$MÑ	MòF
ó ôF
r0   r<  z…
    The Falcon Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings).
    )Úcustom_introc                   ó€  ‡ — e Zd ZddiZdefˆ fd„Zdej                  fd„Ze		 	 	 	 	 	 	 	 	 	 	 ddej                  dz  d	edz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  deej                  z  deej                     ez  fd„«       Zˆ xZS )ÚFalconForCausalLMzlm_head.weightz"transformer.word_embeddings.weightrO   c                 óÆ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        | j                  «        y ©NFr®   )
rU   rV   r<  r#  r   ÚLinearrk   rB  Úlm_headrK  ©r,   rO   ra   s     €r.   rV   zFalconForCausalLM.__init__M  sI   ø€ Ü‰Ñ˜Ô Ü& vÓ.ˆÔÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Õr0   rP  c                 ó   — || _         y r(   )r~  rR  s     r.   Úset_output_embeddingsz'FalconForCausalLM.set_output_embeddingsU  s	   € Ø%ˆ�r0   NrT  rU  r‰   r}   rV  ÚlabelsrÔ   rÕ   rW  rX  Úlogits_to_keepr&   c                 óÊ  — |
�|
n| j                   j                  }
| j                  ||||||||	|
¬«	      }|d   }t        |t        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }d}|�* | j                  ||fd| j                   j                  i|¤Ž}|
s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                  ¬«      S )a\  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
            (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N)rU  r‰   r}   rV  rÔ   rÕ   rW  rX  r   rB  r    ©ÚlossÚlogitsrU  r-   re  )rO   rX  r#  ry   r„   Úslicer~  Úloss_functionrB  r   rU  r-   re  )r,   rT  rU  r‰   r}   rV  r‚  rÔ   rÕ   rW  rX  rƒ  rá   Útransformer_outputsr-   Úslice_indicesÚ	lm_logitsr†  r   s                      r.   r/   zFalconForCausalLM.forwardX  s,  € ðD &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø%Ø'ØØ/Ø!5Ø#ð /ó 

Ðð ,¨AÑ.ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—L‘L ªq°-ÂÐ/BÑ!CÓDˆ	àˆØÐØ%�4×%Ñ%ØØñð  Ÿ;™;×1Ñ1ðð ñ	ˆDñ Ø�\Ð$7¸¸Ð$;Ñ;ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä0ØØØ/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r0   )NNNNNNNNNNr   )r1   r2   r3   Ú_tied_weights_keysr!   rV   r4   r5   r�  r   rï   r   rð   r„   r…   r   r/   r‡   rˆ   s   @r.   rz  rz  E  sE  ø„ ð +Ð,PÐQÐð˜|õ ð&°E·L±Ló &ð ð .2Ø(,Ø.2Ø04Ø-1Ø&*Ø!%Ø)-Ø,0Ø#'Ø-.ñF
à×#Ñ# dÑ*ðF
ð  ™ðF
ð Ÿ™ tÑ+ð	F
ð
 ×&Ñ&¨Ñ-ðF
ð —|‘| dÑ*ðF
ð —‘˜tÑ#ðF
ð ˜$‘;ðF
ð   $™;ðF
ð # T™kðF
ð ˜D‘[ðF
ð ˜eŸl™lÑ*ðF
ð 
ˆu�|‰|Ñ	Ð@Ñ	@òF
ó ôF
r0   rz  aØ  
    The Falcon Model transformer with a sequence classification head on top (linear layer).

    [`FalconForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-1) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   ó  ‡ — e Zd Zdefˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dedz  dej                  dz  dej                  dz  dej                  dz  d	e
dz  d
e
dz  de
dz  de
dz  deej                     ez  fd„«       Zˆ xZS )ÚFalconForSequenceClassificationrO   c                 óè   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  |j                  d¬«      | _        | j                  «        y r|  )
rU   rV   Ú
num_labelsr<  r#  r   r}  rk   ÚscorerK  r  s     €r.   rV   z(FalconForSequenceClassification.__init__±  sV   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ& vÓ.ˆÔÜ—Y‘Y˜v×1Ñ1°6×3DÑ3DÈ5ÔQˆŒ
ð 	�‰Õr0   NrT  rU  r‰   rV  r‚  rÔ   rÕ   rW  rX  r&   c
           
      ór  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }| j                  |«      }|�|j                  d   }n|j                  d   }| j                   j
                  €|dk7  rt        d«      ‚| j                   j
                  €d}nÃ|�“|| j                   j
                  k7  j                  |j                  t        j                  «      }t        j                  |j                  d   |j                  t        j                  ¬«      }||z  j                  d«      }n.d}t        j                  | j                  j                   › d�«       |t        j                  ||j                  ¬	«      |f   }d}|��^| j                   j"                  €�| j$                  dk(  rd
| j                   _        nl| j$                  dkD  rL|j&                  t        j(                  k(  s|j&                  t        j*                  k(  rd| j                   _        nd| j                   _        | j                   j"                  d
k(  rIt-        «       }| j$                  dk(  r& ||j/                  «       |j/                  «       «      }nc |||«      }nY| j                   j"                  dk(  rt1        «       } |||«      }n,| j                   j"                  dk(  rt3        «       } |||«      }|	s|f|dd z   }|�|f|z   S |S t5        |||j6                  |j8                  |j:                  ¬«      S )á6  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
            (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N©rU  r‰   rV  rÔ   rÕ   rW  rX  r   r    z=Cannot handle batch sizes > 1 if no padding token is defined.r8   ri   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r[  Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr…  )rO   rX  r#  r’  r<   Úpad_token_idrµ   ro   r_   r4   r‘   rm   Úargmaxr³   r´   ra   r1   Úproblem_typer‘  rh   rh  r„   r	   Úsqueezer   r   r   rU  r-   re  )r,   rT  rU  r‰   rV  r‚  rÔ   rÕ   rW  rX  rá   rŠ  r-   r‡  r—   Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsr†  Úloss_fctr   s                         r.   r/   z'FalconForSequenceClassification.forwardº  sî  € ð@ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ó 	
Ðð ,¨AÑ.ˆØ—‘˜MÓ*ˆàÐ Ø"Ÿ™¨Ñ+‰Jà&×,Ñ,¨QÑ/ˆJà�;‰;×#Ñ#Ð+°
¸a²ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ð+Ø!#ÑØÐ"à%¨¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸL™L¨¯©¸Ñ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Ñ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|™|¨J¸v¿}¹}ÔMÐOaÐaÑbˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# M×$9Ñ$9Ó$;¸V¿^¹^Ó=MÓN‘Dá# M°6Ó:‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù ¨vÓ6‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨vÓ6�ÙØ#Ð%Ð(;¸A¸BÐ(?Ñ?ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä/ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r0   rw  )r1   r2   r3   r!   rV   r   r4   rï   r   r5   rð   r…   r   r/   r‡   rˆ   s   @r.   r�  r�  ¢  s÷   ø„ ð˜|õ ð ð .2Ø(,Ø.2Ø-1Ø&*Ø!%Ø)-Ø,0Ø#'ñf
à×#Ñ# dÑ*ðf
ð  ™ðf
ð Ÿ™ tÑ+ð	f
ð
 —|‘| dÑ*ðf
ð —‘˜tÑ#ðf
ð ˜$‘;ðf
ð   $™;ðf
ð # T™kðf
ð ˜D‘[ðf
ð 
ˆu�|‰|Ñ	Ð?Ñ	?òf
ó ôf
r0   r�  c                   ó  ‡ — e Zd Zdefˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dedz  dej                  dz  dej                  dz  dej                  dz  d	e
dz  d
e
dz  de
dz  de
dz  deej                     ez  fd„«       Zˆ xZS )ÚFalconForTokenClassificationrO   c                 ó„  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        |dd «      �|j                  }nt        |dd «      �|j                  }nd}t        j                  |«      | _
        t        j                  |j                  |j                  «      | _        | j                  «        y )NÚclassifier_dropoutr°   gš™™™™™¹?)rU   rV   r‘  r<  r#  rj   r¥  r°   r   r¾   r¨   r}  rk   Ú
classifierrK  )r,   rO   r¥  ra   s      €r.   rV   z%FalconForTokenClassification.__init__&  s£   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä& vÓ.ˆÔÜ�6Ð/°Ó6ÐBØ!'×!:Ñ!:ÑÜ�VÐ-¨tÓ4Ð@Ø!'×!6Ñ!6Ñà!$ÐÜ—z‘zÐ"4Ó5ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr0   NrT  rU  r‰   rV  r‚  rÔ   rÕ   rW  rX  r&   c
           
      óÐ  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }| j                  |«      }| j	                  |«      }d}|�Q|j
                  \  }}t        «       } ||j                  ||z  | j                  «      |j                  ||z  «      «      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )r”  Nr•  r   r9   )r†  r‡  r-   re  )rO   rX  r#  r¨   r¦  r<   r   rÄ   r‘  r   r-   re  )r,   rT  rU  r‰   rV  r‚  rÔ   rÕ   rW  rX  rá   rŠ  r-   r‡  r†  r—   r˜   r¡  r   s                      r.   r/   z$FalconForTokenClassification.forward7  s  € ð@ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ó 	
Ðð ,¨AÑ.ˆØŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÐØ%+§\¡\Ñ"ˆJ˜
Ü'Ó)ˆHÙØ—‘˜J¨Ñ3°T·_±_ÓEÀvÇ{Á{ÐS]Ð`jÑSjÓGkóˆDñ Ø�YÐ!4°Q°RÐ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ-×;Ñ;Ø*×5Ñ5ô	
ð 	
r0   rw  )r1   r2   r3   r!   rV   r   r4   rï   r   r5   rð   r…   r   r/   r‡   rˆ   s   @r.   r£  r£  $  s÷   ø„ ð˜|õ ð" ð .2Ø(,Ø.2Ø-1Ø&*Ø!%Ø)-Ø,0Ø#'ñA
à×#Ñ# dÑ*ðA
ð  ™ðA
ð Ÿ™ tÑ+ð	A
ð
 —|‘| dÑ*ðA
ð —‘˜tÑ#ðA
ð ˜$‘;ðA
ð   $™;ðA
ð # T™kðA
ð ˜D‘[ðA
ð 
ˆu�|‰|Ñ	Ð4Ñ	4òA
ó ôA
r0   r£  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  d	edz  d
edz  de	e
z  fd„«       Zˆ xZS )ÚFalconForQuestionAnsweringc                 ó®   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  d«      | _        | j                  «        y )Nr9   )	rU   rV   r<  r#  r   r}  rk   Ú
qa_outputsrK  r  s     €r.   rV   z#FalconForQuestionAnswering.__init__~  sA   ø€ Ü‰Ñ˜Ô Ü& vÓ.ˆÔÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õr0   NrT  r‰   rV  Ústart_positionsÚend_positionsrÕ   rW  rX  r&   c	                 ó"  — |�|n| j                   j                  }| j                  ||||||¬«      }
|
d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      j                  «       }|j                  d«      j                  «       }d}|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }|s||f|
dd z   }|�|f|z   S |S t        ||||
j                  |
j                  ¬	«      S )
rZ  N)r‰   rV  rÕ   rW  rX  r   r    r8   r:   )Úignore_indexr9   )r†  Ústart_logitsÚ
end_logitsr-   re  )rO   rX  r#  r«  Úsplitrœ  Ú
contiguousÚlenÚsizeÚclampr   r   r-   re  )r,   rT  r‰   rV  r¬  r­  rÕ   rW  rX  rá   rv  Úsequence_outputr‡  r°  r±  Ú
total_lossÚignored_indexr¡  Ú
start_lossÚend_lossr   s                        r.   r/   z"FalconForQuestionAnswering.forward†  s»  € ð4 &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà×"Ñ"ØØ)Ø'Ø/Ø!5Ø#ð #ó 
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r0   )NNNNNNNN)r1   r2   r3   rV   r   r4   rï   ÚFloatTensorrð   r…   r   r/   r‡   rˆ   s   @r.   r©  r©  |  sâ   ø„ ôð ð .2Ø37Ø26Ø37Ø15Ø)-Ø,0Ø#'ñF
à×#Ñ# dÑ*ðF
ð ×)Ñ)¨DÑ0ðF
ð ×(Ñ(¨4Ñ/ð	F
ð
 ×)Ñ)¨DÑ0ðF
ð ×'Ñ'¨$Ñ.ðF
ð   $™;ðF
ð # T™kðF
ð ˜D‘[ðF
ð 
Ð-Ñ	-òF
ó ôF
r0   r©  )rz  r<  r"  r�  r£  r©  )r    )Or   rŒ   Úcollections.abcr   Útypingr   r4   r   Útorch.nnr   r   r   r	   r
   r§   Ú r   r)  Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úmasking_utilsr   Úmodeling_flash_attention_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   Úutilsr   r   Úutils.genericr   Úconfiguration_falconr!   r"   Ú
get_loggerr1   r³   r}  r$   rA   rK   r9  rM   r5   r„   rh   r¡   rp   rð   rª   r¬   rò   r  r  r  r"  r<  rz  r�  r£  r©  Ú__all__r6   r0   r.   ú<module>rÏ     s:  ðñ ã Ý $Ý ã Ý ß LÓ LÝ $å &Ý )ß .Ý )Ý /ß hÝ 9÷õ ÷õ .÷õ ,Ý .ñ ÔÝJà	ˆ×	Ñ	˜HÓ	%€ô
)�2—9‘9ô )ò(óô4><˜BŸI™Iô ><ðBJ u§|¡|ð JÀð JÈEÏKÉKð JÐ\a×\hÑ\hó Jð:�5—<‘<ð ¨5¯<©<ð ¸uð ÐPTð ÐY^×YeÑYeó ô&S0�b—i‘iô S0ôlb)˜Oô b)ôJ�—	‘	ô ð" ØØ.ñÐ ôR$Ð3ô R$ðj ô˜Oó ó ðð< ôc
Ð'ó c
ó ðc
ñL ðôô
U
Ð-¨ó U
óð
U
ñp ðôôq
Ð&;ó q
óðq
ðh ôT
Ð#8ó T
ó ðT
ðn ôP
Ð!6ó P
ó ðP
òf�r0   