Ë
    ÛÓ^jA¢  ã            
       óÔ  — d Z ddlZddlZddlmZ ddlmZmZmZmZ ddlm	Z
 ddlmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZ ddlmZ ddlmZmZ ddl m!Z!  ejD                  e#«      Z$dejJ                  de&dejN                  dejJ                  fd„Z(dejJ                  dejJ                  de)de*dejJ                  f
d„Z+dejJ                  dejJ                  fd„Z,dejJ                  dejJ                  dejJ                  fd„Z- G d„ dej\                  j^                  «      Z0 G d„ d ejb                  «      Z2 G d!„ d"ejb                  «      Z3 G d#„ d$ejb                  «      Z4 G d%„ d&e«      Z5e G d'„ d(e«      «       Z6e G d)„ d*e6«      «       Z7 ed+¬,«       G d-„ d.e6e«      «       Z8 ed/¬,«       G d0„ d1e6«      «       Z9e G d2„ d3e6«      «       Z:e G d4„ d5e6«      «       Z;g d6¢Z<y)7zPyTorch BLOOM model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚ	LayerNormÚMSELoss)Ú
functionalé   )ÚCacheÚDynamicCacheÚStaticCache)ÚGenerationMixin)Úcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚQuestionAnsweringModelOutputÚ SequenceClassifierOutputWithPastÚTokenClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )ÚBloomConfigÚattention_maskÚ	num_headsÚdtypeÚreturnc                 óØ  — | j                   \  }}dt        j                  t        j                  |«      «      z  }t	        j
                  ddt        j                  |«      dz
   z   z  | j                  t        j                  ¬«      }t	        j                  dd|z   | j                  t        j                  ¬«      }t	        j                  ||«      }||k7  rÅt	        j
                  ddt        j                  d|z  «      dz
   z   z  | j                  t        j                  ¬«      }	t        |||z
  «      }
t	        j                  ddd|
z  z   d| j                  t        j                  ¬«      }t	        j                  |t	        j                  |	|«      gd¬«      }| j                  d¬«      dz
  | z  dd…ddd…f   }|d	   |z  }|j                  ||z  d|«      j                  |«      S )
aŽ  
    Link to paper: https://huggingface.co/papers/2108.12409 Alibi tensor is not causal as the original paper mentions, it
    relies on a translation invariance of softmax for quick implementation: with l being a tensor, and a fixed value
    `softmax(l+a) = softmax(l)`. Based on
    https://github.com/ofirpress/attention_with_linear_biases/blob/a35aaca144e0eb6b789dfcb46784c4b8e31b7983/fairseq/models/transformer.py#L742
    TODO @thomasw21 this doesn't work as nicely due to the masking strategy, and so masking varies slightly.

    Args:
    Returns tensor shaped (batch_size * num_heads, 1, max_seq_len)
        attention_mask (`torch.Tensor`):
            Token-wise attention mask, this should be of shape (batch_size, max_seq_len).
        num_heads (`int`):
            number of heads
        dtype (`torch.dtype`, *optional*, default=`torch.bfloat16`):
            dtype of the output tensor
    é   r	   ©Údevicer   r   r   ©ÚdiméÿÿÿÿN).N)ÚshapeÚmathÚfloorÚlog2ÚtorchÚtensorr!   Úfloat32ÚarangeÚint32ÚpowÚminÚcatÚcumsumÚreshapeÚto)r   r   r   Ú
batch_sizeÚ
seq_lengthÚclosest_power_of_2ÚbaseÚpowersÚslopesÚ
extra_baseÚnum_remaining_headsÚextra_powersÚarange_tensorÚalibis                 ús/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/bloom/modeling_bloom.pyÚbuild_alibi_tensorr@   -   sº  € ð" ,×1Ñ1Ñ€J�
ØœdŸj™j¬¯©°9Ó)=Ó>Ñ>ÐÜ�<‰<Ø	�”t—y‘yÐ!3Ó4°qÑ8Ð9Ñ9Ð:Ñ;ÀN×DYÑDYÔaf×anÑanô€Dô �\‰\˜!˜QÐ!3Ñ3¸N×<QÑ<QÔY^×YdÑYdÔe€FÜ�Y‰Y�t˜VÓ$€Fà˜YÒ&Ü—\‘\Ø�Aœ4Ÿ9™9 QÐ);Ñ%;Ó<¸qÑ@ÐAÑAÐBÑCÈN×LaÑLaÔin×ivÑivô
ˆ
ô "Ð"4°iÐBTÑ6TÓUÐÜ—|‘| A q¨1Ð/BÑ+BÑ'BÀAÈn×NcÑNcÔkp×kvÑkvÔwˆÜ—‘˜F¤E§I¡I¨j¸,Ó$GÐHÈaÔPˆð %×+Ñ+°Ð+Ó3°aÑ7¸>ÑIÊ1ÈdÒTUÈ:ÑV€MØ�9Ñ Ñ-€EØ�=‰=˜ iÑ/°°JÓ?×BÑBÀ5ÓIÐIó    ÚxÚresidualÚprobÚtrainingc                 ó@   — t        j                  | ||¬«      }||z   }|S )a
  
    Dropout add function

    Args:
        x (`torch.tensor`):
            input tensor
        residual (`torch.tensor`):
            residual tensor
        prob (`float`):
            dropout probability
        training (`bool`):
            training mode
    )ÚprE   )ÚFÚdropout)rB   rC   rD   rE   Úouts        r?   Údropout_addrK   Y   s$   € ô �)‰)�A˜¨Ô
1€CØ
�S‰.€CØ€JrA   c                 ó\   — | dz  dt        j                  d| z  dd| z  | z  z   z  «      z   z  S )zà
    Custom bias GELU function. Adapted from Megatron-DeepSpeed code. Here we use a simple implementation (inference) to
    make the model jitable.

    Args:
        x (`torch.tensor`):
            input hidden states
    ç      à?ç      ð?ç Þe3Eˆé?r   ç÷Hmâä¦?©r)   Útanh)rB   s    r?   Úbloom_gelu_forwardrS   l   s8   € ð ˆs‰7�cœEŸJ™J z°A¡~¸¸XÈ¹\ÈAÑ=MÑ9MÑ'NÓOÑOÑPÐPrA   Úgc                 ó¤   — |d   }t        j                  d|z  dd|z  |z  z   z  «      }d|z  d||z  z
  dd|z  |z  z   z  z  dd|z   z  z   }|| z  S )a   
    gradient of tanh approximation of gelu gradient of actual gelu is: 0.5 * (1. + torch.erf(x * 0.70710678)) +
    0.3989423 * x * torch.exp(-0.5 * x * x)

    Args:
        g (`torch.tensor`):
            gradient output tensor
        x (`torch.tensor`):
            input tensor
    r   rO   r   rP   rM   g6”ü¾vf»?rQ   )rT   rB   Útanh_outÚffs       r?   Úbloom_gelu_backrX   x   sz   € ð 	
ˆ!‰€AÜ�z‰z˜* q™.¨A°¸1±¸qÑ0@Ñ,@ÑAÓB€Hà	ˆq‰�Q˜ HÑ,Ñ,°¸lÈQÑ>NÐQRÑ>RÑ1RÑSÑ	TÐWZÐ^_ÐbjÑ^jÑWkÑ	k€BØ�‰6€MrA   c                   ó�   — e Zd Zedej
                  dej
                  fd„«       Zedej
                  dej
                  fd„«       Zy)ÚGeLUFunctionÚinputr   c                 ó:   — | j                  |«       t        |«      S ©N)Úsave_for_backwardrS   )Úctxr[   s     r?   ÚforwardzGeLUFunction.forward‹   s   € à×Ñ˜eÔ$Ü! %Ó(Ð(rA   Úgrad_outputc                 ó6   — | j                   }t        ||«      }|S r]   )Úsaved_tensorsrX   )r_   ra   r[   Útmps       r?   ÚbackwardzGeLUFunction.backward�   s   € à×!Ñ!ˆÜ˜k¨5Ó1ˆØˆ
rA   N)Ú__name__Ú
__module__Ú__qualname__Ústaticmethodr)   ÚTensorr`   re   © rA   r?   rZ   rZ   Š   sT   „ Øð)˜EŸL™Lð )¨U¯\©\ò )ó ð)ð ð 5§<¡<ð °E·L±Lò ó ñrA   rZ   c                   óZ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  fd„Zˆ xZS )Ú	BloomGeluzN
    Partly copied from Megatron-DeepSpeed code and adapted for our needs
    c                 ó"   •— t         ‰| �  «        y r]   )ÚsuperÚ__init__)ÚselfÚ	__class__s    €r?   rp   zBloomGelu.__init__œ   s   ø€ Ü‰ÑÕrA   rB   r   c                 ó,   — t         j                  |«      S r]   )rZ   Úapply)rq   rB   s     r?   r`   zBloomGelu.forwardŸ   s   € Ü×!Ñ! !Ó$Ð$rA   )	rf   rg   rh   Ú__doc__rp   r)   rj   r`   Ú__classcell__©rr   s   @r?   rm   rm   —   s(   ø„ ñôð%˜Ÿ™ð %¨%¯,©,÷ %rA   rm   c                   óV  ‡ — e Zd Zddededz  fˆ fd„Zdej                  deej                  ej                  ej                  f   fd„Z	dej                  dej                  fd	„Z
	 	 	 dd
ej                  dej                  dej                  dej                  dedz  dedefd„Zˆ xZS )ÚBloomAttentionNÚconfigÚ	layer_idxc                 ó˜  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | _        |j                  | _	        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚dt        j                  | j                  «      z  | _        d| _        || _        |€-t         j#                  d| j$                  j&                  › d�«       t)        j*                  | j                  d| j                  z  d¬	«      | _        t)        j*                  | j                  | j                  «      | _        t)        j0                  |j2                  «      | _        y )
NzA`hidden_size` must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).rN   zInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.r	   T©Úbias)ro   rp   Úpretraining_tpÚslow_but_exactÚhidden_sizeÚn_headr   Úhead_dimÚ
split_sizeÚhidden_dropoutÚ
ValueErrorr&   ÚsqrtÚinv_norm_factorÚbetar{   ÚloggerÚwarning_oncerr   rf   r   ÚLinearÚquery_key_valueÚdenseÚDropoutÚattention_dropout)rq   rz   r{   rr   s      €r?   rp   zBloomAttention.__init__¤   sz  ø€ Ü‰ÑÔà$×3Ñ3ˆÔØ$×3Ñ3ˆÔà!×-Ñ-ˆÔØŸ™ˆŒØ×(Ñ(¨D¯N©NÑ:ˆŒØ×*Ñ*ˆŒØ$×3Ñ3ˆÔà�=‰=˜4Ÿ>™>Ñ)¨T×-=Ñ-=Ò=ÜØSÐTX×TdÑTdÐSeð fØ—N‘NÐ# 2ð'óð ð  #¤T§Y¡Y¨t¯}©}Ó%=Ñ=ˆÔØˆŒ	Ø"ˆŒØÐÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôô  "Ÿy™y¨×)9Ñ)9¸1¸t×?OÑ?OÑ;OÐVZÔ[ˆÔÜ—Y‘Y˜t×/Ñ/°×1AÑ1AÓBˆŒ
Ü!#§¡¨F×,DÑ,DÓ!EˆÕrA   Ú	fused_qkvr   c                 ó  — |j                   \  }}}|j                  ||| j                  d| j                  «      }|dddd…f   j	                  dd«      }|dddd…f   j	                  dd«      }|dddd…f   j	                  dd«      }|||fS )a  
        Split the last dimension into (num_heads, head_dim) and reshapes to (bs, heads, len, dim) shape
        without making any copies, results share same memory storage as `fused_qkv`

        Args:
            fused_qkv (`torch.tensor`): [batch_size, seq_length, num_heads * 3 * head_dim]

        Returns:
            query: [batch_size, num_heads, seq_length, head_dim]
            key: [batch_size, num_heads, seq_length, head_dim]
            value: [batch_size, num_heads, seq_length, head_dim]
        r	   .r   Nr   r   )r%   Úviewr   rƒ   Ú	transpose)rq   r‘   r4   r5   Úthree_times_hidden_sizeÚquery_layerÚ	key_layerÚvalue_layers           r?   Ú_reshapezBloomAttention._reshapeÅ   s˜   € ð ;D¿/¹/Ñ7ˆ
�JÐ 7Ø—N‘N :¨z¸4¿>¹>È1ÈdÏmÉmÓ\ˆ	Ø  Qª 	Ñ*×4Ñ4°Q¸Ó:ˆØ˜c 1¢a˜iÑ(×2Ñ2°1°aÓ8ˆ	Ø  Qª 	Ñ*×4Ñ4°Q¸Ó:ˆØ˜I {Ð2Ð2rA   rB   c                 ó  — |j                   \  }}}|| j                  z  }|j                  || j                  || j                  «      }|j	                  dddd«      }|j                  ||| j                  | j                  z  «      S )z÷
        Merge heads together over the last dimension

        Args:
            x (`torch.tensor`): [batch_size * num_heads, seq_length, head_dim]

        Returns:
            torch.tensor: [batch_size, seq_length, num_heads * head_dim]
        r   r   r   r	   )r%   r   r“   rƒ   Úpermuter2   )rq   rB   Úbatch_size_and_num_headsr5   Ú_r4   s         r?   Ú_merge_headszBloomAttention._merge_headsÙ   sy   € ð 34·'±'Ñ/Ð  *¨aØ-°·±Ñ?ˆ
ð �F‰F�:˜tŸ~™~¨z¸4¿=¹=ÓIˆð �I‰I�a˜˜A˜qÓ!ˆð �y‰y˜ Z°·±À$Ç-Á-Ñ1OÓPÐPrA   Úhidden_statesrC   r>   r   Ú
layer_pastÚ	use_cacheÚoutput_attentionsc                 ó¾  — |j                   \  }	}
}| j                  |«      }| j                  |«      \  }}}|� |j                  ||| j                  «      \  }}|j                  |	| j                  z  d| j                  «      }|j                  |	| j                  z  d| j                  «      j                  dd«      }|j                  |	| j                  z  d| j                  «      }|j                  ||| j                  | j                  ¬«      }|j                  |	| j                  |
d«      }|�||z   }t        j                  |dt        j                   ¬«      j#                  |j$                  «      }| j'                  |«      }|j                  |	| j                  z  |
d«      }t        j(                  ||«      }| j+                  |«      }| j,                  dkD  rÍ| j.                  rÁ| j0                  | j,                  z  }t        j2                  |«      }t5        | j,                  «      D ]z  }|t        j6                  |d d …d d …t9        ||z  «      t9        |dz   |z  «      …f   | j:                  j<                  d d …t9        ||z  «      t9        |dz   |z  «      …f   «      z   }Œ| n| j;                  |«      }t?        ||| j@                  | jB                  «      }||fS )Nr$   éþÿÿÿ)Úbatch1Úbatch2r‰   Úalpha)r#   r   r   )"r%   r�   r™   Úupdater{   r2   r   rƒ   r”   Úbaddbmmr‰   rˆ   r“   rH   Úsoftmaxr)   r+   r3   r   r�   Úbmmrž   r   r€   r�   Ú
zeros_likeÚrangeÚlinearÚintrŽ   ÚweightrK   r…   rE   )rq   rŸ   rC   r>   r   r    r¡   r¢   Úkwargsr4   Úq_lengthr�   r‘   r–   r—   r˜   Úattention_scoresÚattn_weightsÚattention_probsÚattention_probs_reshapedÚcontext_layerÚslicesÚoutput_tensorÚis                           r?   r`   zBloomAttention.forwardò   s¯  € ð #0×"5Ñ"5Ñˆ
�H˜aØ×(Ñ(¨Ó7ˆ	à.2¯m©m¸IÓ.FÑ+ˆ�Y àÐ!Ø%/×%6Ñ%6°yÀ+ÈtÏ~É~Ó%^Ñ"ˆI�{ð "×)Ñ)¨*°t·~±~Ñ*EÀrÈ4Ï=É=ÓYˆØ×%Ñ% j°4·>±>Ñ&AÀ2ÀtÇ}Á}ÓU×_Ñ_Ð`bÐdfÓgˆ	Ø!×)Ñ)¨*°t·~±~Ñ*EÀrÈ4Ï=É=ÓYˆð !Ÿ=™=ØØØ—‘Ø×&Ñ&ð	 )ó 
Ðð (×,Ñ,¨Z¸¿¹ÈÐSUÓVˆØÐ%Ø'¨.Ñ8ˆLô Ÿ)™) L°bÄÇÁÔN×QÑQÐR]×RcÑRcÓdˆð ×0Ñ0°ÓAˆð $3×#7Ñ#7¸
ÀTÇ^Á^Ñ8SÐU]Ð_aÓ#bÐ ô Ÿ	™	Ð":¸KÓHˆð ×)Ñ)¨-Ó8ˆð ×Ñ Ò" t×':Ò':Ø×%Ñ%¨×(;Ñ(;Ñ;ˆFÜ!×,Ñ,¨]Ó;ˆMÜ˜4×.Ñ.Ó/ò �Ø -´·±Ø!¢!¢Q¬¨A°©J«¼#¸qÀ1¹uÈÑ>NÓ:OÐ(OÐ"OÑPØ—J‘J×%Ñ%¢a¬¨Q°©Z«¼3ÀÀAÁÈÑ?OÓ;PÐ)PÐ&PÑQó1ñ !‘ñð !ŸJ™J }Ó5ˆMä# M°8¸T×=PÑ=PÐRV×R_ÑR_Ó`ˆØ˜oÐ-Ð-rA   r]   ©NFF)rf   rg   rh   r   r¯   rp   r)   rj   Útupler™   rž   r
   Úboolr`   rv   rw   s   @r?   ry   ry   £   sÞ   ø„ ñF˜{ð F°s¸T±zõ FðB3 %§,¡,ð 3°5¸¿¹ÀuÇ|Á|ÐUZ×UaÑUaÐ9aÑ3bó 3ð(Q˜eŸl™lð Q¨u¯|©|ó Qð> $(ØØ"'ñA.à—|‘|ðA.ð —,‘,ðA.ð �|‰|ð	A.ð
 Ÿ™ðA.ð ˜D‘LðA.ð ðA.ð  ÷A.rA   ry   c                   ót   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  dej                  fd„Zˆ xZS )ÚBloomMLPrz   c                 ó6  •— t         ‰| �  «        |j                  }|j                  | _        |j                  | _        t        j                  |d|z  «      | _        t        «       | _	        t        j                  d|z  |«      | _
        |j                  | _        y )Né   )ro   rp   r�   r   r€   r   rŒ   Údense_h_to_4hrm   Ú	gelu_implÚdense_4h_to_hr…   )rq   rz   r�   rr   s      €r?   rp   zBloomMLP.__init__7  sz   ø€ Ü‰ÑÔØ×(Ñ(ˆà$×3Ñ3ˆÔØ$×3Ñ3ˆÔÜŸY™Y {°A¸±OÓDˆÔÜ"›ˆŒÜŸY™Y q¨;¡¸ÓDˆÔØ$×3Ñ3ˆÕrA   rŸ   rC   r   c                 ó’  — | j                  | j                  |«      «      }| j                  dkD  rä| j                  rØt	        j
                  |«      }| j                  j                  j                  d   | j                  z  }t        | j                  «      D ]z  }|t        j                  |d d …d d …t        ||z  «      t        |dz   |z  «      …f   | j                  j                  d d …t        ||z  «      t        |dz   |z  «      …f   «      z   }Œ| n| j                  |«      }t        ||| j                  | j                  «      }|S )Nr   r$   )rÃ   rÂ   r   r€   r)   r¬   rÄ   r°   r%   r­   rH   r®   r¯   rK   r…   rE   )rq   rŸ   rC   Úintermediate_outputr¸   rº   Úoutputs          r?   r`   zBloomMLP.forwardB  s.  € ØŸ™ t×'9Ñ'9¸-Ó'HÓIˆà×Ñ Ò" t×':Ò':Ü"'×"2Ñ"2°8Ó"<ÐØ×'Ñ'×.Ñ.×4Ñ4°RÑ8¸4×;NÑ;NÑNˆFÜ˜4×.Ñ.Ó/ò �Ø&9¼A¿H¹HØ!¢!¢Q¬¨A°©J«¼#¸qÀ1¹uÈÑ>NÓ:OÐ(OÐ"OÑPØ×&Ñ&×-Ñ-ªa´°Q¸±Z³Ä3ÈÈAÉÐQWÑGWÓCXÐ1XÐ.XÑYó=ñ 'Ñ#ñð #'×"4Ñ"4°]Ó"CÐäÐ0°(¸D×<OÑ<OÐQU×Q^ÑQ^Ó_ˆàˆrA   )	rf   rg   rh   r   rp   r)   rj   r`   rv   rw   s   @r?   r¿   r¿   6  s5   ø„ ð	4˜{õ 	4ð U§\¡\ð ¸U¿\¹\ð ÈeÏlÉl÷ rA   r¿   c                   óš   ‡ — e Zd Zddededz  fˆ fd„Z	 	 	 ddej                  dej                  dej                  dedz  d	e	d
e	fd„Z
ˆ xZS )Ú
BloomBlockNrz   r{   c                 óR  •— t         ‰| �  «        |j                  }t        ||j                  ¬«      | _        |j                  | _        t        ||«      | _	        t        ||j                  ¬«      | _
        t        |«      | _        |j                  | _        |j                  | _        y )N©Úeps)ro   rp   r�   r   Úlayer_norm_epsilonÚinput_layernormr‚   r   ry   Úself_attentionÚpost_attention_layernormr¿   ÚmlpÚ(apply_residual_connection_post_layernormr…   )rq   rz   r{   r�   rr   s       €r?   rp   zBloomBlock.__init__V  s…   ø€ Ü‰ÑÔØ×(Ñ(ˆä(¨¸&×:SÑ:SÔTˆÔØŸ™ˆŒÜ,¨V°YÓ?ˆÔÜ(1°+À6×C\ÑC\Ô(]ˆÔ%ä˜FÓ#ˆŒà8>×8gÑ8gˆÔ5Ø$×3Ñ3ˆÕrA   rŸ   r>   r   r    r¡   r¢   c           	      óì   — | j                  |«      }| j                  r|}	n|}	| j                  ||	|||||¬«      \  }
}| j                  |
«      }| j                  r|}	n|
}	| j	                  ||	«      }||fS )N)r    r   r>   r¡   r¢   )rÎ   rÒ   rÏ   rÐ   rÑ   )rq   rŸ   r>   r   r    r¡   r¢   r±   Úlayernorm_outputrC   Úattention_outputr´   rÇ   s                r?   r`   zBloomBlock.forwardd  s£   € ð  ×/Ñ/°Ó>Ðð ×8Ò8Ø'‰Hà$ˆHð *.×)<Ñ)<ØØØ!Ø)ØØØ/ð *=ó *
Ñ&Ð˜,ð  ×8Ñ8Ð9IÓJÐð ×8Ò8Ø'‰Hà'ˆHð —‘Ð*¨HÓ5ˆà�|Ð#Ð#rA   r]   r»   )rf   rg   rh   r   r¯   rp   r)   rj   r
   r½   r`   rv   rw   s   @r?   rÉ   rÉ   U  st   ø„ ñ4˜{ð 4°s¸T±zõ 4ð& $(ØØ"'ñ+$à—|‘|ð+$ð �|‰|ð+$ð Ÿ™ð	+$ð
 ˜D‘Lð+$ð ð+$ð  ÷+$rA   rÉ   c                   ó0   — e Zd ZU eed<   dZdZdgZdgZdZ	y)ÚBloomPreTrainedModelrz   ÚtransformerTrÉ   Úpast_key_valuesN)
rf   rg   rh   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_can_compile_fullgraphrk   rA   r?   r×   r×   ’  s+   … àÓØ%ÐØ&*Ð#Ø%˜ÐØ#4Ð"5ÐØ!ÑrA   r×   c                   óv  ‡ — e Zd Zdefˆ fd„Zdej                  dedej                  dej                  fd„Z	d„ Z
d	ej                  fd
„Ze	 	 	 	 	 	 	 	 ddej                  dz  dedz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  deej                  df   ez  fd„«       Zˆ xZS )Ú
BloomModelrz   c           	      ó"  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  | j                  «      | _	        t        | j                  |j                  ¬«      | _        t        j                  t        |j                  «      D �cg c]  }t!        ||¬«      ‘Œ c}«      | _        t        | j                  |j                  ¬«      | _        d| _        | j)                  «        y c c}w )NrË   )r{   F)ro   rp   r�   Ú	embed_dimr‚   r   r   Ú	EmbeddingÚ
vocab_sizeÚword_embeddingsr   rÍ   Úword_embeddings_layernormÚ
ModuleListr­   Únum_hidden_layersrÉ   ÚhÚln_fÚgradient_checkpointingÚ	post_init)rq   rz   rº   rr   s      €r?   rp   zBloomModel.__init__ž  sÃ   ø€ Ü‰Ñ˜Ô à×+Ñ+ˆŒØŸ™ˆŒô  "Ÿ|™|¨F×,=Ñ,=¸t¿~¹~ÓNˆÔÜ)2°4·>±>Àv×G`ÑG`Ô)aˆÔ&ô —‘ÌÈv×OgÑOgÓIhÖiÀA¤
¨6¸QÖ ?ÒiÓjˆŒô ˜dŸn™n°&×2KÑ2KÔLˆŒ	à&+ˆÔ#ð 	�‰Õùò  js   Â.Dr   r   r   r   c                 ó   — t        |||«      S r]   )r@   )rq   r   r   r   s       r?   r@   zBloomModel.build_alibi_tensor³  s   € Ü! .°)¸UÓCÐCrA   c                 ó   — | j                   S r]   ©ræ   )rq   s    r?   Úget_input_embeddingszBloomModel.get_input_embeddings¶  s   € Ø×#Ñ#Ð#rA   Únew_embeddingsc                 ó   — || _         y r]   rð   ©rq   rò   s     r?   Úset_input_embeddingszBloomModel.set_input_embeddings¹  s
   € Ø-ˆÕrA   NÚ	input_idsrÙ   Úinputs_embedsr¡   r¢   Úoutput_hidden_statesÚreturn_dict.c	           	      óN  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|du |duz  rt        d«      ‚| j                  r%| j                  r|rt        j                  d«       d}|€| j                  |«      }|r|€t        | j                   ¬«      }|j                  \  }
}}|�|j                  «       nd}||z   }| j                  |«      }|rdnd}|rdnd}|€$t        j                   |
|f|j"                  ¬«      }n|j%                  |j"                  «      }| j'                  || j(                  |j*                  ¬	«      }t-        | j                   |||¬
«      }t/        | j0                  «      D ],  \  }}|r||fz   } |||||||¬«      }|d   }|sŒ$||d   fz   }Œ. | j3                  |«      }|r||fz   }|st5        d„ ||||fD «       «      S t7        ||||¬«      S )á²  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
            (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)rz   r   rk   ©r!   )r   )rz   r÷   r   rÙ   )r    r   r¡   r¢   r>   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr]   rk   )Ú.0Úvs     r?   ú	<genexpr>z%BloomModel.forward.<locals>.<genexpr>  s   è ø€ ò ØÐghÑgt”ñùs   ‚Š)Úlast_hidden_staterÙ   rŸ   Ú
attentions)rz   r¢   rø   r¡   rù   r†   rì   rE   rŠ   r‹   ræ   r   r%   Úget_seq_lengthrç   r)   Úonesr!   r3   r@   r   r   r   Ú	enumeraterê   rë   r¼   r   )rq   rö   rÙ   r   r÷   r¡   r¢   rø   rù   r±   r4   r5   r�   Úpast_lengthÚseq_length_with_pastrŸ   Úall_self_attentionsÚall_hidden_statesr>   Úcausal_maskrº   ÚblockÚoutputss                          r?   r`   zBloomModel.forward¼  sx  € ð4 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×BYÑBYˆà˜Ð -°tÐ";Ò<ÜÐYÓZÐZà×&Ò&¨4¯=ª=¹YÜ×ÑØlôð ˆIàÐ Ø ×0Ñ0°Ó;ˆMá˜Ð0Ü*°$·+±+Ô>ˆOà$1×$7Ñ$7Ñ!ˆ
�J Ø:IÐ:U�o×4Ñ4Ô6Ð[\ˆØ)¨KÑ7Ðà×6Ñ6°}ÓEˆá$5™b¸4ÐÙ"6™B¸DÐð Ð!Ü"ŸZ™Z¨Ð5IÐ(JÐS`×SgÑSgÔh‰Nà+×.Ñ.¨}×/CÑ/CÓDˆNà×'Ñ'¨¸¿¹Èm×NaÑNaÐ'ÓbˆÜ(Ø—;‘;Ø'Ø)Ø+ô	
ˆô " $§&¡&Ó)ò 	J‰HˆAˆuÙ#Ø$5¸Ð8HÑ$HÐ!áØØ*Ø*Ø#Ø"3ØôˆGð $ A™JˆMÚ Ø&9¸WÀQ¹Z¸MÑ&IÑ#ð	Jð$ Ÿ	™	 -Ó0ˆáØ 1°]Ð4DÑ DÐáÜñ Ø)¨?Ð<MÐObÐcôó ð ô 9Ø+Ø+Ø+Ø*ô	
ð 	
rA   ©NNNNNNNN)rf   rg   rh   r   rp   r)   rj   r¯   r   r@   rñ   rõ   r   Ú
LongTensorr
   r½   r¼   r   r`   rv   rw   s   @r?   rá   rá   œ  s4  ø„ ð˜{õ ð*D°·±ð DÈ#ð DÐV[×VaÑVað DÐfk×frÑfró Dò$ð.°5·<±<ó .ð ð .2Ø(,Ø.2Ø15Ø!%Ø)-Ø,0Ø#'ñg
à×#Ñ# dÑ*ðg
ð  ™ðg
ð Ÿ™ tÑ+ð	g
ð
 ×'Ñ'¨$Ñ.ðg
ð ˜$‘;ðg
ð   $™;ðg
ð # T™kðg
ð ˜D‘[ðg
ð 
ˆu�|‰|˜SÐ Ñ	!Ð$MÑ	Mòg
ó ôg
rA   rá   zˆ
    The Bloom Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    )Úcustom_introc                   óv  ‡ — e Zd ZddiZdefˆ fd„Zdej                  fd„Z	 	 	 	 	 dˆ fd„	Z	e
	 	 	 	 	 	 	 	 	 	 dd	ej                  dz  d
edz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  deej                  z  deej                     ez  fd„«       Zˆ xZS )ÚBloomForCausalLMzlm_head.weightz"transformer.word_embeddings.weightrz   c                 óÆ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        | j                  «        y ©NFr}   )
ro   rp   rá   rØ   r   rŒ   r�   rå   Úlm_headrí   ©rq   rz   rr   s     €r?   rp   zBloomForCausalLM.__init__0  sI   ø€ Ü‰Ñ˜Ô Ü% fÓ-ˆÔÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰ÕrA   rò   c                 ó   — || _         y r]   )r  rô   s     r?   Úset_output_embeddingsz&BloomForCausalLM.set_output_embeddings8  s	   € Ø%ˆ�rA   Nc           	      ó6  •— t        ‰| �  |f|||||dœ|¤Ž}t        |t        «      rq|�o|j	                  «       }	|j
                  \  }
}|	|z
  }t        j                  |
||j                  |j                  ¬«      }t        j                  ||gd¬«      }||d<   |S )N)rÙ   r   r÷   r¡   Úis_first_iterationr    r$   r"   r   )ro   Úprepare_inputs_for_generationÚ
isinstancer   Úget_max_lengthr%   r)   Úzerosr!   r   r0   )rq   rö   rÙ   r   r÷   r¡   r  r±   Úmodel_inputsÚtarget_lengthr4   r5   ÚdiffÚnew_attn_maskrr   s                 €r?   r  z.BloomForCausalLM.prepare_inputs_for_generation;  sµ   ø€ ô ‘wÑ<Øð
à+Ø)Ø'ØØ1ñ
ð ñ
ˆô �o¤{Ô3¸Ð8RØ+×:Ñ:Ó<ˆMØ%3×%9Ñ%9Ñ"ˆJ˜
Ø  :Ñ-ˆDä!ŸK™K¨
°DÀ×AVÑAVÐ^l×^rÑ^rÔsˆMÜ"ŸY™Y¨¸Ð'FÈBÔOˆNØ-;ˆLÐ)Ñ*àÐrA   rö   rÙ   r   r÷   Úlabelsr¡   r¢   rø   rù   Úlogits_to_keepr   c           
      óä  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }t        |
t        «      rt        |
 d«      n|
}| j                  |dd…|dd…f   «      }d}|�8| j                  ||| j                   j                  |j                  d«      ¬«      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                  ¬«      S )a\  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
            (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N©rÙ   r   r÷   r¡   r¢   rø   rù   r   Únum_items_in_batch)rå   r&  r   ©ÚlossÚlogitsrÙ   rŸ   r  )rz   rù   rØ   r  r¯   Úslicer  Úloss_functionrå   Úgetr   rÙ   rŸ   r  )rq   rö   rÙ   r   r÷   r"  r¡   r¢   rø   rù   r#  r±   Útransformer_outputsrŸ   Úslice_indicesr)  r(  rÇ   s                     r?   r`   zBloomForCausalLM.forward^  s(  € ð@ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ó 	
Ðð ,¨AÑ.ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ×%Ñ%ØØØŸ;™;×1Ñ1Ø#)§:¡:Ð.BÓ#Cð	 &ó ˆDñ Ø�YÐ!4°Q°RÐ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä0ØØØ/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
rA   )NNNTF)
NNNNNNNNNr   )rf   rg   rh   Ú_tied_weights_keysr   rp   r)   rj   r  r  r   r  r
   r½   r¯   r¼   r   r`   rv   rw   s   @r?   r  r  '  sC  ø„ ð +Ð,PÐQÐð˜{õ ð&°E·L±Ló &ð ØØØØ õ!ðF ð .2Ø(,Ø.2Ø-1Ø&*Ø!%Ø)-Ø,0Ø#'Ø-.ñD
à×#Ñ# dÑ*ðD
ð  ™ðD
ð Ÿ™ tÑ+ð	D
ð
 —|‘| dÑ*ðD
ð —‘˜tÑ#ðD
ð ˜$‘;ðD
ð   $™;ðD
ð # T™kðD
ð ˜D‘[ðD
ð ˜eŸl™lÑ*ðD
ð 
ˆu�|‰|Ñ	Ð@Ñ	@òD
ó ôD
rA   r  aÖ  
    The Bloom Model transformer with a sequence classification head on top (linear layer).

    [`BloomForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-1) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   ó  ‡ — e Zd Zdefˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dedz  dej                  dz  dej                  dz  dej                  dz  d	e
dz  d
e
dz  de
dz  de
dz  deej                     ez  fd„«       Zˆ xZS )ÚBloomForSequenceClassificationrz   c                 óè   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  |j                  d¬«      | _        | j                  «        y r  )
ro   rp   Ú
num_labelsrá   rØ   r   rŒ   r�   Úscorerí   r  s     €r?   rp   z'BloomForSequenceClassification.__init__µ  sV   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ% fÓ-ˆÔÜ—Y‘Y˜v×1Ñ1°6×3DÑ3DÈ5ÔQˆŒ
ð 	�‰ÕrA   Nrö   rÙ   r   r÷   r"  r¡   r¢   rø   rù   r   c
           
      ór  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }| j                  |«      }|�|j                  d   }n|j                  d   }| j                   j
                  €|dk7  rt        d«      ‚| j                   j
                  €d}nÃ|�“|| j                   j
                  k7  j                  |j                  t        j                  «      }t        j                  |j                  d   |j                  t        j                  ¬«      }||z  j                  d«      }n.d}t        j                  | j                  j                   › d�«       |t        j                  ||j                  ¬	«      |f   }d}|��^| j                   j"                  €�| j$                  dk(  rd
| j                   _        nl| j$                  dkD  rL|j&                  t        j(                  k(  s|j&                  t        j*                  k(  rd| j                   _        nd| j                   _        | j                   j"                  d
k(  rIt-        «       }| j$                  dk(  r& ||j/                  «       |j/                  «       «      }nc |||«      }nY| j                   j"                  dk(  rt1        «       } |||«      }n,| j                   j"                  dk(  rt3        «       } |||«      }|	s|f|dd z   }|�|f|z   S |S t5        |||j6                  |j8                  |j:                  ¬«      S )á6  
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()`
            (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.

            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr%  r   r   z=Cannot handle batch sizes > 1 if no padding token is defined.r$   r    zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rü   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr'  )rz   rù   rØ   r4  r%   Úpad_token_idr†   r3   r!   r)   r-   r,   ÚargmaxrŠ   r‹   rr   rf   Úproblem_typer3  r   Úlongr¯   r   Úsqueezer   r   r   rÙ   rŸ   r  )rq   rö   rÙ   r   r÷   r"  r¡   r¢   rø   rù   r±   r-  rŸ   r)  r4   Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsr(  Úloss_fctrÇ   s                         r?   r`   z&BloomForSequenceClassification.forward¾  sí  € ð> &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ó 	
Ðð ,¨AÑ.ˆØ—‘˜MÓ*ˆàÐ Ø"Ÿ™¨Ñ+‰Jà&×,Ñ,¨QÑ/ˆJà�;‰;×#Ñ#Ð+°
¸a²ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ð+Ø!#ÑØÐ"à%¨¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸL™L¨¯©¸Ñ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Ñ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|™|¨J¸v¿}¹}ÔMÐOaÐaÑbˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# M×$9Ñ$9Ó$;¸V¿^¹^Ó=MÓN‘Dá# M°6Ó:‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù ¨vÓ6‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨vÓ6�ÙØ#Ð%Ð(;¸A¸BÐ(?Ñ?ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä/ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
rA   ©	NNNNNNNNN)rf   rg   rh   r   rp   r   r)   r  r
   rj   r½   r¼   r   r`   rv   rw   s   @r?   r1  r1  ¦  s÷   ø„ ð˜{õ ð ð .2Ø(,Ø.2Ø-1Ø&*Ø!%Ø)-Ø,0Ø#'ñe
à×#Ñ# dÑ*ðe
ð  ™ðe
ð Ÿ™ tÑ+ð	e
ð
 —|‘| dÑ*ðe
ð —‘˜tÑ#ðe
ð ˜$‘;ðe
ð   $™;ðe
ð # T™kðe
ð ˜D‘[ðe
ð 
ˆu�|‰|Ñ	Ð?Ñ	?òe
ó ôe
rA   r1  c                   ó  ‡ — e Zd Zdefˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dedz  dej                  dz  dej                  dz  dej                  dz  d	e
dz  d
e
dz  de
dz  de
dz  deej                     ez  fd„«       Zˆ xZS )ÚBloomForTokenClassificationrz   c                 ó°  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        |d«      r|j                  �|j                  }n't        |d«      r|j                  �|j                  }nd}t        j                  |«      | _
        t        j                  |j                  |j                  «      | _        | j                  «        y )NÚclassifier_dropoutr…   gš™™™™™¹?)ro   rp   r3  rá   rØ   ÚhasattrrH  r…   r   r�   rI   rŒ   r�   Ú
classifierrí   )rq   rz   rH  rr   s      €r?   rp   z$BloomForTokenClassification.__init__)  s¯   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä% fÓ-ˆÔÜ�6Ð/Ô0°V×5NÑ5NÐ5ZØ!'×!:Ñ!:ÑÜ�VÐ-Ô.°6×3HÑ3HÐ3TØ!'×!6Ñ!6Ñà!$ÐÜ—z‘zÐ"4Ó5ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrA   Nrö   rÙ   r   r÷   r"  r¡   r¢   rø   rù   r   c
           
      ó  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }| j                  |«      }| j	                  |«      }d}|�l|j                  |j                  «      }|j                  \  }}t        «       } ||j                  ||z  | j                  «      |j                  ||z  «      «      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )r6  Nr%  r   r   )r(  r)  rŸ   r  )rz   rù   rØ   rI   rJ  r3   r!   r%   r   r“   r3  r   rŸ   r  )rq   rö   rÙ   r   r÷   r"  r¡   r¢   rø   rù   r±   r-  rŸ   r)  r(  r4   r5   rC  rÇ   s                      r?   r`   z#BloomForTokenClassification.forward:  s+  € ð> &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà"×.Ñ.ØØ+Ø)Ø'ØØ/Ø!5Ø#ð /ó 	
Ðð ,¨AÑ.ˆØŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÐà—Y‘Y˜vŸ}™}Ó-ˆFØ%+§\¡\Ñ"ˆJ˜
Ü'Ó)ˆHÙØ—‘˜J¨Ñ3°T·_±_ÓEÀvÇ{Á{ÐS]Ð`jÑSjÓGkóˆDñ Ø�YÐ!4°Q°RÐ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ-×;Ñ;Ø*×5Ñ5ô	
ð 	
rA   rD  )rf   rg   rh   r   rp   r   r)   r  r
   rj   r½   r¼   r   r`   rv   rw   s   @r?   rF  rF  '  s÷   ø„ ð˜{õ ð" ð .2Ø(,Ø.2Ø-1Ø&*Ø!%Ø)-Ø,0Ø#'ñB
à×#Ñ# dÑ*ðB
ð  ™ðB
ð Ÿ™ tÑ+ð	B
ð
 —|‘| dÑ*ðB
ð —‘˜tÑ#ðB
ð ˜$‘;ðB
ð   $™;ðB
ð # T™kðB
ð ˜D‘[ðB
ð 
ˆu�|‰|Ñ	Ð4Ñ	4òB
ó ôB
rA   rF  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  d	edz  d
edz  de	e
z  fd„«       Zˆ xZS )ÚBloomForQuestionAnsweringc                 ó®   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  d«      | _        | j                  «        y )Nr   )	ro   rp   rá   rØ   r   rŒ   r�   Ú
qa_outputsrí   r  s     €r?   rp   z"BloomForQuestionAnswering.__init__‚  sA   ø€ Ü‰Ñ˜Ô Ü% fÓ-ˆÔÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÕrA   Nrö   r   r÷   Ústart_positionsÚend_positionsr¢   rø   rù   r   c	                 ó"  — |�|n| j                   j                  }| j                  ||||||¬«      }
|
d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      j                  «       }|j                  d«      j                  «       }d}|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }|s||f|
dd z   }|�|f|z   S |S t        ||||
j                  |
j                  ¬	«      S )
rû   N)r   r÷   r¢   rø   rù   r   r   r$   r"   )Úignore_indexr   )r(  Ústart_logitsÚ
end_logitsrŸ   r  )rz   rù   rØ   rO  Úsplitr>  Ú
contiguousÚlenÚsizeÚclampr   r   rŸ   r  )rq   rö   r   r÷   rP  rQ  r¢   rø   rù   r±   r  Úsequence_outputr)  rT  rU  Ú
total_lossÚignored_indexrC  Ú
start_lossÚend_lossrÇ   s                        r?   r`   z!BloomForQuestionAnswering.forwardŠ  s»  € ð4 &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà×"Ñ"ØØ)Ø'Ø/Ø!5Ø#ð #ó 
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rA   r  )rf   rg   rh   rp   r   r)   r  ÚFloatTensorr½   r¼   r   r`   rv   rw   s   @r?   rM  rM  €  sâ   ø„ ôð ð .2Ø37Ø26Ø37Ø15Ø)-Ø,0Ø#'ñF
à×#Ñ# dÑ*ðF
ð ×)Ñ)¨DÑ0ðF
ð ×(Ñ(¨4Ñ/ð	F
ð
 ×)Ñ)¨DÑ0ðF
ð ×'Ñ'¨$Ñ.ðF
ð   $™;ðF
ð # T™kðF
ð ˜D‘[ðF
ð 
Ð-Ñ	-òF
ó ôF
rA   rM  )r  rá   r×   r1  rF  rM  )=ru   r&   r)   r   Útorch.nnr   r   r   r   r   rH   Úcache_utilsr
   r   r   Ú
generationr   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_bloomr   Ú
get_loggerrf   rŠ   rj   r¯   r   r@   Úfloatr½   rK   rS   rX   ÚautogradÚFunctionrZ   ÚModulerm   ry   r¿   rÉ   r×   rá   r  r1  rF  rM  Ú__all__rk   rA   r?   ú<module>rp     s9  ðñ ã ã Ý ß LÓ LÝ $ç ;Ñ ;Ý )Ý /Ý 9÷õ õ .÷õ -ð 
ˆ×	Ñ	˜HÓ	%€ð)J u§|¡|ð )JÀð )JÈEÏKÉKð )JÐ\a×\hÑ\hó )JðX�5—<‘<ð ¨5¯<©<ð ¸uð ÐPTð ÐY^×YeÑYeó ð&	Q˜%Ÿ,™,ð 	Q¨5¯<©<ó 	Qð�u—|‘|ð ¨¯©ð ¸¿¹ó ô$
�5—>‘>×*Ñ*ô 
ô	%�—	‘	ô 	%ôP.�R—Y‘Yô P.ôfˆr�y‰yô ô>:$Ð+ô :$ðz ô"˜?ó "ó ð"ð ôG
Ð%ó G
ó ðG
ñT ðôôv
Ð+¨_ó v
óðv
ñr ðôôp
Ð%9ó p
óðp
ðf ôU
Ð"6ó U
ó ðU
ðp ôP
Ð 4ó P
ó ðP
òf�rA   