Ë
    ÜÓ^jæ  ã                   ó*  — d Z ddlmZ ddlZddlmZ ddlmZmZmZ ddl	m
Z ddlmZmZ dd	lmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4  e-jj                  e6«      Z7 G d„ dejp                  «      Z9	 	 dNdejp                  dejt                  dejt                  dejt                  dejt                  dz  de;dz  de;de'e+   fd „Z< G d!„ d"ejp                  «      Z= G d#„ d$ejp                  «      Z> G d%„ d&ejp                  «      Z? G d'„ d(ejp                  «      Z@ G d)„ d*ejp                  «      ZA G d+„ d,ejp                  «      ZB G d-„ d.ejp                  «      ZC G d/„ d0e«      ZD G d1„ d2ejp                  «      ZE G d3„ d4ejp                  «      ZFe, G d5„ d6e%«      «       ZG e,d7¬8«       G d9„ d:eG«      «       ZH e,d;¬8«       G d<„ d=eGe«      «       ZIe, G d>„ d?eG«      «       ZJ G d@„ dAejp                  «      ZK e,dB¬8«       G dC„ dDeG«      «       ZLe, G dE„ dFeG«      «       ZMe, G dG„ dHeG«      «       ZN G dI„ dJejp                  «      ZOe, G dK„ dLeG«      «       ZPg dM¢ZQy)OzPyTorch X-MOD model.é    )ÚCallableN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FNÚgelu)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )Ú
XmodConfigc                   óì   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  ded	ej                  fd
„Z
ed„ «       Zedd„«       Zˆ xZS )ÚXmodEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óT  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j
                  «      | _	        t        j                  |j
                  |j                  ¬«      | _
        t        j                  |j                  «      | _        | j                  dt!        j"                  |j$                  «      j'                  d«      d¬«       | j                  dt!        j(                  | j*                  j-                  «       t         j.                  ¬«      d¬«       |j                  | _        t        j                  |j$                  |j
                  | j0                  ¬«      | _        y )	N)Úpadding_idx©ÚepsÚposition_ids©r%   éÿÿÿÿF)Ú
persistentÚtoken_type_ids)Údtype)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferÚtorchÚarangeÚmax_position_embeddingsÚexpandÚzerosr-   ÚsizeÚlongr*   Úposition_embeddings©ÚselfÚconfigÚ	__class__s     €úq/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/xmod/modeling_xmod.pyr4   zXmodEmbeddings.__init__6   s4  ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒà×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	ô 	
ð 	×ÑØœeŸk™k¨$×*;Ñ*;×*@Ñ*@Ó*BÌ%Ï*É*ÔUÐbgð 	ô 	
ð "×.Ñ.ˆÔÜ#%§<¡<Ø×*Ñ*¨F×,>Ñ,>ÈD×L\ÑL\ô$
ˆÕ ó    NÚ	input_idsr1   r-   Úinputs_embedsÚpast_key_values_lengthÚreturnc                 ó  — |€<|�| j                  || j                  |«      }n| j                  || j                  «      }|�|j                  «       }n|j                  «       d d }|\  }}|€³t	        | d«      rm| j
                  j                  |j                  «      j                  |j                  d   d«      }	t        j                  |	d|¬«      }	|	j                  ||«      }n:t        j                  |t        j                  | j                  j                  ¬«      }|€| j                  |«      }| j!                  |«      }
||
z   }| j#                  |«      }||z   }| j%                  |«      }| j'                  |«      }|S )Nr/   r1   r   r%   )ÚdimÚindex©r2   Údevice)Ú"create_position_ids_from_input_idsr*   Ú&create_position_ids_from_inputs_embedsrG   Úhasattrr1   ÚtorX   rE   ÚshaperB   ÚgatherrF   rH   r-   r9   r;   rI   r<   r@   )rK   rP   r1   r-   rQ   rR   Úinput_shapeÚ
batch_sizeÚ
seq_lengthÚbuffered_token_type_idsr;   Ú
embeddingsrI   s                rN   ÚforwardzXmodEmbeddings.forwardJ   s€  € ð ÐØÐ$à#×FÑFØ˜t×/Ñ/Ð1Gó ‘ð  $×JÑJÈ=ÐZ^×ZjÑZjÓk�àÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà!,Ñˆ
�Jð
 Ð!Ü�tÐ-Ô.à*.×*=Ñ*=×*@Ñ*@À×ATÑATÓ*U×*\Ñ*\Ð]i×]oÑ]oÐpqÑ]rÐtvÓ*wÐ'Ü*/¯,©,Ð7NÐTUÐ]iÔ*jÐ'Ø!8×!?Ñ!?À
ÈJÓ!W‘ä!&§¡¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÐ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐØ"Ð%:Ñ:ˆ
à"×6Ñ6°|ÓDÐØÐ"5Ñ5ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐrO   c                 óæ   — | j                  «       dd }|d   }t        j                  |dz   ||z   dz   t        j                  | j                  ¬«      }|j                  d«      j                  |«      S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr/   r%   rW   r   )rG   rB   rC   rH   rX   Ú	unsqueezerE   )rQ   r*   r_   Úsequence_lengthr-   s        rN   rZ   z5XmodEmbeddings.create_position_ids_from_inputs_embedsz   sp   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a™.ˆä—|‘|Ø˜!‰O˜_¨{Ñ:¸QÑ>ÄeÇjÁjÐYf×YmÑYmô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<rO   c                 ó¾   — | j                  |«      j                  «       }t        j                  |d¬«      j	                  |«      |z   |z  }|j                  «       |z   S )a  
        Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
        are ignored. This is modified from fairseq's `utils.make_positions`.

        Args:
            x: torch.Tensor x:

        Returns: torch.Tensor
        r%   ©rU   )ÚneÚintrB   ÚcumsumÚtype_asrH   )rP   r*   rR   ÚmaskÚincremental_indicess        rN   rY   z1XmodEmbeddings.create_position_ids_from_input_idsŒ   sW   € ð �|‰|˜KÓ(×,Ñ,Ó.ˆÜ$Ÿ|™|¨D°aÔ8×@Ñ@ÀÓFÐI_Ñ_ÐcgÑgÐØ"×'Ñ'Ó)¨KÑ7Ð7rO   )NNNNr   )r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r4   rB   Ú
LongTensorÚFloatTensorrk   ÚTensorrd   ÚstaticmethodrZ   rY   Ú__classcell__©rM   s   @rN   r(   r(   3   s´   ø„ ÙQô
ð, .2Ø26Ø04Ø26Ø&'ñ.à×#Ñ# dÑ*ð.ð ×(Ñ(¨4Ñ/ð.ð ×&Ñ&¨Ñ-ð	.ð
 ×(Ñ(¨4Ñ/ð.ð !$ð.ð 
�‰ó.ð` ñ=ó ð=ð" ò8ó ô8rO   r(   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingr@   Úkwargsc                 óœ  — |€|j                  d«      dz  }t        j                  ||j                  dd«      «      |z  }|�||z   }t        j
                  j                  |d¬«      }t        j
                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Nr/   ç      à¿é   r   ri   )ÚpÚtrainingr%   )
rG   rB   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr@   r…   Ú
contiguous)
rz   r{   r|   r}   r~   r   r@   r€   Úattn_weightsÚattn_outputs
             rN   Úeager_attention_forwardr�   ž   s»   € ð €Ø—*‘*˜R“. DÑ(ˆô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$rO   c                   ó–   ‡ — e Zd Zd	ˆ fd„	Z	 	 d
dej
                  dej                  dz  dedz  dee	   de
ej
                     f
d„Zˆ xZS )ÚXmodSelfAttentionNc                 ó@  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|| _        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _	        | j                  dz  | _
        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                   |j"                  «      | _        |j&                  | _        || _        || _        y ©Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)r‚   )r3   r4   r7   Únum_attention_headsr[   Ú
ValueErrorrL   rk   Úattention_head_sizeÚall_head_sizer   r   ÚLinearr{   r|   r}   r>   Úattention_probs_dropout_probr@   Ú
is_decoderÚ	is_causalÚ	layer_idx©rK   rL   r›   rœ   rM   s       €rN   r4   zXmodSelfAttention.__init__¼   sP  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð ˆŒà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ×/Ñ/°Ñ5ˆŒä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒà ×+Ñ+ˆŒØ"ˆŒØ"ˆ�rO   Úhidden_statesr~   Úpast_key_valuesr€   rS   c                 ó   — |j                   d d }g |¢d‘| j                  ‘­} | j                  |«      j                  |Ž j	                  dd«      } | j                  |«      j                  |Ž j	                  dd«      } | j                  |«      j                  |Ž j	                  dd«      }	|�>|}
t        |t        «      r|j                  }
|
j                  ||	| j                  «      \  }}	t        j                  | j                  j                  t         «      } || |||	|f| j"                  sdn| j$                  j&                  | j(                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }||fS )Nr/   r%   rƒ   ç        ©r@   r   )r]   r–   r{   Úviewr‡   r|   r}   Ú
isinstancer   Úself_attention_cacheÚupdaterœ   r   Úget_interfacerL   Ú_attn_implementationr�   r…   r@   r„   r   ÚreshaperŠ   )rK   rž   r~   rŸ   r€   r_   Úhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerÚcurrent_past_key_valuesÚattention_interfacerŒ   r‹   s                 rN   rd   zXmodSelfAttention.forwardÔ   s�  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆð 5�d—j‘j Ó/×4Ñ4°lÐC×MÑMÈaÐQRÓSˆØ0�D—H‘H˜]Ó+×0Ñ0°,Ð?×IÑIÈ!ÈQÓOˆ	Ø4�d—j‘j Ó/×4Ñ4°lÐC×MÑMÈaÐQRÓSˆàÐ&à&5Ð#Ü˜/Ô+>Ô?Ø*9×*NÑ*NÐ'ð &=×%CÑ%CÀIÈ{Ð\`×\jÑ\jÓ%kÑ"ˆI�{ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$·,±,·.±.Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ˜LÐ(Ð(rO   ©FN)NN)rp   rq   rr   r4   rB   rv   ru   r   r   r   Útuplerd   rx   ry   s   @rN   r�   r�   »   sg   ø„ õ#ð6 48Ø(,ñ	')à—|‘|ð')ð ×)Ñ)¨DÑ0ð')ð  ™ð	')ð
 Ð+Ñ,ð')ð 
ˆu�|‰|Ñ	÷')rO   r�   c                   ó¶   ‡ — e Zd Zd
ˆ fd„	Z	 	 	 ddej
                  dej                  dz  dej                  dz  dedz  dee	   de
ej
                     fd	„Zˆ xZS )ÚXmodCrossAttentionNc                 ó  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|| _        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _	        | j                  dz  | _
        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                   |j"                  «      | _        || _        || _        y r‘   )r3   r4   r7   r”   r[   r•   rL   rk   r–   r—   r   r   r˜   r{   r|   r}   r>   r™   r@   r›   rœ   r�   s       €rN   r4   zXmodCrossAttention.__init__   sC  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð ˆŒà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ×/Ñ/°Ñ5ˆŒä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒà"ˆŒØ"ˆ�rO   rž   Úencoder_hidden_statesr~   rŸ   r€   rS   c                 óf  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }|�%|j
                  j                  | j                  «      nd}	|�]|	r[|j                  j                  | j                     j                  }
|j                  j                  | j                     j                  }nÅg |j                   d d ¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      j                  |«      j	                  dd«      }|�C|j                  j                  |
|| j                  «      \  }
}d|j
                  | j                  <   t        j                   | j"                  j$                  t&        «      } || ||
||f| j(                  sdn| j*                  j,                  | j.                  dœ|¤Ž\  }} |j0                  g |¢d‘­Ž j3                  «       }||fS )Nr/   r%   rƒ   FTr¡   r¢   )r]   r–   r{   r£   r‡   Ú
is_updatedÚgetrœ   Úcross_attention_cacheÚlayersÚkeysÚvaluesr|   r}   r¦   r   r§   rL   r¨   r�   r…   r@   r„   r   r©   rŠ   )rK   rž   rµ   r~   rŸ   r€   r_   rª   r«   r·   r¬   r­   Úkv_shaper¯   rŒ   r‹   s                   rN   rd   zXmodCrossAttention.forward  s  € ð $×)Ñ)¨#¨2Ð.ˆàC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆàGVÐGb�_×/Ñ/×3Ñ3°D·N±NÔCÐhmˆ
ØÐ&©:à'×=Ñ=×DÑDÀTÇ^Á^ÑT×YÑYˆIØ)×?Ñ?×FÑFÀtÇ~Á~ÑV×]Ñ]‰KàXÐ.×4Ñ4°S°bÐ9ÐX¸2ÐX¸t×?WÑ?WÑXˆHØŸ™Ð!6Ó7×<Ñ<¸XÓF×PÑPÐQRÐTUÓVˆIØŸ*™*Ð%:Ó;×@Ñ@ÀÓJ×TÑTÐUVÐXYÓZˆKàÐ*à)8×)NÑ)N×)UÑ)UØ˜{¨D¯N©Nó*Ñ&�	˜;ð >B�×*Ñ*¨4¯>©>Ñ:ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$·,±,·.±.Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ˜LÐ(Ð(rO   r°   )NNN)rp   rq   rr   r4   rB   rv   ru   r   r   r   r±   rd   rx   ry   s   @rN   r³   r³   ÿ   s€   ø„ õ#ð4 ;?Ø37Ø6:ñ1)à—|‘|ð1)ð  %×0Ñ0°4Ñ7ð1)ð ×)Ñ)¨DÑ0ð	1)ð
 -¨tÑ3ð1)ð Ð+Ñ,ð1)ð 
ˆu�|‰|Ñ	÷1)rO   r³   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚXmodSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©Nr+   )r3   r4   r   r˜   r7   Údenser<   r=   r>   r?   r@   rJ   s     €rN   r4   zXmodSelfOutput.__init__M  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�rO   rž   Úinput_tensorrS   c                 óT   — | j                  |«      }| j                  |«      }||z   }|S ©N)rÂ   r@   )rK   rž   rÃ   s      rN   rd   zXmodSelfOutput.forwardS  s.   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØ%¨Ñ4ˆØÐrO   ©rp   rq   rr   r4   rB   rv   rd   rx   ry   s   @rN   r¿   r¿   K  s1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ rO   r¿   c                   óö   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  deeej                        dz  dee	   d	eej
                     fd
„Z
ˆ xZS )ÚXmodAttentionNc                 ó®   •— t         ‰| �  «        || _        |rt        nt        } ||||¬«      | _        t        |«      | _        |j                  | _        y )N©r›   rœ   )	r3   r4   Úis_cross_attentionr³   r�   rK   r¿   ÚoutputÚpre_norm)rK   rL   r›   rœ   rË   Úattention_classrM   s         €rN   r4   zXmodAttention.__init__[  sH   ø€ Ü‰ÑÔØ"4ˆÔÙ0BÕ,ÔHYˆÙ# F°iÈ9ÔUˆŒ	Ü$ VÓ,ˆŒàŸ™ˆ�rO   rž   r~   rµ   Úencoder_attention_maskrŸ   r€   rS   c                 ó"  — |}| j                   r| j                  j                  |«      }| j                  s|n|} | j                  |f|||dœ|¤Ž\  }}	| j                  ||«      }| j                   s| j                  j                  |«      }||	fS )N)rµ   r~   rŸ   )rÍ   rÌ   r<   rË   rK   )
rK   rž   r~   rµ   rÏ   rŸ   r€   ÚresidualÚattention_outputr‹   s
             rN   rd   zXmodAttention.forwardd  s¡   € ð !ˆØ�=Š=Ø ŸK™K×1Ñ1°-Ó@ˆMà/3×/FÒ/F™ÐLbˆØ)2¨¯©Øð*
à"7Ø)Ø+ñ	*
ð
 ñ*
Ñ&Ð˜,ð  Ÿ;™;Ð'7¸ÓBÐà�}Š}Ø#Ÿ{™{×4Ñ4Ð5EÓFÐà Ð-Ð-rO   )FNF©NNNN)rp   rq   rr   r4   rB   rv   ru   r±   r   r   rd   rx   ry   s   @rN   rÈ   rÈ   Z  s¨   ø„ õ(ð 48Ø:>Ø;?ØBFñ.à—|‘|ð.ð ×)Ñ)¨DÑ0ð.ð  %×0Ñ0°4Ñ7ð	.ð
 !&× 1Ñ 1°DÑ 8ð.ð ˜u U×%6Ñ%6Ñ7Ñ8¸4Ñ?ð.ð Ð+Ñ,ð.ð 
ˆu�|‰|Ñ	÷.rO   rÈ   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚXmodIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y rÅ   )r3   r4   r   r˜   r7   Úintermediate_sizerÂ   r¤   Ú
hidden_actÚstrr
   Úintermediate_act_fnrJ   s     €rN   r4   zXmodIntermediate.__init__ƒ  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$rO   rž   rS   c                 óJ   — | j                  |«      }| j                  |«      }|S rÅ   )rÂ   rÚ   ©rK   rž   s     rN   rd   zXmodIntermediate.forward‹  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐrO   rÆ   ry   s   @rN   rÕ   rÕ   ‚  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ rO   rÕ   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚXmodAdapterc                 ó¢  •— t         ‰| �  «        |j                  |j                  z  | _        t        j                  |j                  | j                  «      | _        t        j                  | j                  |j                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y rÅ   )r3   r4   r7   Úadapter_reduction_factorÚbottleneck_sizer   r˜   Údense1Údense2r¤   rØ   rÙ   r
   Úadapter_act_fnrJ   s     €rN   r4   zXmodAdapter.__init__’  s”   ø€ Ü‰ÑÔØ%×1Ñ1°V×5TÑ5TÑTˆÔÜ—i‘i × 2Ñ 2°D×4HÑ4HÓIˆŒÜ—i‘i × 4Ñ 4°f×6HÑ6HÓIˆŒÜ�f×'Ñ'¬Ô-Ü"(¨×):Ñ):Ñ";ˆDÕà"(×"3Ñ"3ˆDÕrO   rž   rS   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rÅ   )râ   rä   rã   rÜ   s     rN   rd   zXmodAdapter.forwardœ  s4   € ØŸ™ MÓ2ˆØ×+Ñ+¨MÓ:ˆØŸ™ MÓ2ˆØÐrO   rÆ   ry   s   @rN   rÞ   rÞ   ‘  s#   ø„ ô4ð U§\¡\ð °e·l±l÷ rO   rÞ   c                   ó¾   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  dej
                  fd„Zdej
                  dej
                  fd„Zˆ xZS )Ú
XmodOutputc                 óŒ  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        |j                  | _	        t        j                  |j                  «      | _        |j                  r1t        j                  |j
                  |j                  ¬«      | _        nd | _        |j                  | _        t        j                  i «      | _        |j"                  D ]#  }t%        |«      | j                   t'        |«      <   Œ% y rÁ   )r3   r4   r   r˜   r×   r7   rÂ   r<   r=   Úln_before_adapterr>   r?   r@   Úadapter_layer_normÚadapter_reuse_layer_normÚ
ModuleDictÚadapter_modulesÚ	languagesrÞ   rÙ   )rK   rL   ÚlanguagerM   s      €rN   r4   zXmodOutput.__init__¤  sí   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒØ!'×!9Ñ!9ˆÔÜ—z‘z &×"<Ñ"<Ó=ˆŒØ×$Ò$Ü&(§l¡l°6×3EÑ3EÈ6×K`ÑK`Ô&aˆDÕ#à&*ˆDÔ#Ø(.×(GÑ(GˆÔ%Ü!Ÿ}™}¨RÓ0ˆÔØ×(Ñ(ò 	FˆHÜ2=¸fÓ2EˆD× Ñ ¤ X£Ò/ñ	FrO   rž   rÃ   Úlang_idsrS   c                 óx   — | j                  |«      }| j                  |«      }||z   }| j                  ||«      }|S rÅ   )rÂ   r@   Úlang_adapter)rK   rž   rÃ   rð   s       rN   rd   zXmodOutput.forward³  s@   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØ%¨Ñ4ˆØ×)Ñ)¨(°MÓBˆØÐrO   c                 ó¨  — | j                   s|}| j                  �| j                  |«      }n| j                  r| j                  |«      }| j                   r|}t	        j
                  |«      }t        | j                  j                  «       «      D ])  \  }}||k(  }||   } | j                  |   |«      }	|	||<   Œ+ | j                  |«      }|z  }|S rÅ   )
ré   rê   rë   r<   rB   Ú
zeros_likeÚ	enumeraterí   r»   r@   )
rK   rð   rž   rÑ   Únew_hidden_statesÚadapter_idxÚlang_keyÚ	lang_maskÚlang_hidden_statesÚadapted_lang_hidden_statess
             rN   rò   zXmodOutput.lang_adapterº  sæ   € Ø×%Ò%Ø$ˆHà×"Ñ"Ð.Ø ×3Ñ3°MÓB‰MØ×*Ò*Ø ŸN™N¨=Ó9ˆMà×!Ò!Ø$ˆHä!×,Ñ,¨]Ó;ÐÜ%.¨t×/CÑ/C×/HÑ/HÓ/JÓ%Kò 	FÑ!ˆK˜Ø  KÑ/ˆIØ!.¨yÑ!9ÐØ)G¨×)=Ñ)=¸hÑ)GÐHZÓ)[Ð&Ø+EÐ˜iÒ(ð		Fð Ÿ™Ð%6Ó7ˆØ˜Ñ!ˆØÐrO   )	rp   rq   rr   r4   rB   rv   rd   rò   rx   ry   s   @rN   rç   rç   £  s[   ø„ ôFð U§\¡\ð ÀÇÁð ÐY^×YeÑYeð Ðjo×jvÑjvó ð U§\¡\ð À%Ç,Á,÷ rO   rç   c                   ó  ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej
                  dej
                  dej                  dz  dej                  dz  dej                  dz  deeej                        dz  d	ee	   d
ej
                  fd„Z
d„ Zˆ xZS )Ú	XmodLayerNc                 ó¦  •— t         ‰| �  «        |j                  | _        d| _        t	        ||j
                  |¬«      | _        |j
                  | _        |j                  | _        | j                  r.| j
                  st        | › d�«      ‚t	        |d|d¬«      | _	        t        |«      | _        t        |«      | _        |j                  | _        y )Nr%   rÊ   z> should be used as a decoder model if cross attention is addedFT)r›   rœ   rË   )r3   r4   Úchunk_size_feed_forwardÚseq_len_dimrÈ   rš   Ú	attentionÚadd_cross_attentionr•   ÚcrossattentionrÕ   Úintermediaterç   rÌ   rÍ   )rK   rL   rœ   rM   s      €rN   r4   zXmodLayer.__init__Ó  s¸   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ& v¸×9JÑ9JÐV_Ô`ˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#Ò#Ø—?’?Ü  D 6Ð)gÐ!hÓiÐiÜ"/ØØØ#Ø#'ô	#ˆDÔô -¨VÓ4ˆÔÜ  Ó(ˆŒØŸ™ˆ�rO   rž   rð   r~   rµ   rÏ   rŸ   r€   rS   c                 óæ  —  | j                   ||fd|i|¤Ž\  }}	|}
| j                  r:|�8t        | d«      st        d| › d�«      ‚ | j                  |
d ||fd|i|¤Ž\  }}	|}
|
}| j
                  r| j                  j                  |
«      }
t        | j                  | j                  | j                  |
«      }| j                  |||«      }| j
                  s| j                  j                  |«      }|S )NrŸ   r  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)r  rš   r[   r•   r  rÍ   rÌ   r<   r   Úfeed_forward_chunkrÿ   r   )rK   rž   rð   r~   rµ   rÏ   rŸ   r€   Úself_attention_outputÚ_rÒ   Úcross_attention_outputrÑ   Úintermediate_outputÚlayer_outputs                  rN   rd   zXmodLayer.forwardç  s8  € ð $2 4§>¡>ØØñ$
ð ,ð$
ð ñ	$
Ñ Ð˜qð 1Ðà�?Š?Ð4Ð@Ü˜4Ð!1Ô2Ü Ø=¸d¸Vð DDð Dóð ð
 )<¨×(;Ñ(;Ø ØØ%Ø&ñ	)ð
 !0ð)ð ñ)Ñ%Ð" Að  6Ðà#ˆØ�=Š=Ø#Ÿ{™{×4Ñ4Ð5EÓFÐÜ7Ø×#Ñ#Ø×(Ñ(Ø×ÑØó	
Ðð —{‘{Ð#6¸À(ÓKˆØ�}Š}ØŸ;™;×0Ñ0°Ó>ˆLàÐrO   c                 ó$   — | j                  |«      S rÅ   )r  )rK   rÒ   s     rN   r  zXmodLayer.feed_forward_chunk  s   € Ø× Ñ Ð!1Ó2Ð2rO   rÅ   rÓ   )rp   rq   rr   r4   rB   rv   ru   r±   r   r   rd   r  rx   ry   s   @rN   rý   rý   Ò  s·   ø„ õ(ð0 48Ø:>Ø;?ØBFñ0à—|‘|ð0ð —,‘,ð0ð ×)Ñ)¨DÑ0ð	0ð
  %×0Ñ0°4Ñ7ð0ð !&× 1Ñ 1°DÑ 8ð0ð ˜u U×%6Ñ%6Ñ7Ñ8¸4Ñ?ð0ð Ð+Ñ,ð0ð 
�‰ó0öd3rO   rý   c                   ó  ‡ — e Zd Zˆ fd„Z	 	 	 	 	 ddej
                  dej
                  dej                  dz  dej                  dz  dej                  dz  deeej                        dz  d	edz  d
e	e
   deej
                     ez  fd„Zˆ xZS )ÚXmodEncoderc           	      ób  •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}«      | _        |j                  | _
        | j                  r1t        j                  |j                  |j                  ¬«      | _        y y c c}w )N)rœ   r+   )r3   r4   rL   r   Ú
ModuleListÚrangeÚnum_hidden_layersrý   ÚlayerrÍ   Úis_pre_normr<   r7   r=   )rK   rL   ÚirM   s      €rN   r4   zXmodEncoder.__init__  s�   ø€ Ü‰ÑÔØˆŒÜ—]‘]ÌEÐRX×RjÑRjÓLkÖ#lÀq¤I¨fÀÖ$BÒ#lÓmˆŒ
Ø!Ÿ?™?ˆÔØ×ÒÜŸ\™\¨&×*<Ñ*<À&×BWÑBWÔXˆD�Nð ùò $ms   ½B,Nrž   rð   r~   rµ   rÏ   rŸ   Ú	use_cacher€   rS   c           	      ó¾   — t        | j                  «      D ]  \  }	}
 |
||||||fi |¤Ž}Œ | j                  r| j                  |«      }t	        ||r|¬«      S d ¬«      S )N)Úlast_hidden_staterŸ   )rõ   r  r  r<   r   )rK   rž   rð   r~   rµ   rÏ   rŸ   r  r€   r  Úlayer_modules              rN   rd   zXmodEncoder.forward&  s�   € ô  )¨¯©Ó4ò 		‰OˆAˆ|Ù(ØØØØ%Ø&Øñð ñ‰Mð		ð ×ÒØ ŸN™N¨=Ó9ˆMä8Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
rO   )NNNNN)rp   rq   rr   r4   rB   rv   ru   r±   Úboolr   r   r   rd   rx   ry   s   @rN   r  r    sÎ   ø„ ôYð 48Ø:>Ø;?ØBFØ!%ñ
à—|‘|ð
ð —,‘,ð
ð ×)Ñ)¨DÑ0ð	
ð
  %×0Ñ0°4Ñ7ð
ð !&× 1Ñ 1°DÑ 8ð
ð ˜u U×%6Ñ%6Ñ7Ñ8¸4Ñ?ð
ð ˜$‘;ð
ð Ð+Ñ,ð
ð 
ˆu�|‰|Ñ	ÐHÑ	H÷
rO   r  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )Ú
XmodPoolerc                 ó²   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  «       | _        y rÅ   )r3   r4   r   r˜   r7   rÂ   ÚTanhÚ
activationrJ   s     €rN   r4   zXmodPooler.__init__G  s9   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�rO   rž   rS   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S ©Nr   )rÂ   r  )rK   rž   Úfirst_token_tensorÚpooled_outputs       rN   rd   zXmodPooler.forwardL  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐrO   rÆ   ry   s   @rN   r  r  F  s#   ø„ ô$ð
 U§\¡\ð °e·l±l÷ rO   r  c                   óˆ   ‡ — e Zd ZeZdZdZg d¢ZdZdZ	dZ
dZeeedœZ ej"                  «       ˆ fd„«       Zdefd„Zd„ Zˆ xZS )	ÚXmodPreTrainedModelÚrobertaT)r(   r�   r³   )rž   Ú
attentionsÚcross_attentionsc                 ó–  •— t         ‰| �  |«       t        |t        «      r t	        j
                  |j                  «       yt        |t        «      ryt	        j                  |j                  t        j                  |j                  j                  d   «      j                  d«      «       t	        j
                  |j                  «       yy)zInitialize the weightsr/   r.   N)r3   Ú_init_weightsr¤   Ú
XmodLMHeadÚinitÚzeros_Úbiasr(   Úcopy_r-   rB   rC   r]   rE   r1   )rK   rz   rM   s     €rN   r*  z!XmodPreTrainedModel._init_weightse  sˆ   ø€ ô 	‰Ñ˜fÔ%Ü�fœjÔ)Ü�K‰K˜Ÿ™Õ$Ü˜¤Ô/Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÔhÜ�K‰K˜×-Ñ-Õ.ð 0rO   rï   c           	      ó¶   — || j                   j                  vr0t        | › d|› dt        | j                   j                  «      › �«      ‚|| j                   _        y)zÜ
        Set the default language code for the model. This is used when the language is not specified in the input.

        Args:
            language (`str`): The language code, such as `"en_XX"` or `"de_DE"`.
        z does not have an adapter for z. Supported languages: N)rL   rî   r•   ÚlistÚdefault_language)rK   rï   s     rN   Úset_default_languagez(XmodPreTrainedModel.set_default_languageo  s[   € ð ˜4Ÿ;™;×0Ñ0Ñ0ÜØ�&Ð6°x°jÐ@WÔX\Ð]a×]hÑ]h×]rÑ]rÓXsÐWtÐuóð ð (0ˆ�‰Õ$rO   c                 óî  — t         j                  d«       | j                  j                  j	                  «       D ]	  }d|_        Œ t         j                  d«       | j                  j                  j                  D ]x  }|j                  j                  �0|j                  j                  j	                  «       D ]	  }d|_        Œ |j                  j                  j	                  «       D ]	  }d|_        Œ Œz y)z¡
        Freeze the embeddings and language adapters of the model. Usually, this is applied before the model is
        fine-tuned on a downstream task.
        zFreezing embeddingsFzFreezing adaptersN)ÚloggerÚinfor&  rc   Ú
parametersÚrequires_gradÚencoderr  rÌ   rê   rí   )rK   Ú	parameterr  s      rN   Ú'freeze_embeddings_and_language_adaptersz;XmodPreTrainedModel.freeze_embeddings_and_language_adapters|  sË   € ô
 	�‰Ð)Ô*ØŸ™×0Ñ0×;Ñ;Ó=ò 	,ˆIØ&+ˆIÕ#ð	,ä�‰Ð'Ô(Ø—\‘\×)Ñ)×/Ñ/ò 	0ˆEØ�|‰|×.Ñ.Ð:Ø!&§¡×!@Ñ!@×!KÑ!KÓ!Mò 4�IØ.3�IÕ+ð4à"Ÿ\™\×9Ñ9×DÑDÓFò 0�	Ø*/�	Õ'ñ0ñ		0rO   )rp   rq   rr   r&   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚno_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrý   r�   r³   Ú_can_record_outputsrB   Úno_gradr*  rÙ   r3  r;  rx   ry   s   @rN   r%  r%  U  so   ø„ à€LØ!ÐØ&*Ð#ÚTÐØÐØ€NØÐØ"&Ðà"Ø'Ø.ñÐð €U‡]�]ƒ_ó/ó ð/ð0¨Só 0ö0rO   r%  a0  
    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in *Attention is
    all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
    Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.

    .. _*Attention is all you need*: https://huggingface.co/papers/1706.03762
    )Úcustom_introc                   óº  ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zeee	 	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  dee	j                     dz  dedz  dee   dee	j                     ez  fd„«       «       «       Zd„ Zˆ xZS )Ú	XmodModelc                 óÈ   •— t         ‰| �  |«       || _        d| _        t	        |«      | _        t        |«      | _        |rt        |«      nd| _	        | j                  «        y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        FN)r3   r4   rL   Úgradient_checkpointingr(   rc   r  r9  r  ÚpoolerÚ	post_init)rK   rL   Úadd_pooling_layerrM   s      €rN   r4   zXmodModel.__init__œ  sU   ø€ ô
 	‰Ñ˜Ô ØˆŒØ&+ˆÔ#ä(¨Ó0ˆŒÜ" 6Ó*ˆŒá,=”j Ô(À4ˆŒð 	�‰ÕrO   c                 ó.   — | j                   j                  S rÅ   ©rc   r9   ©rK   s    rN   Úget_input_embeddingszXmodModel.get_input_embeddings®  s   € Ø�‰×.Ñ.Ð.rO   c                 ó&   — || j                   _        y rÅ   rO  )rK   r}   s     rN   Úset_input_embeddingszXmodModel.set_input_embeddings²  s   € Ø*/ˆ�‰Õ'rO   NrP   rð   r~   r1   r-   rQ   rµ   rÏ   rŸ   r  r€   rS   c                 óP  — |du |duz  rt        d«      ‚| j                  j                  r|
�|
n| j                  j                  }
nd}
|
rd|	€b|€| j                  j                  r4t        t        | j                  ¬«      t        | j                  ¬«      «      nt        | j                  ¬«      }	|�|j                  d   n|j                  d   }|�|j                  n|j                  }|	�|	j                  «       nd}|€¤| j                  j                  €t        d«      ‚t        | j                  j                  d   j                  j                  j!                  «       «      }|j#                  | j                  j                  «      }|t%        j&                  ||¬«      z  }| j)                  |||||¬«      }| j+                  |||||	¬	«      \  }} | j                  |f|||||	|
|d
œ|¤Ž}|d   }| j,                  �| j-                  |«      nd}t/        |||j0                  ¬«      S )á  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        Nz:You must specify exactly one of input_ids or inputs_embedsF)rL   r   zPInput language unknown. Please call `XmodPreTrainedModel.set_default_language()`)rX   )rP   r-   r1   rQ   rR   )r~   rÏ   Úembedding_outputrµ   rŸ   )rð   r~   rµ   rÏ   rŸ   r  r-   )r  Úpooler_outputrŸ   )r•   rL   rš   r  Úis_encoder_decoderr   r   r]   rX   Úget_seq_lengthr2  r1  r9  r  rÌ   rí   r»   rV   rB   Úonesrc   Ú_create_attention_masksrK  r   rŸ   )rK   rP   rð   r~   r1   r-   rQ   rµ   rÏ   rŸ   r  r€   r`   rX   rR   Úadapter_languagesÚdefault_lang_idrV  Úencoder_outputsÚsequence_outputr#  s                        rN   rd   zXmodModel.forwardµ  s5  € ð, ˜Ð -°tÐ";Ò<ÜÐYÓZÐZà�;‰;×!Ò!Ø%.Ð%:™	ÀÇÁ×@UÑ@U‰IàˆIá˜Ð0ð )Ð4¸¿¹×8VÒ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð ,5Ð+@�Y—_‘_ QÒ'Àm×FYÑFYÐZ[ÑF\ˆ
Ø%.Ð%:�×!Ò!À×@TÑ@TˆØETÐE` ×!?Ñ!?Ô!AÐfgÐàÐØ�{‰{×+Ñ+Ð3Ü Ð!sÓtÐtÜ $ T§\¡\×%7Ñ%7¸Ñ%:×%AÑ%A×%QÑ%Q×%VÑ%VÓ%XÓ YÐØ/×5Ñ5°d·k±k×6RÑ6RÓSˆOØ&¬¯©°JÀvÔ)NÑNˆHàŸ?™?ØØ%Ø)Ø'Ø#9ð +ó 
Ðð 26×1MÑ1MØ)Ø#9Ø-Ø"7Ø+ð 2Nó 2
Ñ.ˆÐ.ð '˜$Ÿ,™,Øð

àØ)Ø"7Ø#9Ø+ØØ%ñ

ð ñ

ˆð *¨!Ñ,ˆØ8<¿¹Ð8O˜Ÿ™ OÔ4ÐUYˆä;Ø-Ø'Ø+×;Ñ;ô
ð 	
rO   c                 óÐ   — | j                   j                  rt        | j                   |||¬«      }nt        | j                   ||¬«      }|�t        | j                   |||¬«      }||fS )N)rL   rQ   r~   rŸ   )rL   rQ   r~   )rL   rQ   r~   rµ   )rL   rš   r   r   )rK   r~   rÏ   rV  rµ   rŸ   s         rN   r[  z!XmodModel._create_attention_masks
  su   € ð �;‰;×!Ò!Ü/Ø—{‘{Ø.Ø-Ø /ô	‰Nô 7Ø—{‘{Ø.Ø-ôˆNð "Ð-Ü%>Ø—{‘{Ø.Ø5Ø&;ô	&Ð"ð Ð5Ð5Ð5rO   )T)
NNNNNNNNNN)rp   rq   rr   r4   rQ  rS  r#   r$   r    rB   rv   rt   r1  ru   r  r   r   r±   r   rd   r[  rx   ry   s   @rN   rH  rH  �  sS  ø„ õò$/ò0ð  ØØð *.Ø,0Ø.2Ø.2Ø,0Ø-1Ø59Ø6:Ø:>Ø!%ñO
à—<‘< $Ñ&ðO
ð ×"Ñ" TÑ)ðO
ð Ÿ™ tÑ+ð	O
ð
 Ÿ™ tÑ+ðO
ð —l‘l TÑ)ðO
ð —|‘| dÑ*ðO
ð  %Ÿ|™|¨dÑ2ðO
ð !&§¡¨tÑ 3ðO
ð ˜e×/Ñ/Ñ0°4Ñ7ðO
ð ˜$‘;ðO
ð Ð+Ñ,ðO
ð 
ˆu�|‰|Ñ	ÐKÑ	KòO
ó ó ó  ðO
öd6rO   rH  zQ
    X-MOD Model with a `language modeling` head on top for CLM fine-tuning.
    c                    óø  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  deee	j                        dz  dedz  dee	j                  z  dee   dee	j                     ez  fd„«       «       Zˆ xZS )ÚXmodForCausalLMú)roberta.embeddings.word_embeddings.weightúlm_head.bias©zlm_head.decoder.weightzlm_head.decoder.biasc                 óÊ   •— t         ‰| �  |«       |j                  st        j	                  d«       t        |d¬«      | _        t        |«      | _        | j                  «        y )NzLIf you want to use `XmodLMHeadModel` as a standalone, add `is_decoder=True.`F©rM  ©
r3   r4   rš   r5  ÚwarningrH  r&  r+  Úlm_headrL  rJ   s     €rN   r4   zXmodForCausalLM.__init__7  sL   ø€ Ü‰Ñ˜Ô à× Ò Ü�N‰NÐiÔjä  ¸5ÔAˆŒÜ! &Ó)ˆŒð 	�‰ÕrO   c                 ó.   — | j                   j                  S rÅ   ©rj  ÚdecoderrP  s    rN   Úget_output_embeddingsz%XmodForCausalLM.get_output_embeddingsD  ó   € Ø�|‰|×#Ñ#Ð#rO   c                 ó&   — || j                   _        y rÅ   rl  ©rK   Únew_embeddingss     rN   Úset_output_embeddingsz%XmodForCausalLM.set_output_embeddingsH  ó   € Ø-ˆ�‰ÕrO   NrP   rð   r~   r1   r-   rQ   rµ   rÏ   ÚlabelsrŸ   r  Úlogits_to_keepr€   rS   c                 ó   — |	�d} | j                   |f||||||||
|ddœ
|¤Ž}|j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      }d}|	�* | j                  d||	| j                  j                  dœ|¤Ž}t        |||j                  |j                  |j                  |j                  ¬«      S )aS  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Example:

        ```python
        >>> from transformers import AutoTokenizer, XmodForCausalLM, AutoConfig
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-base")
        >>> config = AutoConfig.from_pretrained("facebook/xmod-base")
        >>> config.is_decoder = True
        >>> model = XmodForCausalLM.from_pretrained("facebook/xmod-base", config=config)
        >>> model.set_default_language("en_XX")

        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> prediction_logits = outputs.logits
        ```NFT)
rð   r~   r1   r-   rQ   rµ   rÏ   rŸ   r  Úreturn_dict)Úlogitsru  r6   )Úlossry  rŸ   rž   r'  r(  © )r&  r  r¤   rk   Úslicerj  Úloss_functionrL   r6   r   rŸ   rž   r'  r(  )rK   rP   rð   r~   r1   r-   rQ   rµ   rÏ   ru  rŸ   r  rv  r€   Úoutputsrž   Úslice_indicesry  rz  s                      rN   rd   zXmodForCausalLM.forwardK  s   € ðX ÐØˆIà@LÀÇÁØðA
àØ)Ø)Ø%Ø'Ø"7Ø#9Ø+ØØñA
ð ñA
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
rO   )NNNNNNNNNNNr   )rp   rq   rr   Ú_tied_weights_keysr4   rn  rs  r"   r    rB   rt   ru   r±   r  rk   rv   r   r   r   rd   rx   ry   s   @rN   rb  rb  +  s™  ø„ ð #NØ .ñÐô
ò$ò.ð Øð .2Ø,0Ø37Ø26Ø04Ø26Ø:>Ø;?Ø*.ØBFØ!%Ø-.ñL
à×#Ñ# dÑ*ðL
ð ×"Ñ" TÑ)ðL
ð ×)Ñ)¨DÑ0ð	L
ð
 ×(Ñ(¨4Ñ/ðL
ð ×&Ñ&¨Ñ-ðL
ð ×(Ñ(¨4Ñ/ðL
ð  %×0Ñ0°4Ñ7ðL
ð !&× 1Ñ 1°DÑ 8ðL
ð × Ñ  4Ñ'ðL
ð ˜u U×%6Ñ%6Ñ7Ñ8¸4Ñ?ðL
ð ˜$‘;ðL
ð ˜eŸl™lÑ*ðL
ð Ð+Ñ,ðL
ð 
ˆu�|‰|Ñ	Ð@Ñ	@òL
ó ó ôL
rO   rb  c                   ó   ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 dde	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  dee   dee	j                     ez  fd„«       «       Zˆ xZS )ÚXmodForMaskedLMrc  rd  re  c                 óÊ   •— t         ‰| �  |«       |j                  rt        j	                  d«       t        |d¬«      | _        t        |«      | _        | j                  «        y )NzkIf you want to use `XmodForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Frg  rh  rJ   s     €rN   r4   zXmodForMaskedLM.__init__¤  sR   ø€ Ü‰Ñ˜Ô à×ÒÜ�N‰Nð1ôô
 ! ¸5ÔAˆŒÜ! &Ó)ˆŒð 	�‰ÕrO   c                 ó.   — | j                   j                  S rÅ   rl  rP  s    rN   rn  z%XmodForMaskedLM.get_output_embeddings´  ro  rO   c                 ó&   — || j                   _        y rÅ   rl  rq  s     rN   rs  z%XmodForMaskedLM.set_output_embeddings¸  rt  rO   NrP   rð   r~   r1   r-   rQ   rµ   rÏ   ru  r€   rS   c
                 ó@  —  | j                   |f|||||||ddœ|
¤Ž}|d   }| j                  |«      }d}|	�Ft        «       } ||j                  d| j                  j
                  «      |	j                  d«      «      }t        |||j                  |j                  ¬«      S )a¶  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        T)rð   r~   r1   r-   rQ   rµ   rÏ   rx  r   Nr/   ©rz  ry  rž   r'  )	r&  rj  r   r£   rL   r6   r   rž   r'  )rK   rP   rð   r~   r1   r-   rQ   rµ   rÏ   ru  r€   r~  r_  Úprediction_scoresÚmasked_lm_lossÚloss_fcts                   rN   rd   zXmodForMaskedLM.forward»  sÁ   € ð0 �$—,‘,Øð
àØ)Ø)Ø%Ø'Ø"7Ø#9Øñ
ð ñ
ˆð " !™*ˆØ ŸL™L¨Ó9ÐàˆØÐÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rO   )	NNNNNNNNN)rp   rq   rr   r€  r4   rn  rs  r"   r    rB   rt   ru   r   r   r±   rv   r   rd   rx   ry   s   @rN   r‚  r‚  œ  s?  ø„ ð #NØ .ñÐôò $ò.ð Øð .2Ø,0Ø37Ø26Ø04Ø26Ø:>Ø;?Ø*.ñ/
à×#Ñ# dÑ*ð/
ð ×"Ñ" TÑ)ð/
ð ×)Ñ)¨DÑ0ð	/
ð
 ×(Ñ(¨4Ñ/ð/
ð ×&Ñ&¨Ñ-ð/
ð ×(Ñ(¨4Ñ/ð/
ð  %×0Ñ0°4Ñ7ð/
ð !&× 1Ñ 1°DÑ 8ð/
ð × Ñ  4Ñ'ð/
ð Ð+Ñ,ð/
ð 
ˆu�|‰|Ñ	˜~Ñ	-ò/
ó ó ô/
rO   r‚  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r+  z*Roberta Head for masked language modeling.c                 ó¬  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  «      | _
        t        j                  t        j                  |j                  «      «      | _        y rÁ   )r3   r4   r   r˜   r7   rÂ   r<   r=   Ú
layer_normr6   rm  Ú	ParameterrB   rF   r.  rJ   s     €rN   r4   zXmodLMHead.__init__ó  s‡   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒä—y‘y ×!3Ñ!3°V×5FÑ5FÓGˆŒÜ—L‘L¤§¡¨V×->Ñ->Ó!?Ó@ˆ�	rO   c                 ó‚   — | j                  |«      }t        |«      }| j                  |«      }| j                  |«      }|S rÅ   )rÂ   r   r�  rm  ©rK   Úfeaturesr€   Úxs       rN   rd   zXmodLMHead.forwardû  s;   € Ø�J‰J�xÓ ˆÜ�‹GˆØ�O‰O˜AÓˆð �L‰L˜‹OˆàˆrO   ©rp   rq   rr   rs   r4   rd   rx   ry   s   @rN   r+  r+  ð  s   ø„ Ù4ôAörO   r+  z�
    X-MOD Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                   óJ  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	e
   deej                     ez  fd„«       «       Zˆ xZS )ÚXmodForSequenceClassificationc                 ó¸   •— t         ‰| �  |«       |j                  | _        || _        t	        |d¬«      | _        t        |«      | _        | j                  «        y ©NFrg  )	r3   r4   Ú
num_labelsrL   rH  r&  ÚXmodClassificationHeadÚ
classifierrL  rJ   s     €rN   r4   z&XmodForSequenceClassification.__init__  sJ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØˆŒä  ¸5ÔAˆŒÜ0°Ó8ˆŒð 	�‰ÕrO   NrP   rð   r~   r1   r-   rQ   ru  r€   rS   c           
      óÀ  —  | j                   |f|||||ddœ|¤Ž}	|	d   }
| j                  |
«      }d}|��‡| j                  j                  €�| j                  dk(  rd| j                  _        nl| j                  dkD  rL|j
                  t        j                  k(  s|j
                  t        j                  k(  rd| j                  _        nd| j                  _        | j                  j                  dk(  rIt        «       }| j                  dk(  r& ||j                  «       |j                  «       «      }nŒ |||«      }n‚| j                  j                  dk(  r=t        «       } ||j                  d	| j                  «      |j                  d	«      «      }n,| j                  j                  dk(  rt        «       } |||«      }t        |||	j                  |	j                   ¬
«      S )a¡  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        T©rð   r~   r1   r-   rQ   rx  r   Nr%   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr/   r‡  )r&  rš  rL   Úproblem_typer˜  r2   rB   rH   rk   r   Úsqueezer   r£   r   r   rž   r'  ©rK   rP   rð   r~   r1   r-   rQ   ru  r€   r~  r_  ry  rz  rŠ  s                 rN   rd   z%XmodForSequenceClassification.forward  sž  € ð, �$—,‘,Øð	
àØ)Ø)Ø%Ø'Øñ	
ð ñ	
ˆð " !™*ˆØ—‘ Ó1ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rO   ©NNNNNNN)rp   rq   rr   r4   r"   r    rB   rt   ru   r   r   r±   rv   r   rd   rx   ry   s   @rN   r•  r•    sõ   ø„ ô	ð Øð .2Ø,0Ø37Ø26Ø04Ø26Ø*.ñ=
à×#Ñ# dÑ*ð=
ð ×"Ñ" TÑ)ð=
ð ×)Ñ)¨DÑ0ð	=
ð
 ×(Ñ(¨4Ñ/ð=
ð ×&Ñ&¨Ñ-ð=
ð ×(Ñ(¨4Ñ/ð=
ð × Ñ  4Ñ'ð=
ð Ð+Ñ,ð=
ð 
ˆu�|‰|Ñ	Ð7Ñ	7ò=
ó ó ô=
rO   r•  c                   óJ  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	e
   deej                     ez  fd„«       «       Zˆ xZS )ÚXmodForMultipleChoicec                 óö   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  «      | _        t	        j                  |j                  d«      | _
        | j                  «        y )Nr%   )r3   r4   rH  r&  r   r>   r?   r@   r˜   r7   rš  rL  rJ   s     €rN   r4   zXmodForMultipleChoice.__init__^  sV   ø€ Ü‰Ñ˜Ô ä  Ó(ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÕrO   NrP   rð   r1   r~   ru  r-   rQ   r€   rS   c           
      óZ  — |�|j                   d   n|j                   d   }	|�!|j                  d|j                  d«      «      nd}
|�2|j                  |j                  d«      |j                  d«      z  «      nd}|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�1|j                  d|j                  d«      |j                  d«      «      nd} | j                  |
f|||||ddœ|¤Ž}|d   }| j                  |«      }| j                  |«      }|j                  d|	«      }d}|�t        «       } |||«      }t        |||j                  |j                  ¬«      S )	a|  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        lang_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nr%   r/   r   éþÿÿÿT)rð   r-   r1   r~   rQ   rx  r‡  )r]   r£   rG   Úrepeatr&  r@   rš  r   r   rž   r'  )rK   rP   rð   r1   r~   ru  r-   rQ   r€   Únum_choicesÚflat_input_idsÚflat_lang_idsÚflat_position_idsÚflat_token_type_idsÚflat_attention_maskÚflat_inputs_embedsr~  r#  ry  Úreshaped_logitsrz  rŠ  s                         rN   rd   zXmodForMultipleChoice.forwardh  sÑ  € ð\ -6Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆàCLÐCX˜Ÿ™¨¨I¯N©N¸2Ó,>Ô?Ð^bˆØRZÐRf˜Ÿ™¨	¯©°qÓ(9¸I¿N¹NÈ1Ó<MÑ(MÔNÐlpˆØLXÐLd˜L×-Ñ-¨b°,×2CÑ2CÀBÓ2GÔHÐjnÐØR`ÐRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐØR`ÐRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð �$—,‘,Øð	
à"Ø*Ø.Ø.Ø,Øñ	
ð ñ	
ˆð   ™
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rO   r£  )rp   rq   rr   r4   r"   r    rB   rt   ru   r   r   r±   rv   r   rd   rx   ry   s   @rN   r¥  r¥  [  s   ø„ ôð Øð .2Ø,0Ø26Ø37Ø*.Ø04Ø26ñS
à×#Ñ# dÑ*ðS
ð ×"Ñ" TÑ)ðS
ð ×(Ñ(¨4Ñ/ð	S
ð
 ×)Ñ)¨DÑ0ðS
ð × Ñ  4Ñ'ðS
ð ×&Ñ&¨Ñ-ðS
ð ×(Ñ(¨4Ñ/ðS
ð Ð+Ñ,ðS
ð 
ˆu�|‰|Ñ	Ð8Ñ	8òS
ó ó ôS
rO   r¥  c                   óJ  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	e
   deej                     ez  fd„«       «       Zˆ xZS )ÚXmodForTokenClassificationc                 ód  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        |j
                  �|j
                  n|j                  }t        j                  |«      | _	        t        j                  |j                  |j                  «      | _        | j                  «        y r—  )r3   r4   r˜  rH  r&  Úclassifier_dropoutr?   r   r>   r@   r˜   r7   rš  rL  ©rK   rL   rµ  rM   s      €rN   r4   z#XmodForTokenClassification.__init__Ã  sŠ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä  ¸5ÔAˆŒà)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z‘zÐ"4Ó5ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrO   NrP   rð   r~   r1   r-   rQ   ru  r€   rS   c           
      óJ  —  | j                   |f|||||ddœ|¤Ž}	|	d   }
| j                  |
«      }
| j                  |
«      }d}|�<t        «       } ||j	                  d| j
                  «      |j	                  d«      «      }t        |||	j                  |	j                  ¬«      S )aï  
        lang_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of the language adapters that should be activated for each sample, respectively. Default: the index
            that corresponds to `self.config.default_language`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Trœ  r   Nr/   r‡  )	r&  r@   rš  r   r£   r˜  r   rž   r'  r¢  s                 rN   rd   z"XmodForTokenClassification.forwardÑ  sº   € ð( �$—,‘,Øð	
àØ)Ø)Ø%Ø'Øñ	
ð ñ	
ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rO   r£  )rp   rq   rr   r4   r"   r    rB   rt   ru   r   r   r±   rv   r   rd   rx   ry   s   @rN   r³  r³  À  sõ   ø„ ôð Øð .2Ø,0Ø37Ø26Ø04Ø26Ø*.ñ,
à×#Ñ# dÑ*ð,
ð ×"Ñ" TÑ)ð,
ð ×)Ñ)¨DÑ0ð	,
ð
 ×(Ñ(¨4Ñ/ð,
ð ×&Ñ&¨Ñ-ð,
ð ×(Ñ(¨4Ñ/ð,
ð × Ñ  4Ñ'ð,
ð Ð+Ñ,ð,
ð 
ˆu�|‰|Ñ	Ð4Ñ	4ò,
ó ó ô,
rO   r³  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r™  z-Head for sentence-level classification tasks.c                 óZ  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        |j                  �|j                  n|j                  }t        j                  |«      | _	        t        j                  |j                  |j                  «      | _        y rÅ   )r3   r4   r   r˜   r7   rÂ   rµ  r?   r>   r@   r˜  Úout_projr¶  s      €rN   r4   zXmodClassificationHead.__init__  s   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
à)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z‘zÐ"4Ó5ˆŒÜŸ	™	 &×"4Ñ"4°f×6GÑ6GÓHˆ�rO   c                 óÐ   — |d d …dd d …f   }| j                  |«      }| j                  |«      }t        j                  |«      }| j                  |«      }| j	                  |«      }|S r!  )r@   rÂ   rB   Útanhrº  r�  s       rN   rd   zXmodClassificationHead.forward  sY   € Ø’Q˜š1�WÑˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆÜ�J‰J�q‹MˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆrO   r“  ry   s   @rN   r™  r™    s   ø„ Ù7ôIörO   r™  c                   ój  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  de	e
   deej                     ez  fd„«       «       Zˆ xZS )ÚXmodForQuestionAnsweringc                 óè   •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r—  )
r3   r4   r˜  rH  r&  r   r˜   r7   Ú
qa_outputsrL  rJ   s     €rN   r4   z!XmodForQuestionAnswering.__init__  sU   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä  ¸5ÔAˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrO   NrP   rð   r~   r1   r-   rQ   Ústart_positionsÚend_positionsr€   rS   c	           
      óÈ  —  | j                   |f|||||ddœ|	¤Ž}
|
d   }| j                  |«      }|j                  dd¬«      \  }}|j                  d«      j	                  «       }|j                  d«      j	                  «       }d}|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   d	z  }t        ||||
j                  |
j                  ¬
«      S )rU  Trœ  r   r%   r/   ri   N)Úignore_indexrƒ   )rz  Ústart_logitsÚ
end_logitsrž   r'  )r&  rÀ  Úsplitr¡  rŠ   ÚlenrG   Úclampr   r   rž   r'  )rK   rP   rð   r~   r1   r-   rQ   rÁ  rÂ  r€   r~  r_  ry  rÅ  rÆ  Ú
total_lossÚignored_indexrŠ  Ú
start_lossÚend_losss                       rN   rd   z XmodForQuestionAnswering.forward&  sƒ  € ð& �$—,‘,Øð	
àØ)Ø)Ø%Ø'Øñ	
ð ñ	
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rO   )NNNNNNNN)rp   rq   rr   r4   r"   r    rB   rt   ru   r   r   r±   rv   r   rd   rx   ry   s   @rN   r¾  r¾    s  ø„ ôð Øð .2Ø,0Ø37Ø26Ø04Ø26Ø37Ø15ñ:
à×#Ñ# dÑ*ð:
ð ×"Ñ" TÑ)ð:
ð ×)Ñ)¨DÑ0ð	:
ð
 ×(Ñ(¨4Ñ/ð:
ð ×&Ñ&¨Ñ-ð:
ð ×(Ñ(¨4Ñ/ð:
ð ×)Ñ)¨DÑ0ð:
ð ×'Ñ'¨$Ñ.ð:
ð Ð+Ñ,ð:
ð 
ˆu�|‰|Ñ	Ð;Ñ	;ò:
ó ó ô:
rO   r¾  )rb  r‚  r¥  r¾  r•  r³  rH  r%  )Nr¡   )Rrs   Úcollections.abcr   rB   r   Útorch.nnr   r   r   Ú r	   r,  Úactivationsr
   r   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r    r!   Úutils.genericr"   r#   Úutils.output_capturingr$   Úconfiguration_xmodr&   Ú
get_loggerrp   r5  ÚModuler(   rv   Úfloatr�   r�   r³   r¿   rÈ   rÕ   rÞ   rç   rý   r  r  r%  rH  rb  r‚  r+  r•  r¥  r³  r™  r¾  Ú__all__r{  rO   rN   ú<module>râ     sä  ðñ å $ã Ý ß AÑ Aå &ß 'ß CÑ CÝ )ß JÝ 9÷	÷ 	ó 	÷ GÝ &Ý 6ß @Ñ @ß IÝ 5Ý *ð 
ˆ×	Ñ	˜HÓ	%€ôg8�R—Y‘Yô g8ðb !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(ó%ô:@)˜Ÿ	™	ô @)ôHI)˜Ÿ™ô I)ôX�R—Y‘Yô ô$.�B—I‘Iô $.ôP�r—y‘yô ô�"—)‘)ô ô$,�—‘ô ,ô^H3Ð*ô H3ôV%
�"—)‘)ô %
ôR�—‘ô ð ô40˜/ó 40ó ð40ñn ðôôM6Ð#ó M6óðM6ñ` ðôô
i
Ð)¨?ó i
óð
i
ðX ôO
Ð)ó O
ó ðO
ôf�—‘ô ñ, ðôôL
Ð$7ó L
óðL
ð^ ôa
Ð/ó a
ó ða
ðH ô>
Ð!4ó >
ó ð>
ôD˜RŸY™Yô ð, ôH
Ð2ó H
ó ðH
òV	�rO   