Ë
    ÜÓ^j÷Ñ  ã                   ó¤  — d Z ddlZddlZddlmZ ddlmZmZmZ ddlm	Z
 ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZmZ ddlmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z" ddl#m$Z$  e"jJ                  e&«      Z'da(d„ Z)d„ Z*dHd„Z+dHd„Z,dHd„Z-d„ Z. G d„ dej^                  j`                  «      Z1 G d„ dej^                  j`                  «      Z2 G d„ d«      Z3dId„Z4d„ Z5	 	 	 dJd„Z6 G d„ d ejn                  «      Z8 G d!„ d"ejn                  «      Z9 G d#„ d$ejn                  «      Z: G d%„ d&ejn                  «      Z; G d'„ d(ejn                  «      Z< G d)„ d*ejn                  «      Z= G d+„ d,e«      Z> G d-„ d.ejn                  «      Z? G d/„ d0ejn                  «      Z@ G d1„ d2ejn                  «      ZA G d3„ d4ejn                  «      ZBe G d5„ d6e«      «       ZCe G d7„ d8eC«      «       ZDe G d9„ d:eC«      «       ZE G d;„ d<ejn                  «      ZF ed=¬>«       G d?„ d@eC«      «       ZGe G dA„ dBeC«      «       ZHe G dC„ dDeC«      «       ZIe G dE„ dFeC«      «       ZJg dG¢ZKy)KzPyTorch MRA model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úapply_chunking_to_forward)Úauto_docstringÚis_cuda_platformÚis_kernels_availableÚis_ninja_availableÚis_torch_cuda_availableÚloggingé   )Ú	MraConfigc                  óN   — t        «       st        d«      ‚ddlm}   | dd¬«      ay )NzFkernels is not installed, please install it with `pip install kernels`r   ©Ú
get_kernelzkernels-community/mrar   )Úversion)r   ÚImportErrorÚintegrations.hub_kernelsr   Úmra_cuda_kernelr   s    úo/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mra/modeling_mra.pyÚload_cuda_kernelsr$   4   s$   € äÔ!ÜÐbÓcÐcÝ6á Ð!8À!ÔD�Oó    c                 óN  — t        | j                  «       «      dk7  rt        d«      ‚t        |j                  «       «      dk7  rt        d«      ‚| j                  d«      dk7  rt        d«      ‚| j                  d«      dk7  rt        d«      ‚| j                  d	¬
«      j                  j                  dd	«      }|j                  «       }|j                  «       }|j                  «       }t        j                  ||||«      \  }}|j                  dd	«      dd…dd…ddd…f   }||fS )z8
    Computes maximum values for softmax stability.
    é   z.sparse_qk_prod must be a 4-dimensional tensor.é   ú'indices must be a 2-dimensional tensor.é    z>The size of the second dimension of sparse_qk_prod must be 32.r   z=The size of the third dimension of sparse_qk_prod must be 32.éþÿÿÿ©ÚdiméÿÿÿÿN)
ÚlenÚsizeÚ
ValueErrorÚmaxÚvaluesÚ	transposeÚ
contiguousÚintr"   Ú	index_max)Úsparse_qk_prodÚindicesÚquery_num_blockÚkey_num_blockÚ
index_valsÚmax_valsÚmax_vals_scatters          r#   Ú
sparse_maxr?   =   s  € ô ˆ>×ÑÓ Ó! QÒ&ÜÐIÓJÐJä
ˆ7�<‰<‹>Ó˜aÒÜÐBÓCÐCà×Ñ˜1Ó Ò#ÜÐYÓZÐZà×Ñ˜1Ó Ò#ÜÐXÓYÐYà×#Ñ#¨Ð#Ó+×2Ñ2×<Ñ<¸RÀÓD€JØ×&Ñ&Ó(€Jà�k‰k‹m€GØ× Ñ Ó"€Gä!0×!:Ñ!:¸:ÀwÐP_ÐanÓ!oÑ€HÐØ'×1Ñ1°"°bÓ9º!ºQÀÂa¸-ÑHÐàÐ%Ð%Ð%r%   c                 ó  — t        | j                  «       «      dk7  rt        d«      ‚t        |j                  «       «      dk7  rt        d«      ‚| j                  d   |j                  d   k7  rt        d«      ‚| j                  \  }}||z  }t	        j
                  |j                  d«      t        j                  |j                  ¬«      }| j                  |||«      } | |dd…df   ||z  j                  «       dd…f   } | S )zN
    Converts attention mask to a sparse mask for high resolution logits.
    r(   z$mask must be a 2-dimensional tensor.r)   r   zBmask and indices must have the same size in the zero-th dimension.©ÚdtypeÚdeviceN)	r/   r0   r1   ÚshapeÚtorchÚarangeÚlongrC   Úreshape)Úmaskr9   Ú
block_sizeÚ
batch_sizeÚseq_lenÚ	num_blockÚ	batch_idxs          r#   Úsparse_maskrO   Y   sà   € ô ˆ4�9‰9‹;Ó˜1ÒÜÐ?Ó@Ð@ä
ˆ7�<‰<‹>Ó˜aÒÜÐBÓCÐCà‡z�z�!�}˜Ÿ™ aÑ(Ò(ÜÐ]Ó^Ð^àŸ*™*Ñ€J�Ø˜:Ñ%€Iä—‘˜WŸ\™\¨!›_´E·J±JÀwÇ~Á~ÔV€IØ�<‰<˜
 I¨zÓ:€DØ�	š!˜T˜'Ñ" W¨yÑ%8×$>Ñ$>Ó$@Â!ÐCÑD€Dà€Kr%   c                 ój  — | j                  «       \  }}}|j                  «       \  }}}||z  dk7  rt        d«      ‚||z  dk7  rt        d«      ‚| j                  |||z  ||«      j                  dd«      } |j                  |||z  ||«      j                  dd«      }t	        | j                  «       «      dk7  rt        d«      ‚t	        |j                  «       «      dk7  rt        d«      ‚t	        |j                  «       «      d	k7  rt        d
«      ‚| j                  d«      dk7  rt        d«      ‚|j                  d«      dk7  rt        d«      ‚| j                  «       } |j                  «       }|j                  «       }|j                  «       }t        j                  | ||j                  «       «      S )z7
    Performs Sampled Dense Matrix Multiplication.
    r   zTquery_size (size of first dimension of dense_query) must be divisible by block_size.úPkey_size (size of first dimension of dense_key) must be divisible by block_size.r.   r+   r'   z+dense_query must be a 4-dimensional tensor.ú)dense_key must be a 4-dimensional tensor.r(   r)   r   r*   z.The third dimension of dense_query must be 32.z,The third dimension of dense_key must be 32.)	r0   r1   rH   r4   r/   r5   r6   r"   Úmm_to_sparse)	Údense_queryÚ	dense_keyr9   rJ   rK   Ú
query_sizer-   Ú_Úkey_sizes	            r#   rS   rS   p   s¡  € ð #.×"2Ñ"2Ó"4Ñ€J�
˜CØ —~‘~Ó'Ñ€A€x�à�JÑ !Ò#ÜÐoÓpÐpà�*Ñ Ò!ÜÐkÓlÐlà×%Ñ% j°*À
Ñ2JÈJÐX[Ó\×fÑfÐgiÐkmÓn€KØ×!Ñ! *¨h¸*Ñ.DÀjÐRUÓV×`Ñ`ÐacÐegÓh€Iä
ˆ;×ÑÓÓ !Ò#ÜÐFÓGÐGä
ˆ9�>‰>ÓÓ Ò!ÜÐDÓEÐEä
ˆ7�<‰<‹>Ó˜aÒÜÐBÓCÐCà×Ñ˜Ó˜bÒ ÜÐIÓJÐJà‡~�~�aÓ˜BÒÜÐGÓHÐHà×(Ñ(Ó*€KØ×$Ñ$Ó&€Ià�k‰k‹m€GØ× Ñ Ó"€Gä×'Ñ'¨°YÀÇÁÃÓNÐNr%   c                 óB  — |j                  «       \  }}}||z  dk7  rt        d«      ‚| j                  d«      |k7  rt        d«      ‚| j                  d«      |k7  rt        d«      ‚|j                  |||z  ||«      j                  dd«      }t	        | j                  «       «      d	k7  rt        d
«      ‚t	        |j                  «       «      d	k7  rt        d«      ‚t	        |j                  «       «      dk7  rt        d«      ‚|j                  d«      dk7  rt        d«      ‚| j                  «       } |j                  «       }|j                  «       }|j                  «       }t        j                  | |||«      }|j                  dd«      j                  |||z  |«      }|S )zP
    Performs matrix multiplication of a sparse matrix with a dense matrix.
    r   rQ   r(   zQThe size of the second dimension of sparse_query must be equal to the block_size.r   zPThe size of the third dimension of sparse_query must be equal to the block_size.r.   r+   r'   ú,sparse_query must be a 4-dimensional tensor.rR   r)   r*   z8The size of the third dimension of dense_key must be 32.)	r0   r1   rH   r4   r/   r5   r6   r"   Úsparse_dense_mm)	Úsparse_queryr9   rU   r:   rJ   rK   rX   r-   Údense_qk_prods	            r#   r[   r[   ˜   s‘  € ð !*§¡Ó 0Ñ€J�˜#à�*Ñ Ò!ÜÐkÓlÐlà×Ñ˜Ó˜zÒ)ÜÐlÓmÐmà×Ñ˜Ó˜zÒ)ÜÐkÓlÐlà×!Ñ! *¨h¸*Ñ.DÀjÐRUÓV×`Ñ`ÐacÐegÓh€Iä
ˆ<×ÑÓÓ 1Ò$ÜÐGÓHÐHä
ˆ9�>‰>ÓÓ Ò!ÜÐDÓEÐEä
ˆ7�<‰<‹>Ó˜aÒÜÐBÓCÐCà‡~�~�aÓ˜BÒÜÐSÓTÐTà×*Ñ*Ó,€Là�k‰k‹m€GØ× Ñ Ó"€GØ×$Ñ$Ó&€Iä#×3Ñ3°LÀ'È9ÐVeÓf€MØ!×+Ñ+¨B°Ó3×;Ñ;¸JÈÐZdÑHdÐfiÓj€MØÐr%   c                 ó`   — | |z  |z  t        j                  | |d¬«      z   j                  «       S )NÚfloor©Úrounding_mode)rE   ÚdivrG   )r9   Údim_1_blockÚdim_2_blocks      r#   Útranspose_indicesre   À   s.   € Ø�{Ñ" kÑ1´E·I±I¸gÀ{ÐbiÔ4jÑj×pÑpÓrÐrr%   c                   ó>   — e Zd Zed„ «       Zed„ «       Zedd„«       Zy)ÚMraSampledDenseMatMulc                 óV   — t        ||||«      }| j                  |||«       || _        |S ©N)rS   Úsave_for_backwardrJ   )ÚctxrT   rU   r9   rJ   r8   s         r#   ÚforwardzMraSampledDenseMatMul.forwardÅ   s1   € ä% k°9¸gÀzÓRˆØ×Ñ˜k¨9°gÔ>Ø#ˆŒØÐr%   c                 ó  — | j                   \  }}}| j                  }|j                  d«      |z  }|j                  d«      |z  }t        |||«      }t	        |j                  dd«      |||«      }	t	        ||||«      }
|
|	d d fS ©Nr   r.   r+   )Úsaved_tensorsrJ   r0   re   r[   r4   )rk   ÚgradrT   rU   r9   rJ   r:   r;   Ú	indices_TÚgrad_keyÚ
grad_querys              r#   ÚbackwardzMraSampledDenseMatMul.backwardÌ   sŽ   € à*-×*;Ñ*;Ñ'ˆ�Y Ø—^‘^ˆ
Ø%×*Ñ*¨1Ó-°Ñ;ˆØ!Ÿ™ qÓ)¨ZÑ7ˆÜ% g¨ÀÓNˆ	Ü" 4§>¡>°"°bÓ#9¸9ÀkÐS`ÓaˆÜ$ T¨7°I¸ÓOˆ
Ø˜8 T¨4Ð/Ð/r%   c                 ó2   — t         j                  | |||«      S ri   )rg   Úapply)rT   rU   r9   rJ   s       r#   Úoperator_callz#MraSampledDenseMatMul.operator_call×   s   € ä$×*Ñ*¨;¸	À7ÈJÓWÐWr%   N©r*   ©Ú__name__Ú
__module__Ú__qualname__Ústaticmethodrl   rt   rw   © r%   r#   rg   rg   Ä   s>   „ Øñó ðð ñ0ó ð0ð òXó ñXr%   rg   c                   ó<   — e Zd Zed„ «       Zed„ «       Zed„ «       Zy)ÚMraSparseDenseMatMulc                 óV   — t        ||||«      }| j                  |||«       || _        |S ri   )r[   rj   r:   )rk   r\   r9   rU   r:   r8   s         r#   rl   zMraSparseDenseMatMul.forwardÝ   s2   € ä(¨°wÀ	È?Ó[ˆØ×Ñ˜l¨G°YÔ?Ø-ˆÔØÐr%   c                 óü   — | j                   \  }}}| j                  }|j                  d«      |j                  d«      z  }t        |||«      }t	        |j                  dd«      |||«      }t        |||«      }	|	d |d fS rn   )ro   r:   r0   re   r[   r4   rS   )
rk   rp   r\   r9   rU   r:   r;   rq   rr   rs   s
             r#   rt   zMraSparseDenseMatMul.backwardä   s†   € à+.×+<Ñ+<Ñ(ˆ�g˜yØ×-Ñ-ˆØ!Ÿ™ qÓ)¨\×->Ñ->¸rÓ-BÑBˆÜ% g¨ÀÓNˆ	Ü" <×#9Ñ#9¸"¸bÓ#AÀ9ÈdÐTaÓbˆÜ! $¨	°7Ó;ˆ
Ø˜4 ¨4Ð/Ð/r%   c                 ó2   — t         j                  | |||«      S ri   )r€   rv   )r\   r9   rU   r:   s       r#   rw   z"MraSparseDenseMatMul.operator_callî   s   € ä#×)Ñ)¨,¸ÀÈOÓ\Ð\r%   Nry   r~   r%   r#   r€   r€   Ü   s>   „ Øñó ðð ñ0ó ð0ð ñ]ó ñ]r%   r€   c                   ó   — e Zd Zed„ «       Zy)ÚMraReduceSumc                 óB  — | j                  «       \  }}}}t        | j                  «       «      dk7  rt        d«      ‚t        |j                  «       «      dk7  rt        d«      ‚| j                  «       \  }}}}|j                  «       \  }}| j                  d¬«      j	                  ||z  |«      } t        j                  |j                  d«      t
        j                  |j                  ¬«      }t        j                  ||d¬	«      j                  «       |d d …d f   |z  z   j	                  ||z  «      }	t        j                  ||z  |f| j                  | j                  ¬«      }
|
j                  d|	| «      j	                  |||«      }|j	                  |||z  «      }|S )
Nr'   rZ   r(   r)   r,   r   rA   r_   r`   )r0   r/   r1   ÚsumrH   rE   rF   rG   rC   rb   ÚzerosrB   Ú	index_add)r\   r9   r:   r;   rK   rM   rJ   rW   rN   Úglobal_idxesÚtempÚoutputs               r#   rw   zMraReduceSum.operator_callô   sy  € à/;×/@Ñ/@Ó/BÑ,ˆ
�I˜z¨1äˆ|× Ñ Ó"Ó# qÒ(ÜÐKÓLÐLäˆw�|‰|‹~Ó !Ò#ÜÐFÓGÐGà*×/Ñ/Ó1Ñˆˆ1ˆj˜!Ø '§¡£Ñˆ
�Ià#×'Ñ'¨AÐ'Ó.×6Ñ6°zÀIÑ7MÈzÓZˆä—L‘L §¡¨a£¼¿
¹
È7Ï>É>ÔZˆ	ä�I‰I�g˜}¸GÔD×IÑIÓKÈiÒXYÐ[_ÐX_ÑN`ÐcrÑNrÑrß
‰'�*˜yÑ(Ó
)ð 	ô �{‰{Ø˜/Ñ)¨:Ð6¸l×>PÑ>PÐYe×YlÑYlô
ˆð —‘  <°Ó>×FÑFÀzÐSbÐdnÓoˆà—‘ 
¨O¸jÑ,HÓIˆØˆr%   N)rz   r{   r|   r}   rw   r~   r%   r#   r…   r…   ó   s   „ Øñó ñr%   r…   c                 ó&  — | j                  «       \  }}}||z  }d}	|�Â|j                  |||«      j                  d¬«      }
| j                  ||||«      j                  d¬«      |
dd…dd…df   dz   z  }|j                  ||||«      j                  d¬«      |
dd…dd…df   dz   z  }|�×|j                  ||||«      j                  d¬«      |
dd…dd…df   dz   z  }	n¢|t        j                  ||t        j
                  | j                  ¬«      z  }
| j                  ||||«      j                  d¬«      }|j                  ||||«      j                  d¬«      }|�$|j                  ||||«      j                  d¬«      }	t        j                  ||j                  dd«      «      t        j                  |«      z  }|j                  dd¬«      j                  }|�0|d	|
dd…ddd…f   |
dd…dd…df   z  d
k  j                  «       z  z
  }||
||	fS )z/
    Compute low resolution approximation.
    Nr.   r,   r+   ç�íµ ÷Æ°>rA   T)r-   Úkeepdimsç     ˆÃ@g      à?)r0   rH   r‡   rE   ÚonesÚfloatrC   ÚmeanÚmatmulr4   ÚmathÚsqrtr2   r3   )ÚqueryÚkeyrJ   rI   ÚvaluerK   rL   Úhead_dimÚnum_block_per_rowÚ	value_hatÚtoken_countÚ	query_hatÚkey_hatÚlow_resolution_logitÚlow_resolution_logit_row_maxs                  r#   Úget_low_resolution_logitr¢     sN  € ð %*§J¡J£LÑ!€J�˜à :Ñ-Ðà€IØÐØ—l‘l :Ð/@À*ÓM×QÑQÐVXÐQÓYˆØ—M‘M *Ð.?ÀÈXÓV×ZÑZÐ_aÐZÓbØšš1˜d˜
Ñ# dÑ*ñ
ˆ	ð —+‘+˜jÐ*;¸ZÈÓR×VÑVÐ[]ÐVÓ^Øšš1˜d˜
Ñ# dÑ*ñ
ˆð ÐØŸ™ jÐ2CÀZÐQYÓZ×^Ñ^ÐceÐ^ÓfØšAšq $˜JÑ'¨$Ñ.ñ‰Ið !¤5§:¡:¨jÐ:KÔSX×S^ÑS^Ðgl×gsÑgsÔ#tÑtˆØ—M‘M *Ð.?ÀÈXÓV×[Ñ[Ð`bÐ[Ócˆ	Ø—+‘+˜jÐ*;¸ZÈÓR×WÑWÐ\^ÐWÓ_ˆØÐØŸ™ jÐ2CÀZÐQYÓZ×_Ñ_ÐdfÐ_ÓgˆIä Ÿ<™<¨	°7×3DÑ3DÀRÈÓ3LÓMÔPT×PYÑPYÐZbÓPcÑcÐà#7×#;Ñ#;ÀÈTÐ#;Ó#R×#YÑ#YÐ àÐà  3¨;²q¸$Â°zÑ+BÀ[ÒQRÒTUÐW[ÐQ[ÑE\Ñ+\Ð`cÑ*c×)jÑ)jÓ)lÑ#lÑlð 	ð   Ð.JÈIÐUÐUr%   c                 ó¨  — | j                   \  }}}|dkD  rf|dz  }t        j                  ||| j                  ¬«      }	t        j                  t        j
                  |	| ¬«      |¬«      }
| |
ddd…dd…f   dz  z   } |dkD  r:| dd…d|…dd…f   dz   | dd…d|…dd…f<   | dd…dd…d|…f   dz   | dd…dd…d|…f<   t        j                  | j                  |d«      |ddd	¬
«      }|j                  }|dk(  rE|j                  j                  d¬«      j                  }| |dd…ddf   k\  j                  «       }||fS |dk(  rd}||fS t        |› d�«      ‚)zZ
    Compute the indices of the subset of components to be used in the approximation.
    r   r(   ©rC   )ÚdiagonalNg     ˆ³@r.   TF)r-   ÚlargestÚsortedÚfullr,   Úsparsez# is not a valid approx_model value.)rD   rE   r‘   rC   ÚtrilÚtriuÚtopkrH   r9   r3   Úminr’   r1   )r    Ú
num_blocksÚapprox_modeÚinitial_prior_first_n_blocksÚinitial_prior_diagonal_n_blocksrK   Útotal_blocks_per_rowrW   ÚoffsetÚ	temp_maskÚdiagonal_maskÚ
top_k_valsr9   Ú	thresholdÚhigh_resolution_masks                  r#   Úget_block_idxesr¹   8  s¬  € ð +?×*DÑ*DÑ'€JÐ$ aà&¨Ò*Ø0°AÑ5ˆÜ—J‘JÐ3Ð5IÐRf×RmÑRmÔnˆ	ÜŸ
™
¤5§:¡:¨iÀ6À'Ô#JÐU[Ô\ˆØ3°mÀDÊ!ÊQÀJÑ6OÐRUÑ6UÑUÐà# aÒ'à ¢Ð$AÐ%AÐ$AÂ1Ð!DÑEÈÑKð 	šQÐ =Ð!=Ð =ºqÐ@ÑAð !¢¢AÐ'DÐ(DÐ'DÐ!DÑEÈÑKð 	šQ¢Ð#@Ð$@Ð#@Ð@ÑAô —‘Ø×$Ñ$ Z°Ó4°jÀbÐRVÐ_dô€Jð × Ñ €Gà�fÒØ×%Ñ%×)Ñ)¨bÐ)Ó1×8Ñ8ˆ	Ø 4¸	Â!ÀTÈ4À-Ñ8PÑ P×WÑWÓYÐð Ð(Ð(Ð(ð 
˜Ò	 Ø#Ðð Ð(Ð(Ð(ô ˜K˜=Ð(KÐLÓMÐMr%   c	                 óÈ  — t         €#t        j                  | «      j                  «       S | j	                  «       \  }	}
}}|	|
z  }||z  dk7  rt        d«      ‚||z  }| j                  |||«      } |j                  |||«      }|j                  |||«      }|�-| |dd…dd…df   z  } ||dd…dd…df   z  }||dd…dd…df   z  }|dk(  rt        | ||||«      \  }}}}nA|dk(  r1t        j                  «       5  t        | |||«      \  }}}}ddd«       nt        d«      ‚t        j                  «       5  z
  }t        |||||«      \  }}ddd«       t        j                  | ||¬«      t        j                  |«      z  }t        ||||«      \  }}||z
  }|�"|dd	t!        ||«      dd…dd…dd…df   z
  z  z
  }t        j"                  |«      }t$        j                  ||||«      }t&        j                  ||||«      }|dk(  �ryt        j"                  z
  dz  z
  «      dd…ddd…f   z  }t        j(                  |«      dd…dd…ddd…f   j+                  d	d	|d	«      j                  |||«      }|j-                  d
¬«      dd…dd…df   j+                  d	d	|«      j                  ||«      }|j+                  d	d	|«      j                  ||«      |z
  } |�| |z  } t        j"                  | | dk  j/                  «       z  «      }!||!dd…dd…df   z  }||!z  }t        j"                  |  | dkD  j/                  «       z  «      }"||"dd…dd…df   z  }||"z  }||z   |dd…dd…df   |dd…dd…df   z   dz   z  }#n#|dk(  r||dd…dd…df   dz   z  }#nt        d«      ‚|�|#|dd…dd…df   z  }#|#j                  |	|
||«      }#|#S # 1 sw Y   �ŒµxY w# 1 sw Y   �ŒŽxY w)z0
    Use Mra to approximate self-attention.
    Nr   z4sequence length must be divisible by the block_size.r¨   r©   z&approx_mode must be "full" or "sparse")rJ   r�   r   r.   r,   rŽ   z-config.approx_mode must be "full" or "sparse")r"   rE   Ú
zeros_likeÚrequires_grad_r0   r1   rH   r¢   Úno_gradÚ	Exceptionr¹   rg   rw   r•   r–   r?   rO   Úexpr€   r…   r”   Úrepeatr‡   r’   )$r—   r˜   r™   rI   r®   r¯   rJ   r°   r±   rK   Únum_headrL   rš   Ú
meta_batchr›   r    r�   r¡   rœ   rW   Úlow_resolution_logit_normalizedr9   r¸   Úhigh_resolution_logitr=   r>   Úhigh_resolution_attnÚhigh_resolution_attn_outÚhigh_resolution_normalizerÚlow_resolution_attnÚlow_resolution_attn_outÚlow_resolution_normalizerÚlog_correctionÚlow_resolution_corrÚhigh_resolution_corrÚcontext_layers$                                       r#   Úmra2_attentionrÏ   ^  s¹  € ô ÐÜ×Ñ Ó&×5Ñ5Ó7Ð7à.3¯j©j«lÑ+€J�˜' 8Ø˜hÑ&€Jà�Ñ˜qÒ ÜÐOÓPÐPà :Ñ-Ðà�M‰M˜* g¨xÓ8€EØ
�+‰+�j '¨8Ó
4€CØ�M‰M˜* g¨xÓ8€EàÐØ˜šQ¢ 4˜ZÑ(Ñ(ˆØ�DššA˜t˜Ñ$Ñ$ˆØ˜šQ¢ 4˜ZÑ(Ñ(ˆà�fÒÜUmØ�3˜
 D¨%óV
ÑRÐ˜kÐ+GÉð 
˜Ò	 Ü�]‰]‹_ñ 	ÜQiØ�s˜J¨óRÑNÐ  +Ð/KÈQ÷	ð 	ô
 Ð@ÓAÐAä	�‰‹ñ 
Ø*>ÐA]Ñ*]Ð'Ü(7Ø+ØØØ(Ø+ó)
Ñ%ˆÐ%÷
ô 2×?Ñ?Øˆs�G¨
ð @ó ä�	‰	�(ÓñÐô ",Ð,AÀ7ÐL]Ð_pÓ!qÑ€HÐØ1Ð4DÑDÐØÐØ 5¸¸qÄ;ÈtÐU\ÓC]Ò^_ÒabÒdeÐgkÐ^kÑClÑ?lÑ8mÑ mÐÜ Ÿ9™9Ð%:Ó;ÐÜ3×AÑAØ˜g uÐ.?ó Ðô ".×!;Ñ!;Ø˜gÐ'8Ð:Kó"Ðð �fÓä�I‰IÐ*Ð-IÑIÈCÐRfÑLfÑfÓgØš!˜T¢1˜*Ñ%ñ&ð 	ô �L‰LÐ,¨iÓ8ººA¸tÂQ¸ÑGß‰V�A�q˜* aÓ(ß‰W�Z ¨(Ó3ð 	 ð  ×#Ñ#¨Ð#Ó+ªAªq°$¨JÑ7×>Ñ>¸qÀ!ÀZÓP×XÑXÐYcÐelÓmð 	"ð 6×<Ñ<¸QÀÀ:ÓN×VÑVÐWaÐcjÓkÐnvÑvˆØÐØ+¨dÑ2ˆNä#Ÿi™i¨¸.ÈAÑ:M×9TÑ9TÓ9VÑ(VÓWÐØ"9Ð<OÒPQÒSTÐVZÐPZÑ<[Ñ"[ÐØ$=Ð@SÑ$SÐ!ä$Ÿy™y¨.¨¸NÈQÑ<N×;UÑ;UÓ;WÑ)WÓXÐØ#;Ð>RÒSTÒVWÐY]ÐS]Ñ>^Ñ#^Ð Ø%?ÐBVÑ%VÐ"à1Ð4KÑKØ&¢qª!¨T zÑ2Ð5NÊqÒRSÐUYÈzÑ5ZÑZÐ]aÑañ
‰ð 
˜Ò	 Ø0Ð4NÊqÒRSÐUYÈzÑ4ZÐ]aÑ4aÑb‰äÐGÓHÐHàÐØ%¨ªQ²°4¨ZÑ(8Ñ8ˆà!×)Ñ)¨*°hÀÈÓR€MàÐ÷S	ñ 	ú÷
ñ 
ús   Ã7O
Ä3OÏ
OÏO!c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚMraEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óL  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  dz   |j
                  «      | _	        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        | j#                  dt%        j&                  |j                  «      j)                  d«      dz   «       | j#                  dt%        j*                  | j,                  j/                  «       t$        j0                  | j,                  j2                  ¬«      d¬	«       y )
N)Úpadding_idxr(   ©ÚepsÚposition_ids©r   r.   Útoken_type_idsrA   F)Ú
persistent)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferrE   rF   Úexpandrˆ   rÖ   r0   rG   rC   ©ÚselfÚconfigÚ	__class__s     €r#   rÛ   zMraEmbeddings.__init__Ô  s*  ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¡<°×0NÑ0NÐQRÑ0RÐTZ×TfÑTfÓ#gˆÔ Ü%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒð 	×Ñ˜^¬U¯\©\¸&×:XÑ:XÓ-Y×-`Ñ-`ÐahÓ-iÐlmÑ-mÔnØ×ÑØÜ�K‰K˜×)Ñ)×.Ñ.Ó0¼¿
¹
È4×K\ÑK\×KcÑKcÔdØð 	õ 	
r%   c                 ó6  — |�|j                  «       }n|j                  «       d d }|d   }|€| j                  d d …d |…f   }|€st        | d«      r-| j                  d d …d |…f   }|j	                  |d   |«      }|}n:t        j                  |t
        j                  | j                  j                  ¬«      }|€| j                  |«      }| j                  |«      }	||	z   }
| j                  |«      }|
|z  }
| j                  |
«      }
| j                  |
«      }
|
S )Nr.   r   rØ   r   rA   )r0   rÖ   ÚhasattrrØ   rë   rE   rˆ   rG   rC   rà   rä   râ   rå   ré   )rí   Ú	input_idsrØ   rÖ   Úinputs_embedsÚinput_shapeÚ
seq_lengthÚbuffered_token_type_idsÚ buffered_token_type_ids_expandedrä   Ú
embeddingsrâ   s               r#   rl   zMraEmbeddings.forwardå  s/  € ØÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
àÐØ×,Ñ,ªQ°°°¨^Ñ<ˆLð
 Ð!Ü�tÐ-Ô.Ø*.×*=Ñ*=ºaÀÀ*À¸nÑ*MÐ'Ø3J×3QÑ3QÐR]Ð^_ÑR`ÐblÓ3mÐ0Ø!A‘ä!&§¡¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÐ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐØ"Ð%:Ñ:ˆ
à"×6Ñ6°|ÓDÐØÐ)Ñ)ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr%   )NNNN©rz   r{   r|   Ú__doc__rÛ   rl   Ú__classcell__©rï   s   @r#   rÑ   rÑ   Ñ  s   ø„ ÙQô
÷" r%   rÑ   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚMraSelfAttentionc                 ó   •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚t        d u}t        «       r!t        «       rt        «       r|s	 t        «        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t#        j$                  |j                  | j                   «      | _        t#        j$                  |j                  | j                   «      | _        t#        j$                  |j                  | j                   «      | _        t#        j,                  |j.                  «      | _        |j2                  dz  |j4                  z  | _        t9        | j6                  t        |j2                  dz  dz  «      «      | _        |j:                  | _        |j<                  | _        |j>                  | _        y # t        $ r#}t        j                  d|› �«       Y d }~�Œ·d }~ww xY w)	Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)zGCould not load the custom kernel for multi-scale deformable attention: r*   r(   ) rÚ   rÛ   rÞ   Únum_attention_headsrñ   r1   r"   r   r   r   r$   r¾   ÚloggerÚwarningr6   Úattention_head_sizeÚall_head_sizer   ÚLinearr—   r˜   r™   rç   Úattention_probs_dropout_probré   rá   Úblock_per_rowrM   r­   r¯   r°   r±   )rí   rî   Úkernel_loadedÚerï   s       €r#   rÛ   zMraSelfAttention.__init__	  sæ  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ô
 (¨tÐ3ˆÜ"Ô$Ô)9Ô);Ô@RÔ@TÑ]jðnÜ!Ô#ð $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒà ×8Ñ8¸BÑ>À&×BVÑBVÑVˆŒÜ˜TŸ^™^¬S°&×2PÑ2PÐTVÑ2VÐ[\Ñ1\Ó-]Ó^ˆŒà!×-Ñ-ˆÔØ,2×,OÑ,OˆÔ)Ø/5×/UÑ/UˆÕ,øô% ò nÜ—‘Ð!hÐijÐhkÐl×mÒmûðnús   Â
H! È!	IÈ*IÉIc           
      ól  — |j                   \  }}}| j                  |«      j                  |d| j                  | j                  «      j                  dd«      }| j                  |«      j                  |d| j                  | j                  «      j                  dd«      }| j                  |«      j                  |d| j                  | j                  «      j                  dd«      }d|dz  z   }|j                  «       j                  d| j                  d«      j                  || j                  z  |«      j                  «       }d}	| j                  |	k  rÅ|| j                  ||	| j                  z
  f}
t        j                  |t        j                  |
|j                  ¬«      gd¬«      }t        j                  |t        j                  |
|j                  ¬«      gd¬«      }t        j                  |t        j                  |
|j                  ¬«      gd¬«      }t!        |j#                  «       |j#                  «       |j#                  «       |j#                  «       | j$                  | j&                  | j(                  | j*                  ¬	«      }| j                  |	k  r|d d …d d …d d …d | j                  …f   }|j                  || j                  || j                  «      }|j-                  d
ddd«      j/                  «       }|j1                  «       d d | j2                  fz   } |j                  |Ž }|f}|S )Nr.   r   r(   g      ð?r�   r*   r¤   r,   )r¯   r°   r±   r   r   r+   )rD   r—   Úviewr  r  r4   r˜   r™   ÚsqueezerÀ   rH   r6   rE   Úcatrˆ   rC   rÏ   r’   rM   r¯   r°   r±   Úpermuter5   r0   r  )rí   Úhidden_statesÚattention_maskrK   rL   rW   Úquery_layerÚ	key_layerÚvalue_layerÚgpu_warp_sizeÚpad_sizerÎ   Únew_context_layer_shapeÚoutputss                 r#   rl   zMraSelfAttention.forward)  sÎ  € Ø!.×!4Ñ!4Ñˆ
�G˜Qà�J‰J�}Ó%ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �H‰H�]Ó#ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �J‰J�}Ó%ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð ˜~°Ñ7Ñ7ˆà×"Ñ"Ó$ß‰V�A�t×/Ñ/°Ó3ß‰W�Z $×":Ñ":Ñ:¸GÓDß‰S‹Uð	 	ð ˆà×#Ñ# mÒ3Ø! 4×#;Ñ#;¸WÀmÐVZ×VnÑVnÑFnÐnˆHäŸ)™) [´%·+±+¸hÈ{×OaÑOaÔ2bÐ$cÐikÔlˆKÜŸ	™	 9¬e¯k©k¸(È9×K[ÑK[Ô.\Ð"]ÐceÔfˆIÜŸ)™) [´%·+±+¸hÈ{×OaÑOaÔ2bÐ$cÐikÔlˆKä&Ø×ÑÓØ�O‰OÓØ×ÑÓØ× Ñ Ó"Ø�N‰NØ×(Ñ(Ø)-×)JÑ)JØ,0×,PÑ,Pô	
ˆð ×#Ñ# mÒ3Ø)ª!ªQ²Ð3M°T×5MÑ5MÐ3MÐ*MÑNˆMà%×-Ñ-¨j¸$×:RÑ:RÐT[Ð]a×]uÑ]uÓvˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ*˜×*Ñ*Ð,CÐDˆà Ð"ˆàˆr%   ri   ©rz   r{   r|   rÛ   rl   rû   rü   s   @r#   rþ   rþ     s   ø„ ôV÷@<r%   rþ   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚMraSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©NrÔ   )rÚ   rÛ   r   r  rÞ   Údenserå   ræ   rç   rè   ré   rì   s     €r#   rÛ   zMraSelfOutput.__init__j  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r%   r  Úinput_tensorÚreturnc                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S ri   ©r  ré   rå   ©rí   r  r   s      r#   rl   zMraSelfOutput.forwardp  ó7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr%   ©rz   r{   r|   rÛ   rE   ÚTensorrl   rû   rü   s   @r#   r  r  i  ó1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r%   r  c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚMraAttentionc                 ób   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        y ri   )rÚ   rÛ   rþ   rí   r  rŒ   rì   s     €r#   rÛ   zMraAttention.__init__x  s&   ø€ Ü‰ÑÔÜ$ VÓ,ˆŒ	Ü# FÓ+ˆ�r%   c                 óf   — | j                  ||«      }| j                  |d   |«      }|f|dd  z   }|S ©Nr   r   )rí   rŒ   )rí   r  r  Úself_outputsÚattention_outputr  s         r#   rl   zMraAttention.forward}  s@   € Ø—y‘y °Ó?ˆØŸ;™; |°A¡¸ÓFÐØ#Ð%¨°Q°RÐ(8Ñ8ˆØˆr%   ri   r  rü   s   @r#   r*  r*  w  s   ø„ ô,÷
r%   r*  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMraIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y ri   )rÚ   rÛ   r   r  rÞ   Úintermediate_sizer  Ú
isinstanceÚ
hidden_actÚstrr	   Úintermediate_act_fnrì   s     €r#   rÛ   zMraIntermediate.__init__†  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r%   r  r!  c                 óJ   — | j                  |«      }| j                  |«      }|S ri   )r  r7  ©rí   r  s     r#   rl   zMraIntermediate.forwardŽ  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr%   r&  rü   s   @r#   r1  r1  …  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r%   r1  c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )Ú	MraOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y r  )rÚ   rÛ   r   r  r3  rÞ   r  rå   ræ   rç   rè   ré   rì   s     €r#   rÛ   zMraOutput.__init__–  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r%   r  r   r!  c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S ri   r#  r$  s      r#   rl   zMraOutput.forwardœ  r%  r%   r&  rü   s   @r#   r;  r;  •  r(  r%   r;  c                   ó,   ‡ — e Zd Zˆ fd„Zdd„Zd„ Zˆ xZS )ÚMraLayerc                 óÔ   •— t         ‰| �  «        |j                  | _        d| _        t	        |«      | _        |j                  | _        t        |«      | _        t        |«      | _
        y ©Nr   )rÚ   rÛ   Úchunk_size_feed_forwardÚseq_len_dimr*  Ú	attentionÚadd_cross_attentionr1  Úintermediater;  rŒ   rì   s     €r#   rÛ   zMraLayer.__init__¤  sW   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ% fÓ-ˆŒØ#)×#=Ñ#=ˆÔ Ü+¨FÓ3ˆÔÜ Ó'ˆ�r%   c                 ó¢   — | j                  ||«      }|d   }|dd  }t        | j                  | j                  | j                  |«      }|f|z   }|S r-  )rD  r   Úfeed_forward_chunkrB  rC  )rí   r  r  Úself_attention_outputsr/  r  Úlayer_outputs          r#   rl   zMraLayer.forward­  sc   € Ø!%§¡°¸~Ó!NÐØ1°!Ñ4Ðà(¨¨Ð,ˆä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð  �/ GÑ+ˆàˆr%   c                 óL   — | j                  |«      }| j                  ||«      }|S ri   )rF  rŒ   )rí   r/  Úintermediate_outputrJ  s       r#   rH  zMraLayer.feed_forward_chunkº  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐr%   ri   )rz   r{   r|   rÛ   rl   rH  rû   rü   s   @r#   r?  r?  £  s   ø„ ô(óör%   r?  c                   ó,   ‡ — e Zd Zˆ fd„Z	 	 	 dd„Zˆ xZS )Ú
MraEncoderc                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w )NF)
rÚ   rÛ   rî   r   Ú
ModuleListÚrangeÚnum_hidden_layersr?  ÚlayerÚgradient_checkpointing)rí   rî   rW   rï   s      €r#   rÛ   zMraEncoder.__init__Á  sN   ø€ Ü‰ÑÔØˆŒÜ—]‘]¼eÀF×D\ÑD\Ó>]Ö#^¸¤H¨VÕ$4Ò#^Ó_ˆŒ
Ø&+ˆÕ#ùò $_s   ½A#c                 óÊ   — |rdnd }t        | j                  «      D ]  \  }}|r||fz   } |||«      }|d   }Œ |r||fz   }|st        d„ ||fD «       «      S t        ||¬«      S )Nr~   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wri   r~   )Ú.0Úvs     r#   ú	<genexpr>z%MraEncoder.forward.<locals>.<genexpr>Ü  s   è ø€ ÒX˜qÈ!É-œÑXùs   ‚Š)Úlast_hidden_stater  )Ú	enumeraterS  Útupler   )	rí   r  r  Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚiÚlayer_moduleÚlayer_outputss	            r#   rl   zMraEncoder.forwardÇ  s‘   € ñ #7™B¸DÐä(¨¯©Ó4ò 	-‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!á(¨¸ÓGˆMà)¨!Ñ,‰Mð	-ñ  Ø 1°]Ð4DÑ DÐáÜÑX ]Ð4EÐ$FÔXÓXÐXÜ1Ø+Ø+ô
ð 	
r%   )NFTr  rü   s   @r#   rN  rN  À  s   ø„ ô,ð Ø"Ø÷
r%   rN  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMraPredictionHeadTransformc                 óh  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        |j                  t        «      rt        |j                     | _
        n|j                  | _
        t        j                  |j                  |j                  ¬«      | _        y r  )rÚ   rÛ   r   r  rÞ   r  r4  r5  r6  r	   Útransform_act_fnrå   ræ   rì   s     €r#   rÛ   z#MraPredictionHeadTransform.__init__å  s{   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬Ô-Ü$*¨6×+<Ñ+<Ñ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆ�r%   r  r!  c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S ri   )r  rf  rå   r9  s     r#   rl   z"MraPredictionHeadTransform.forwardî  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐr%   r&  rü   s   @r#   rd  rd  ä  s$   ø„ ôUð U§\¡\ð °e·l±l÷ r%   rd  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚMraLMPredictionHeadc                 ó  •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        t	        j                  t        j                  |j                  «      «      | _        y )NT)Úbias)rÚ   rÛ   rd  Ú	transformr   r  rÞ   rÝ   ÚdecoderÚ	ParameterrE   rˆ   rk  rì   s     €r#   rÛ   zMraLMPredictionHead.__init__÷  s[   ø€ Ü‰ÑÔÜ3°FÓ;ˆŒô —y‘y ×!3Ñ!3°V×5FÑ5FÈTÔRˆŒÜ—L‘L¤§¡¨V×->Ñ->Ó!?Ó@ˆ�	r%   c                 óJ   — | j                  |«      }| j                  |«      }|S ri   )rl  rm  r9  s     r#   rl   zMraLMPredictionHead.forward   s$   € ØŸ™ }Ó5ˆØŸ™ ]Ó3ˆØÐr%   r  rü   s   @r#   ri  ri  ö  s   ø„ ôAör%   ri  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMraOnlyMLMHeadc                 óB   •— t         ‰| �  «        t        |«      | _        y ri   )rÚ   rÛ   ri  Úpredictionsrì   s     €r#   rÛ   zMraOnlyMLMHead.__init__  s   ø€ Ü‰ÑÔÜ.¨vÓ6ˆÕr%   Úsequence_outputr!  c                 ó(   — | j                  |«      }|S ri   )rs  )rí   rt  Úprediction_scoress      r#   rl   zMraOnlyMLMHead.forward  s   € Ø ×,Ñ,¨_Ó=ÐØ Ð r%   r&  rü   s   @r#   rq  rq    s#   ø„ ô7ð! u§|¡|ð !¸¿¹÷ !r%   rq  c                   ót   ‡ — e Zd ZU eed<   dZdZ ej                  «       de	j                  fˆ fd„«       Zˆ xZS )ÚMraPreTrainedModelrî   ÚmraTÚmodulec                 óœ  •— t         ‰| �  |«       t        |t        «      r t	        j
                  |j                  «       yt        |t        «      r|t	        j                  |j                  t        j                  |j                  j                  d   «      j                  d«      dz   «       t	        j
                  |j                  «       yy)zInitialize the weightsr.   r×   r(   N)rÚ   Ú_init_weightsr4  ri  ÚinitÚzeros_rk  rÑ   Úcopy_rÖ   rE   rF   rD   rë   rØ   )rí   rz  rï   s     €r#   r|  z MraPreTrainedModel._init_weights  s�   ø€ ô 	‰Ñ˜fÔ%Ü�fÔ1Ô2Ü�K‰K˜Ÿ™Õ$Ü˜¤Ô.Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÐjkÑ,kÔlÜ�K‰K˜×-Ñ-Õ.ð /r%   )rz   r{   r|   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingrE   r½   r   ÚModuler|  rû   rü   s   @r#   rx  rx    s<   ø… ð ÓØÐØ&*Ð#à€U‡]�]ƒ_ð/ B§I¡Iô /ó ô/r%   rx  c                   ó   ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	dz  de	dz  de
ez  fd„«       Zˆ xZS )ÚMraModelc                 ó’   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        | j                  «        y ri   )rÚ   rÛ   rî   rÑ   rø   rN  ÚencoderÚ	post_initrì   s     €r#   rÛ   zMraModel.__init__%  s;   ø€ Ü‰Ñ˜Ô ØˆŒä'¨Ó/ˆŒÜ! &Ó)ˆŒð 	�‰Õr%   c                 ó.   — | j                   j                  S ri   ©rø   rà   ©rí   s    r#   Úget_input_embeddingszMraModel.get_input_embeddings/  s   € Ø�‰×.Ñ.Ð.r%   c                 ó&   — || j                   _        y ri   rŠ  )rí   r™   s     r#   Úset_input_embeddingszMraModel.set_input_embeddings2  s   € Ø*/ˆ�‰Õ'r%   Nrò   r  rØ   rÖ   ró   r]  r^  r!  c                 ó†  — |�|n| j                   j                  }|�|n| j                   j                  }|�|�t        d«      ‚|�#| j	                  ||«       |j                  «       }	n!|�|j                  «       d d }	nt        d«      ‚|	\  }
}|�|j                  n|j                  }|€t        j                  |
|f|¬«      }|€pt        | j                  d«      r4| j                  j                  d d …d |…f   }|j                  |
|«      }|}n&t        j                  |	t        j                  |¬«      }| j                  ||||¬«      }t        | j                   |d d …dd	…d d …f   |d
„ ¬«      }| j!                  ||||¬«      }|d   }|s	|f|d	d  z   S t#        ||j$                  |j&                  |j(                  ¬«      S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer.   z5You have to specify either input_ids or inputs_embedsr¤   rØ   rA   )rò   rÖ   rØ   ró   r   r   c                  óL   — t        j                  dt         j                  ¬«      S )NT)rB   )rE   ÚtensorÚbool)Úargss    r#   ú<lambda>z"MraModel.forward.<locals>.<lambda>j  s   € ¬E¯L©L¸ÄUÇZÁZÔ,P€ r%   )rî   ró   r  Úand_mask_function)r  r]  r^  )rZ  r  Ú
attentionsÚcross_attentions)rî   r]  r^  r1   Ú%warn_if_padding_and_no_attention_maskr0   rC   rE   r‘   rñ   rø   rØ   rë   rˆ   rG   r
   r‡  r   r  r–  r—  )rí   rò   r  rØ   rÖ   ró   r]  r^  Úkwargsrô   rK   rõ   rC   rö   r÷   Úembedding_outputÚencoder_outputsrt  s                     r#   rl   zMraModel.forward5  sñ  € ð %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
�JØ%.Ð%:�×!Ò!À×@TÑ@TˆàÐ!Ü"ŸZ™Z¨*°jÐ)AÈ6ÔRˆNàÐ!Ü�t—‘Ð(8Ô9Ø*.¯/©/×*HÑ*HÊÈKÈZÈKÈÑ*XÐ'Ø3J×3QÑ3QÐR\Ð^hÓ3iÐ0Ø!A‘ä!&§¡¨[ÄÇ
Á
ÐSYÔ!Z�àŸ?™?ØØ%Ø)Ø'ð	 +ó 
Ðô 3Ø—;‘;Ø*ª1¨a°¨c²1¨9Ñ5Ø)áPô
ˆð Ÿ,™,ØØ)Ø!5Ø#ð	 'ó 
ˆð *¨!Ñ,ˆáØ#Ð%¨¸¸Ð(;Ñ;Ð;ä1Ø-Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ô	
ð 	
r%   )NNNNNNN)rz   r{   r|   rÛ   rŒ  rŽ  r   rE   r'  r’  r\  r   rl   rû   rü   s   @r#   r…  r…  #  sÐ   ø„ ôò/ò0ð ð *.Ø.2Ø.2Ø,0Ø-1Ø,0Ø#'ñG
à—<‘< $Ñ&ðG
ð Ÿ™ tÑ+ðG
ð Ÿ™ tÑ+ð	G
ð
 —l‘l TÑ)ðG
ð —|‘| dÑ*ðG
ð # T™kðG
ð ˜D‘[ðG
ð 
Ð3Ñ	3òG
ó ôG
r%   r…  c                   ó*  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  de
dz  de
dz  deez  fd„«       Zˆ xZS )ÚMraForMaskedLMzcls.predictions.biasz%mra.embeddings.word_embeddings.weight)zcls.predictions.decoder.biaszcls.predictions.decoder.weightc                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y ri   )rÚ   rÛ   r…  ry  rq  Úclsrˆ  rì   s     €r#   rÛ   zMraForMaskedLM.__init__‡  s4   ø€ Ü‰Ñ˜Ô ä˜FÓ#ˆŒÜ! &Ó)ˆŒð 	�‰Õr%   c                 óB   — | j                   j                  j                  S ri   )rŸ  rs  rm  r‹  s    r#   Úget_output_embeddingsz$MraForMaskedLM.get_output_embeddings�  s   € Ø�x‰x×#Ñ#×+Ñ+Ð+r%   c                 ó„   — || j                   j                  _        |j                  | j                   j                  _        y ri   )rŸ  rs  rm  rk  )rí   Únew_embeddingss     r#   Úset_output_embeddingsz$MraForMaskedLM.set_output_embeddings“  s,   € Ø'5ˆ�‰×ÑÔ$Ø$2×$7Ñ$7ˆ�‰×ÑÕ!r%   Nrò   r  rØ   rÖ   ró   Úlabelsr]  r^  r!  c	           	      ó–  — |�|n| j                   j                  }| j                  |||||||¬«      }
|
d   }| j                  |«      }d}|�Ft	        «       } ||j                  d| j                   j                  «      |j                  d«      «      }|s|f|
dd z   }|�|f|z   S |S t        |||
j                  |
j                  ¬«      S )a£  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N©r  rØ   rÖ   ró   r]  r^  r   r.   r   ©ÚlossÚlogitsr  r–  )
rî   r^  ry  rŸ  r   r  rÝ   r   r  r–  )rí   rò   r  rØ   rÖ   ró   r¥  r]  r^  r™  r  rt  rv  Úmasked_lm_lossÚloss_fctrŒ   s                   r#   rl   zMraForMaskedLM.forward—  sô   € ð& &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—(‘(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ó 
ˆð " !™*ˆØ ŸH™H _Ó5ÐàˆØÐÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNáØ'Ð)¨G°A°B¨KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r%   ©NNNNNNNN)rz   r{   r|   Ú_tied_weights_keysrÛ   r¡  r¤  r   rE   r'  r’  r\  r   rl   rû   rü   s   @r#   r�  r�  €  së   ø„ ð )?Ø*QñÐô
ò,ò8ð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø,0Ø#'ñ/
à—<‘< $Ñ&ð/
ð Ÿ™ tÑ+ð/
ð Ÿ™ tÑ+ð	/
ð
 —l‘l TÑ)ð/
ð —|‘| dÑ*ð/
ð —‘˜tÑ#ð/
ð # T™kð/
ð ˜D‘[ð/
ð 
�Ñ	ò/
ó ô/
r%   r�  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚMraClassificationHeadz-Head for sentence-level classification tasks.c                 ó4  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _
        || _        y ri   )rÚ   rÛ   r   r  rÞ   r  rç   rè   ré   Ú
num_labelsÚout_projrî   rì   s     €r#   rÛ   zMraClassificationHead.__init__Î  sg   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆŒÜŸ	™	 &×"4Ñ"4°f×6GÑ6GÓHˆŒàˆ�r%   c                 óê   — |d d …dd d …f   }| j                  |«      }| j                  |«      }t        | j                  j                     |«      }| j                  |«      }| j                  |«      }|S )Nr   )ré   r  r	   rî   r5  r³  )rí   Úfeaturesr™  Úxs       r#   rl   zMraClassificationHead.forwardÖ  se   € Ø’Q˜š1�WÑˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆÜ�4—;‘;×)Ñ)Ñ*¨1Ó-ˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆr%   rù   rü   s   @r#   r°  r°  Ë  s   ø„ Ù7ôör%   r°  z›
    MRA Model transformer with a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks.
    )Úcustom_introc                   ó  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dee	z  fd„«       Z
ˆ xZS )ÚMraForSequenceClassificationc                 ó¦   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        |«      | _        | j                  «        y ri   )rÚ   rÛ   r²  r…  ry  r°  Ú
classifierrˆ  rì   s     €r#   rÛ   z%MraForSequenceClassification.__init__ç  sA   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ˜FÓ#ˆŒÜ/°Ó7ˆŒð 	�‰Õr%   Nrò   r  rØ   rÖ   ró   r¥  r]  r^  r!  c	           	      ó  — |�|n| j                   j                  }| j                  |||||||¬«      }
|
d   }| j                  |«      }d}|��‡| j                   j                  €�| j
                  dk(  rd| j                   _        nl| j
                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt        «       }| j
                  dk(  r& ||j                  «       |j                  «       «      }nŒ |||«      }n‚| j                   j                  dk(  r=t        «       } ||j                  d| j
                  «      |j                  d«      «      }n,| j                   j                  dk(  rt        «       } |||«      }|s|f|
dd z   }|�|f|z   S |S t        |||
j                   |
j"                  ¬	«      S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr§  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr.   r¨  )rî   r^  ry  r»  Úproblem_typer²  rB   rE   rG   r6   r   r  r   r  r   r   r  r–  )rí   rò   r  rØ   rÖ   ró   r¥  r]  r^  r™  r  rt  rª  r©  r¬  rŒ   s                   r#   rl   z$MraForSequenceClassification.forwardð  sÔ  € ð& &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—(‘(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ó 
ˆð " !™*ˆØ—‘ Ó1ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ÙØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r%   r­  )rz   r{   r|   rÛ   r   rE   r'  r’  r\  r   rl   rû   rü   s   @r#   r¹  r¹  à  sÝ   ø„ ôð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø,0Ø#'ñ@
à—<‘< $Ñ&ð@
ð Ÿ™ tÑ+ð@
ð Ÿ™ tÑ+ð	@
ð
 —l‘l TÑ)ð@
ð —|‘| dÑ*ð@
ð —‘˜tÑ#ð@
ð # T™kð@
ð ˜D‘[ð@
ð 
Ð)Ñ	)ò@
ó ô@
r%   r¹  c                   ó  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dee	z  fd„«       Z
ˆ xZS )ÚMraForMultipleChoicec                 ó  •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  «      | _        t	        j
                  |j                  d«      | _        | j                  «        y rA  )
rÚ   rÛ   r…  ry  r   r  rÞ   Úpre_classifierr»  rˆ  rì   s     €r#   rÛ   zMraForMultipleChoice.__init__6  s_   ø€ Ü‰Ñ˜Ô ä˜FÓ#ˆŒÜ Ÿi™i¨×(:Ñ(:¸F×<NÑ<NÓOˆÔÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õr%   Nrò   r  rØ   rÖ   ró   r¥  r]  r^  r!  c	           	      óŽ  — |�|n| j                   j                  }|�|j                  d   n|j                  d   }
|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�1|j                  d|j	                  d«      |j	                  d«      «      nd}| j                  |||||||¬«      }|d   }|dd…df   }| j                  |«      } t        j                  «       |«      }| j                  |«      }|j                  d|
«      }d}|�t        «       } |||«      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r.   r+   r§  r   r¨  )rî   r^  rD   r  r0   ry  rÄ  r   ÚReLUr»  r   r   r  r–  )rí   rò   r  rØ   rÖ   ró   r¥  r]  r^  r™  Únum_choicesr  Úhidden_stateÚpooled_outputrª  Úreshaped_logitsr©  r¬  rŒ   s                      r#   rl   zMraForMultipleChoice.forward@  sú  € ðV &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ,5Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÐG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —(‘(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ó 
ˆð ˜q‘zˆØ$¢Q¨ TÑ*ˆØ×+Ñ+¨MÓ:ˆØ!œŸ™›	 -Ó0ˆØ—‘ Ó/ˆà Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDáØ%Ð'¨'°!°"¨+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r%   r­  )rz   r{   r|   rÛ   r   rE   r'  r’  r\  r   rl   rû   rü   s   @r#   rÂ  rÂ  4  sÝ   ø„ ôð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø,0Ø#'ñW
à—<‘< $Ñ&ðW
ð Ÿ™ tÑ+ðW
ð Ÿ™ tÑ+ð	W
ð
 —l‘l TÑ)ðW
ð —|‘| dÑ*ðW
ð —‘˜tÑ#ðW
ð # T™kðW
ð ˜D‘[ðW
ð 
Ð*Ñ	*òW
ó ôW
r%   rÂ  c                   ó  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dee	z  fd„«       Z
ˆ xZS )ÚMraForTokenClassificationc                 ó,  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y ri   )rÚ   rÛ   r²  r…  ry  r   rç   rè   ré   r  rÞ   r»  rˆ  rì   s     €r#   rÛ   z"MraForTokenClassification.__init__�  si   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä˜FÓ#ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr%   Nrò   r  rØ   rÖ   ró   r¥  r]  r^  r!  c	           	      óÀ  — |�|n| j                   j                  }| j                  |||||||¬«      }
|
d   }| j                  |«      }| j	                  |«      }d}|�Êt        «       }|�Œ|j                  d«      dk(  }|j                  d| j                  «      }t        j                  ||j                  d«      t        j                  |j                  «      j                  |«      «      } |||«      }n2 ||j                  d| j                  «      |j                  d«      «      }|s|f|
dd z   }|�|f|z   S |S t        |||
j                  |
j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr§  r   r.   r   r¨  )rî   r^  ry  ré   r»  r   r  r²  rE   Úwherer‘  Úignore_indexÚtype_asr   r  r–  )rí   rò   r  rØ   rÖ   ró   r¥  r]  r^  r™  r  rt  rª  r©  r¬  Úactive_lossÚactive_logitsÚactive_labelsrŒ   s                      r#   rl   z!MraForTokenClassification.forward¨  se  € ð" &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—(‘(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ó 
ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHàÐ)Ø,×1Ñ1°"Ó5¸Ñ:�Ø &§¡¨B°·±Ó @�Ü %§¡Ø §¡¨R£´%·,±,¸x×?TÑ?TÓ2U×2]Ñ2]Ð^dÓ2eó!�ñ   ¨}Ó=‘á §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR�áØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r%   r­  )rz   r{   r|   rÛ   r   rE   r'  r’  r\  r   rl   rû   rü   s   @r#   rÌ  rÌ  ›  sÒ   ø„ ô	ð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø,0Ø#'ñ8
à—<‘< $Ñ&ð8
ð Ÿ™ tÑ+ð8
ð Ÿ™ tÑ+ð	8
ð
 —l‘l TÑ)ð8
ð —|‘| dÑ*ð8
ð —‘˜tÑ#ð8
ð # T™kð8
ð ˜D‘[ð8
ð 
Ð&Ñ	&ò8
ó ô8
r%   rÌ  c                   ó4  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
edz  dedz  dee	z  fd„«       Z
ˆ xZS )ÚMraForQuestionAnsweringc                 óò   •— t         ‰| �  |«       d|_        |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y )Nr(   )
rÚ   rÛ   r²  r…  ry  r   r  rÞ   Ú
qa_outputsrˆ  rì   s     €r#   rÛ   z MraForQuestionAnswering.__init__æ  s[   ø€ Ü‰Ñ˜Ô àˆÔØ ×+Ñ+ˆŒä˜FÓ#ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr%   Nrò   r  rØ   rÖ   ró   Ústart_positionsÚend_positionsr]  r^  r!  c
           	      óì  — |	�|	n| j                   j                  }	| j                  |||||||	¬«      }|d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      }|j                  d«      }d }|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }|	s||f|dd  z   }|�|f|z   S |S t        ||||j                  |j                  ¬«      S )	Nr§  r   r   r.   r,   )rÐ  r(   )r©  Ústart_logitsÚ
end_logitsr  r–  )rî   r^  ry  rØ  Úsplitr  r/   r0   Úclampr   r   r  r–  )rí   rò   r  rØ   rÖ   ró   rÙ  rÚ  r]  r^  r™  r  rt  rª  rÜ  rÝ  Ú
total_lossÚignored_indexr¬  Ú
start_lossÚend_lossrŒ   s                         r#   rl   zMraForQuestionAnswering.forwardò  sª  € ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—(‘(ØØ)Ø)Ø%Ø'Ø!5Ø#ð ó 
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/ˆØ×'Ñ'¨Ó+ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r%   )	NNNNNNNNN)rz   r{   r|   rÛ   r   rE   r'  r’  r\  r   rl   rû   rü   s   @r#   rÖ  rÖ  ä  sè   ø„ ô
ð ð *.Ø.2Ø.2Ø,0Ø-1Ø/3Ø-1Ø,0Ø#'ñ;
à—<‘< $Ñ&ð;
ð Ÿ™ tÑ+ð;
ð Ÿ™ tÑ+ð	;
ð
 —l‘l TÑ)ð;
ð —|‘| dÑ*ð;
ð Ÿ™¨Ñ,ð;
ð —|‘| dÑ*ð;
ð # T™kð;
ð ˜D‘[ð;
ð 
Ð-Ñ	-ò;
ó ô;
r%   rÖ  )r�  rÂ  rÖ  r¹  rÌ  r?  r…  rx  rx   )NN)r*   r   r   )Lrú   r•   rE   r   Útorch.nnr   r   r   Ú r   r}  Úactivationsr	   Úmasking_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   r   r   Úconfiguration_mrar   Ú
get_loggerrz   r  r"   r$   r?   rO   rS   r[   re   ÚautogradÚFunctionrg   r€   r…   r¢   r¹   rÏ   rƒ  rÑ   rþ   r  r*  r1  r;  r?  rN  rd  ri  rq  rx  r…  r�  r°  r¹  rÂ  rÌ  rÖ  Ú__all__r~   r%   r#   ú<module>rò     s‰  ðñ ã ã Ý ß AÑ Aå &Ý !Ý 6Ý 9÷÷ õ .Ý 6÷÷ õ )ð 
ˆ×	Ñ	˜HÓ	%€à€òEò&ó8ó.%OóP%òPsôX˜EŸN™N×3Ñ3ô Xô0]˜5Ÿ>™>×2Ñ2ô ]÷.ñ ó:%VòP#)ðZ Ø!"Ø$%ópôf4�B—I‘Iô 4ôn]�r—y‘yô ]ôB�B—I‘Iô ô
�2—9‘9ô 
ô�b—i‘iô ô �—	‘	ô ôÐ)ô ô: 
�—‘ô  
ôH §¡ô ô$˜"Ÿ)™)ô ô"!�R—Y‘Yô !ð ô/˜ó /ó ð/ð  ôY
Ð!ó Y
ó ðY
ðx ôF
Ð'ó F
ó ðF
ôT˜BŸI™Iô ñ* ðôôK
Ð#5ó K
óðK
ð\ ôc
Ð-ó c
ó ðc
ðL ôE
Ð 2ó E
ó ðE
ðP ôI
Ð0ó I
ó ðI
òX	�r%   