Ë
    ÜÓ^j×³  ã                   óR  — d Z ddlZddlZddlmZ ddlmZmZmZ ddlm	Z
 ddlmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lmZ ddlmZ ddlmZmZmZmZmZ ddl m!Z!  ejD                  e#«      Z$da%d„ Z&d„ Z'd„ Z(d„ Z) G d„ dejT                  jV                  «      Z, G d„ dejT                  jV                  «      Z- G d„ dej\                  «      Z/ G d„ dej\                  «      Z0 G d„ dej\                  «      Z1 G d„ dej\                  «      Z2 G d„ d ej\                  «      Z3 G d!„ d"ej\                  «      Z4 G d#„ d$e«      Z5 G d%„ d&ej\                  «      Z6 G d'„ d(ej\                  «      Z7 G d)„ d*ej\                  «      Z8 G d+„ d,ej\                  «      Z9e G d-„ d.e«      «       Z:e G d/„ d0e:«      «       Z;e G d1„ d2e:«      «       Z< G d3„ d4ej\                  «      Z= ed5¬6«       G d7„ d8e:«      «       Z>e G d9„ d:e:«      «       Z?e G d;„ d<e:«      «       Z@e G d=„ d>e:«      «       ZAg d?¢ZBy)@zPyTorch YOSO model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úapply_chunking_to_forward)Úauto_docstringÚis_kernels_availableÚis_ninja_availableÚis_torch_cuda_availableÚloggingé   )Ú
YosoConfigc                  óf   — t        «       st        d«      ‚ddlm}   | dd¬«      }|j                  ay )NzFkernels is not installed, please install it with `pip install kernels`r   )Ú
get_kernelzkernels-community/yosor   )Úversion)r   ÚImportErrorÚintegrations.hub_kernelsr   Úlsh_cumulation)r   Úyosos     úq/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/yoso/modeling_yoso.pyÚload_cuda_kernelsr"   3   s/   € äÔ!ÜÐbÓcÐcÝ6áÐ.¸Ô:€DØ×(Ñ(�Nó    c                 óÞ   — t        | t        «      r<g }| D ]3  }|j                  «       s|j                  «       }|j	                  |«       Œ5 |S | j                  «       s| j                  «       } | S ©N)Ú
isinstanceÚlistÚis_contiguousÚ
contiguousÚappend©Úinput_tensorsÚoutÚtensors      r!   Úto_contiguousr/   =   sm   € Ü�-¤Ô&ØˆØ#ò 	ˆFØ×'Ñ'Ô)Ø×*Ñ*Ó,�Ø�J‰J�vÕð	ð ˆ
à×*Ñ*Ô,Ø)×4Ñ4Ó6ˆMØÐr#   c                 óÞ   — t        | t        «      r<g }| D ]3  }|j                  t        j                  j                  |dd¬«      «       Œ5 |S t        j                  j                  | dd¬«      S )Né   éÿÿÿÿ)ÚpÚdim)r&   r'   r*   r   Ú
functionalÚ	normalizer+   s      r!   r6   r6   K   se   € Ü�-¤Ô&ØˆØ#ò 	EˆFØ�J‰J”r—}‘}×.Ñ.¨v¸ÀÐ.ÓCÕDð	Eàˆ
ä�}‰}×&Ñ& }¸¸rÐ&ÓBÐBr#   c                 óz  — t        | j                  «       «      dk7  rt        d«      ‚t        |j                  «       «      dk7  rt        d«      ‚t        j                  | j                  d«      | j                  d«      ||z  | j
                  ¬«      }dt        j                  || j
                  ¬«      z  }t        j                  | |«      j                  | j                  d«      | j                  d«      ||«      }t        j                  ||«      j                  |j                  d«      |j                  d«      ||«      }|dkD  j                  «       }|dkD  j                  «       }	t        j                  ||z  d¬	«      }
t        j                  |	|z  d¬	«      }
|
j                  «       |
j                  «       fS )
Nr   zQuery has incorrect size.zKey has incorrect size.r   r1   ©Údevicer   r2   ©r4   )ÚlenÚsizeÚ
ValueErrorÚtorchÚrandnr9   ÚarangeÚmatmulÚreshapeÚintÚsum)ÚqueryÚkeyÚnum_hashÚhash_lenÚrmatÚ	raise_powÚquery_projectionÚkey_projectionÚquery_binaryÚ
key_binaryÚ
query_hashs              r!   ÚhashingrP   U   sX  € Ü
ˆ5�:‰:‹<Ó˜AÒÜÐ4Ó5Ð5Ü
ˆ3�8‰8‹:ƒ˜!ÒÜÐ2Ó3Ð3ä�;‰;�u—z‘z !“} e§j¡j°£m°XÀÑ5HÐQV×Q]ÑQ]Ô^€DØ”U—\‘\ (°5·<±<Ô@Ñ@€Iä—|‘| E¨4Ó0×8Ñ8¸¿¹ÀA»ÈÏ
É
ÐSTËÐW_ÐaiÓjÐÜ—\‘\ # tÓ,×4Ñ4°S·X±X¸a³[À#Ç(Á(È1Ã+ÈxÐYaÓb€NØ$ qÑ(×-Ñ-Ó/€LØ  1Ñ$×)Ñ)Ó+€JÜ—‘˜<¨)Ñ3¸Ô<€JÜ—‘˜:¨	Ñ1°rÔ:€Jà�>‰>Ó˜ZŸ^™^Ó-Ð-Ð-r#   c                   ó,   — e Zd Zed„ «       Zed„ «       Zy)ÚYosoCumulationc           
      óN  — |d   }dt        j                  t        j                  ||j                  dd«      «      «      t        j
                  z  z
  |z  }||d d …d d …d f   z  |d d …d d d …f   z  }t        j                  ||«      }	| j                  ||||||«       || _        |	S )NÚhash_code_lenr   r2   éþÿÿÿ)r>   ÚacosrA   Ú	transposeÚmathÚpiÚsave_for_backwardÚconfig)
ÚctxÚ
query_maskÚkey_maskrE   rF   Úvaluer[   rT   ÚexpectationÚcumulation_values
             r!   ÚforwardzYosoCumulation.forwardi   s¢   € à˜Ñ/ˆàœ5Ÿ:™:¤e§l¡l°5¸#¿-¹-ÈÈBÓ:OÓ&PÓQÔTX×T[ÑT[Ñ[Ñ[Ð`mÑmˆØ! Jªq²!°T¨zÑ$:Ñ:¸XÂaÈÊqÀjÑ=QÑQˆÜ Ÿ<™<¨°UÓ;Ðà×Ñ˜j¨(°KÀÈÈUÔSØˆŒ
àÐr#   c                 óž  — t        |«      }| j                  \  }}}}}}| j                  }|d   }	t        j                  ||j                  dd«      «      |z  }
t        j                  |
|	dz  |z  «      }t        j                  |
j                  dd«      |	dz  |z  «      }t        j                  |j                  dd«      |«      }d d |||d fS )NrT   r2   rU   r1   )r/   Úsaved_tensorsr[   r>   rA   rW   )r\   Úgradr]   r^   r`   rE   rF   r_   r[   rT   Úweighted_expÚ
grad_queryÚgrad_keyÚ
grad_values                 r!   ÚbackwardzYosoCumulation.backwardv   sÊ   € ä˜TÓ"ˆà?B×?PÑ?PÑ<ˆ
�H˜k¨5°#°uØ—‘ˆà˜Ñ/ˆä—|‘| D¨%¯/©/¸"¸bÓ*AÓBÀ[ÑPˆÜ—\‘\ ,°ÀÑ1BÀcÑ0IÓJˆ
Ü—<‘< × 6Ñ 6°r¸2Ó >ÀÐQRÑARÐV[Ñ@[Ó\ˆÜ—\‘\ +×"7Ñ"7¸¸BÓ"?ÀÓFˆ
à�T˜: x°¸TÐAÐAr#   N©Ú__name__Ú
__module__Ú__qualname__Ústaticmethodrb   rj   © r#   r!   rR   rR   h   s*   „ Øñ
 ó ð
 ð ñBó ñBr#   rR   c                   ó,   — e Zd Zed„ «       Zed„ «       Zy)ÚYosoLSHCumulationc           
      ó‚  — |j                  d«      |j                  d«      k7  rt        d«      ‚|j                  d«      |j                  d«      k7  rt        d«      ‚|j                  d«      |j                  d«      k7  rt        d«      ‚|j                  d«      |j                  d«      k7  rt        d«      ‚|j                  d«      |j                  d«      k7  rt        d«      ‚|j                  d«      |j                  d«      k7  rt        d	«      ‚t        |||||g«      \  }}}}}|j                  }|d
   }|d   }	t	        d|	z  «      }
|d   r t
        j                  ||||||	|d«      \  }}nt        ||||	«      \  }}t
        j                  ||||||
|d«      }| j                  |||||||«       || _	        |S )Nr   z6Query mask and Key mask differ in sizes in dimension 0z3Query mask and Query differ in sizes in dimension 0z1Query mask and Key differ in sizes in dimension 0z8Query mask and Value mask differ in sizes in dimension 0r   z,Key and Value differ in sizes in dimension 1r1   z,Query and Key differ in sizes in dimension 2rG   rT   Úuse_fast_hash)
r<   r=   r/   Úis_cudarC   r   Ú	fast_hashrP   rZ   r[   )r\   r]   r^   rE   rF   r_   r[   Úuse_cudarG   rT   Úhashtable_capacityÚquery_hash_codeÚkey_hash_codera   s                 r!   rb   zYosoLSHCumulation.forwardˆ   sÈ  € à�?‰?˜1Ó §¡¨qÓ!1Ò1ÜÐUÓVÐVØ�?‰?˜1Ó §¡¨A£Ò.ÜÐRÓSÐSØ�?‰?˜1Ó §¡¨!£Ò,ÜÐPÓQÐQØ�?‰?˜1Ó §¡¨A£Ò.ÜÐWÓXÐXØ�8‰8�A‹;˜%Ÿ*™* Q›-Ò'ÜÐKÓLÐLØ�:‰:�a‹=˜CŸH™H Q›KÒ'ÜÐKÓLÐLä2?ÀÈXÐW\Ð^aÐchÐ@iÓ2jÑ/ˆ
�H˜e S¨%à×%Ñ%ˆØ˜*Ñ%ˆØ˜Ñ/ˆÜ   MÑ!1Ó2Ðà�/Ò"Ü-;×-EÑ-EØ˜E 8¨S°(¸MÈ8ÐUVó.Ñ*ˆO™]ô .5°U¸CÀÈ=Ó-YÑ*ˆO˜]ä)×8Ñ8Ø˜¨°=À%ÐI[Ð]eÐghó
Ðð 	×Ñ˜j¨(°OÀ]ÐTYÐ[^Ð`eÔfØˆŒ
àÐr#   c                 ó‚  — t        |«      }| j                  \  }}}}}}}| j                  }	|j                  }
|	d   }t	        d|z  «      }|	d   rft
        j                  |||||||
d«      }t
        j                  |||||||dz  |z  ||
d«
      }t
        j                  |||||||dz  |z  ||
d«
      }�ndt        j                  t        j                  ||j                  dd«      «      «      t        j                  z  z
  |z  }||d d …d d …d f   z  |d d …d d d …f   z  }t        j                  ||j                  dd«      «      |z  }t        j                  ||dz  |z  «      }t        j                  |j                  dd«      |dz  |z  «      }t        j                  |j                  dd«      |«      }d d |||d fS )NrT   r1   Úlsh_backwardr   é   r2   rU   )r/   rd   r[   ru   rC   r   Úlsh_weighted_cumulationr>   rV   rA   rW   rX   rY   )r\   re   r]   r^   ry   rz   rE   rF   r_   r[   rw   rT   rx   ri   rg   rh   r`   rf   s                     r!   rj   zYosoLSHCumulation.backward®   sé  € ä˜TÓ"ˆàRU×RcÑRcÑOˆ
�H˜o¨}¸eÀSÈ%Ø—‘ˆà—<‘<ˆØ˜Ñ/ˆÜ   MÑ!1Ó2Ðà�.Ò!Ü'×6Ñ6Ø˜-¨°_ÀdÐL^Ð`hÐjkóˆJô (×?Ñ?ØØØØØØØ Ñ" cÑ)Ø"ØØóˆJô &×=Ñ=ØØØØØØØ Ñ" eÑ+Ø"ØØóŠHð œuŸz™z¬%¯,©,°u¸c¿m¹mÈBÐPRÓ>SÓ*TÓUÔX\×X_ÑX_Ñ_Ñ_ÐdqÑqˆKØ%¨
²1²a¸°:Ñ(>Ñ>ÀÊ!ÈTÒSTÈ*ÑAUÑUˆKÜ Ÿ<™<¨¨e¯o©o¸bÀ"Ó.EÓFÈÑTˆLÜŸ™ l°]ÀQÑ5FÈ#Ñ4MÓNˆJÜ—|‘| L×$:Ñ$:¸2¸rÓ$BÀ]ÐUVÑEVÐZ_ÑD_Ó`ˆHÜŸ™ k×&;Ñ&;¸BÀÓ&CÀTÓJˆJà�T˜: x°¸TÐAÐAr#   Nrk   rp   r#   r!   rr   rr   ‡   s+   „ Øñ# ó ð# ðJ ñ.Bó ñ.Br#   rr   c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚYosoEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óP  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  dz   |j
                  «      | _	        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        | j#                  dt%        j&                  |j                  «      j)                  d«      dz   d¬«       | j#                  dt%        j*                  | j,                  j/                  «       t$        j0                  | j,                  j2                  ¬	«      d¬«       y )
N)Úpadding_idxr1   ©ÚepsÚposition_ids©r   r2   F)Ú
persistentÚtoken_type_ids©Údtyper9   )ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferr>   r@   ÚexpandÚzerosr…   r<   Úlongr9   ©Úselfr[   Ú	__class__s     €r!   rŒ   zYosoEmbeddings.__init__ä   s7  ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¡<°×0NÑ0NÐQRÑ0RÐTZ×TfÑTfÓ#gˆÔ Ü%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒð 	×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐ[\Ñ\Ðinð 	ô 	
ð 	×ÑØÜ�K‰K˜×)Ñ)×.Ñ.Ó0¼¿
¹
È4×K\ÑK\×KcÑKcÔdØð 	õ 	
r#   c                 ó6  — |�|j                  «       }n|j                  «       d d }|d   }|€| j                  d d …d |…f   }|€st        | d«      r-| j                  d d …d |…f   }|j	                  |d   |«      }|}n:t        j                  |t
        j                  | j                  j                  ¬«      }|€| j                  |«      }| j                  |«      }	||	z   }
| j                  |«      }|
|z  }
| j                  |
«      }
| j                  |
«      }
|
S )Nr2   r   rˆ   r   r‰   )r<   r…   Úhasattrrˆ   rœ   r>   r�   rž   r9   r‘   r•   r“   r–   rš   )r    Ú	input_idsrˆ   r…   Úinputs_embedsÚinput_shapeÚ
seq_lengthÚbuffered_token_type_idsÚ buffered_token_type_ids_expandedr•   Ú
embeddingsr“   s               r!   rb   zYosoEmbeddings.forward÷   s/  € ØÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
àÐØ×,Ñ,ªQ°°°¨^Ñ<ˆLð
 Ð!Ü�tÐ-Ô.Ø*.×*=Ñ*=ºaÀÀ*À¸nÑ*MÐ'Ø3J×3QÑ3QÐR]Ð^_ÑR`ÐblÓ3mÐ0Ø!A‘ä!&§¡¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÐ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐØ"Ð%:Ñ:ˆ
à"×6Ñ6°|ÓDÐØÐ)Ñ)ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr#   )NNNN©rl   rm   rn   Ú__doc__rŒ   rb   Ú__classcell__©r¡   s   @r!   r€   r€   á   s   ø„ ÙQô
÷& r#   r€   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚYosoSelfAttentionc                 ó€  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚t        d u}t        «       rt        «       r|s	 t        «        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t!        j"                  |j                  | j                  «      | _        t!        j"                  |j                  | j                  «      | _        t!        j"                  |j                  | j                  «      | _        t!        j*                  |j,                  «      | _        |j0                  | _        |j2                  | _        |j4                  d u| _        |j8                  | _        |j:                  | _        |j<                  | _        | j2                  | j8                  | j:                  | j<                  dœ| _        |j4                  �Zt!        j@                  |j                  |j                  |j4                  df|j4                  d	z  dfd
|j                  ¬«      | _!        y y # t        $ r#}t        j                  d|› �«       Y d }~�Œ1d }~ww xY w)Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)zGCould not load the custom kernel for multi-scale deformable attention: )rT   rt   rG   r|   r   r1   F)Úin_channelsÚout_channelsÚkernel_sizeÚpaddingÚbiasÚgroups)"r‹   rŒ   r�   Únum_attention_headsr£   r=   r   r   r   r"   Ú	ExceptionÚloggerÚwarningrC   Úattention_head_sizeÚall_head_sizer   ÚLinearrE   rF   r_   r˜   Úattention_probs_dropout_probrš   Úuse_expectationrT   Úconv_windowÚuse_convrt   rG   r|   Ú
lsh_configÚConv2dÚconv)r    r[   Úkernel_loadedÚer¡   s       €r!   rŒ   zYosoSelfAttention.__init__  sS  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ô '¨dÐ2ˆÜ"Ô$Ô);Ô)=ÁmðnÜ!Ô#ð $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒà%×5Ñ5ˆÔØ#×1Ñ1ˆÔØ×*Ñ*°$Ð6ˆŒØ#×1Ñ1ˆÔØŸ™ˆŒØ"×/Ñ/ˆÔð "×/Ñ/Ø!×/Ñ/ØŸ™Ø ×-Ñ-ñ	
ˆŒð ×ÑÐ)ÜŸ	™	Ø"×6Ñ6Ø#×7Ñ7Ø#×/Ñ/°Ð3Ø×+Ñ+¨qÑ0°!Ð4ØØ×1Ñ1ôˆD�Ið *øô7 ò nÜ—‘Ð!hÐijÐhkÐl×mÒmûðnús   Á=
J Ê	J=ÊJ8Ê8J=c                 óH  — |j                   \  }}}| j                  |«      j                  |d| j                  | j                  «      j                  dd«      }| j                  |«      j                  |d| j                  | j                  «      j                  dd«      }| j                  |«      j                  |d| j                  | j                  «      j                  dd«      }	| j                  r| j                  |	|d d …d d d …d f   z  «      }
|j                  «       \  }}}}|j                  ||z  ||«      }|j                  ||z  ||«      }|	j                  ||z  ||«      }	d|dz  z   }|j                  d«      j                  |d¬«      j                  ||z  |«      j                  «       }d}| j                  s¸||k  r³||z  |||z
  f}t!        j"                  |t!        j$                  ||j&                  ¬«      gd¬«      }t!        j"                  |t!        j$                  ||j&                  ¬«      gd¬«      }t!        j"                  |	t!        j$                  ||	j&                  ¬«      gd¬«      }	| j                  s| j(                  rt+        ||g«      \  }}| j                  r%t,        j/                  |||||	| j0                  «      }n$t2        j/                  |||||	| j0                  «      }| j                  s||k  r|d d …d d …d |…f   }t+        |«      }|j                  ||||«      }| j                  r|
z  }|j5                  d	ddd
«      j7                  «       }|j                  «       d d | j8                  fz   } |j                  |Ž }|r||f}|S |f}|S )Nr2   r   r1   g      ð?g     ˆÃ@r:   é    r8   r   r   rU   )ÚshaperE   Úviewrº   r¾   rW   rF   r_   rÄ   rÇ   r<   rB   Ú	unsqueezeÚrepeat_interleaverC   rÂ   r>   Úcatr�   r9   Útrainingr6   rR   ÚapplyrÅ   rr   Úpermuter)   r¿   )r    Úhidden_statesÚattention_maskÚoutput_attentionsÚ
batch_sizer§   Ú_Úquery_layerÚ	key_layerÚvalue_layerÚconv_value_layerÚ	num_headsÚseq_lenÚhead_dimÚgpu_warp_sizeÚpad_sizeÚcontext_layerÚnew_context_layer_shapeÚoutputss                      r!   rb   zYosoSelfAttention.forwardK  sž  € Ø$1×$7Ñ$7Ñ!ˆ
�J à�J‰J�}Ó%ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �H‰H�]Ó#ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �J‰J�}Ó%ß‰T�*˜b $×":Ñ":¸D×<TÑ<TÓUß‰Y�q˜!‹_ð 	ð �=Š=Ø#Ÿy™y¨°~ÂaÈÊqÐRVÐFVÑ7WÑ)WÓXÐà3>×3CÑ3CÓ3EÑ0ˆ
�I˜w¨à!×)Ñ)¨*°yÑ*@À'È8ÓTˆØ×%Ñ% j°9Ñ&<¸gÀxÓPˆ	Ø!×)Ñ)¨*°yÑ*@À'È8ÓTˆà˜~°Ñ7Ñ7ˆà×$Ñ$ QÓ'ßÑ˜y¨aÐÓ0ß‰W�Z )Ñ+¨WÓ5ß‰S‹Uð	 	ð ˆà×$Ò$¨(°]Ò*BØ! IÑ-¨w¸ÈÑ8PÐPˆHäŸ)™)àÜ—K‘K °×1CÑ1CÔDðð ôˆKô Ÿ	™	àÜ—K‘K °×1AÑ1AÔBðð ôˆIô  Ÿ)™)àÜ—K‘K °×1CÑ1CÔDðð ôˆKð ×Ò 4§=¢=Ü%.°¸YÐ/GÓ%HÑ"ˆK˜à×ÒÜ*×0Ñ0Ø °¸YÈÐUY×UdÑUdó‰Mô .×3Ñ3Ø °¸YÈÐUY×UdÑUdóˆMð ×$Ò$¨(°]Ò*BØ)ª!ªQ°	°°	¨/Ñ:ˆMä! -Ó0ˆà%×-Ñ-¨j¸)ÀWÈhÓWˆà�=Š=ØÐ-Ñ-ˆMà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ*˜×*Ñ*Ð,CÐDˆá4E�= -Ð0ˆàˆð MZÐK[ˆàˆr#   ©NF©rl   rm   rn   rŒ   rb   r­   r®   s   @r!   r°   r°     s   ø„ ô.÷`\r#   r°   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚYosoSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©Nrƒ   )r‹   rŒ   r   rÀ   r�   Údenser–   r—   r˜   r™   rš   rŸ   s     €r!   rŒ   zYosoSelfOutput.__init__¬  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r#   rÔ   Úinput_tensorÚreturnc                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r%   ©rë   rš   r–   ©r    rÔ   rì   s      r!   rb   zYosoSelfOutput.forward²  ó7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr#   ©rl   rm   rn   rŒ   r>   ÚTensorrb   r­   r®   s   @r!   rè   rè   «  ó1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r#   rè   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚYosoAttentionc                 ób   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        y r%   )r‹   rŒ   r°   r    rè   ÚoutputrŸ   s     €r!   rŒ   zYosoAttention.__init__º  s&   ø€ Ü‰ÑÔÜ% fÓ-ˆŒ	Ü$ VÓ,ˆ�r#   c                 óh   — | j                  |||«      }| j                  |d   |«      }|f|dd  z   }|S )Nr   r   )r    rø   )r    rÔ   rÕ   rÖ   Úself_outputsÚattention_outputrä   s          r!   rb   zYosoAttention.forward¿  sC   € Ø—y‘y °Ð@QÓRˆØŸ;™; |°A¡¸ÓFÐØ#Ð%¨°Q°RÐ(8Ñ8ˆØˆr#   rå   ræ   r®   s   @r!   rö   rö   ¹  s   ø„ ô-÷
r#   rö   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚYosoIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y r%   )r‹   rŒ   r   rÀ   r�   Úintermediate_sizerë   r&   Ú
hidden_actÚstrr	   Úintermediate_act_fnrŸ   s     €r!   rŒ   zYosoIntermediate.__init__È  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r#   rÔ   rí   c                 óJ   — | j                  |«      }| j                  |«      }|S r%   )rë   r  ©r    rÔ   s     r!   rb   zYosoIntermediate.forwardÐ  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr#   rò   r®   s   @r!   rý   rý   Ç  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r#   rý   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )Ú
YosoOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y rê   )r‹   rŒ   r   rÀ   rÿ   r�   rë   r–   r—   r˜   r™   rš   rŸ   s     €r!   rŒ   zYosoOutput.__init__Ø  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r#   rÔ   rì   rí   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r%   rï   rð   s      r!   rb   zYosoOutput.forwardÞ  rñ   r#   rò   r®   s   @r!   r  r  ×  rô   r#   r  c                   ó,   ‡ — e Zd Zˆ fd„Zdd„Zd„ Zˆ xZS )Ú	YosoLayerc                 óÔ   •— t         ‰| �  «        |j                  | _        d| _        t	        |«      | _        |j                  | _        t        |«      | _        t        |«      | _
        y ©Nr   )r‹   rŒ   Úchunk_size_feed_forwardÚseq_len_dimrö   Ú	attentionÚadd_cross_attentionrý   Úintermediater  rø   rŸ   s     €r!   rŒ   zYosoLayer.__init__æ  sW   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ& vÓ.ˆŒØ#)×#=Ñ#=ˆÔ Ü,¨VÓ4ˆÔÜ  Ó(ˆ�r#   c                 ó¦   — | j                  |||¬«      }|d   }|dd  }t        | j                  | j                  | j                  |«      }|f|z   }|S )N)rÖ   r   r   )r  r   Úfeed_forward_chunkr  r  )r    rÔ   rÕ   rÖ   Úself_attention_outputsrû   rä   Úlayer_outputs           r!   rb   zYosoLayer.forwardï  sh   € Ø!%§¡°¸~Ðar Ó!sÐØ1°!Ñ4Ðà(¨¨Ð,ˆä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð  �/ GÑ+ˆàˆr#   c                 óL   — | j                  |«      }| j                  ||«      }|S r%   )r  rø   )r    rû   Úintermediate_outputr  s       r!   r  zYosoLayer.feed_forward_chunkü  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐr#   rå   )rl   rm   rn   rŒ   rb   r  r­   r®   s   @r!   r
  r
  å  s   ø„ ô)óör#   r
  c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚYosoEncoderc                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w rå   )
r‹   rŒ   r[   r   Ú
ModuleListÚrangeÚnum_hidden_layersr
  ÚlayerÚgradient_checkpointing)r    r[   rØ   r¡   s      €r!   rŒ   zYosoEncoder.__init__  sN   ø€ Ü‰ÑÔØˆŒÜ—]‘]¼uÀV×E]ÑE]Ó?^Ö#_¸!¤I¨fÕ$5Ò#_Ó`ˆŒ
Ø&+ˆÕ#ùò $`s   ½A#c                 óô   — |rdnd }|rdnd }t        | j                  «      D ](  \  }}	|r||fz   } |	|||«      }
|
d   }|sŒ ||
d   fz   }Œ* |r||fz   }|st        d„ |||fD «       «      S t        |||¬«      S )Nrp   r   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr%   rp   )Ú.0Úvs     r!   ú	<genexpr>z&YosoEncoder.forward.<locals>.<genexpr>"  s   è ø€ Òm˜qÐ_`Ñ_lœÑmùs   ‚Š)Úlast_hidden_staterÔ   Ú
attentions)Ú	enumerater  Útupler   )r    rÔ   rÕ   rÖ   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚall_self_attentionsÚiÚlayer_moduleÚlayer_outputss              r!   rb   zYosoEncoder.forward	  s¼   € ñ #7™B¸DÐÙ$5™b¸4Ðä(¨¯©Ó4ò 	P‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!á(¨¸ÐHYÓZˆMà)¨!Ñ,ˆMÚ Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð	Pñ  Ø 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜ1Ø+Ø+Ø*ô
ð 	
r#   )NFFTræ   r®   s   @r!   r  r    s   ø„ ô,ð ØØ"Ø÷
r#   r  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚYosoPredictionHeadTransformc                 óh  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        |j                  t        «      rt        |j                     | _
        n|j                  | _
        t        j                  |j                  |j                  ¬«      | _        y rê   )r‹   rŒ   r   rÀ   r�   rë   r&   r   r  r	   Útransform_act_fnr–   r—   rŸ   s     €r!   rŒ   z$YosoPredictionHeadTransform.__init__,  s{   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬Ô-Ü$*¨6×+<Ñ+<Ñ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆ�r#   rÔ   rí   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r%   )rë   r3  r–   r  s     r!   rb   z#YosoPredictionHeadTransform.forward5  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐr#   rò   r®   s   @r!   r1  r1  +  s$   ø„ ôUð U§\¡\ð °e·l±l÷ r#   r1  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚYosoLMPredictionHeadc                 ó  •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        t	        j                  t        j                  |j                  «      «      | _        y )NT)r¸   )r‹   rŒ   r1  Ú	transformr   rÀ   r�   rŽ   ÚdecoderÚ	Parameterr>   r�   r¸   rŸ   s     €r!   rŒ   zYosoLMPredictionHead.__init__>  s[   ø€ Ü‰ÑÔÜ4°VÓ<ˆŒô —y‘y ×!3Ñ!3°V×5FÑ5FÈTÔRˆŒÜ—L‘L¤§¡¨V×->Ñ->Ó!?Ó@ˆ�	r#   c                 óJ   — | j                  |«      }| j                  |«      }|S r%   )r8  r9  r  s     r!   rb   zYosoLMPredictionHead.forwardG  s$   € ØŸ™ }Ó5ˆØŸ™ ]Ó3ˆØÐr#   ræ   r®   s   @r!   r6  r6  =  s   ø„ ôAör#   r6  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚYosoOnlyMLMHeadc                 óB   •— t         ‰| �  «        t        |«      | _        y r%   )r‹   rŒ   r6  ÚpredictionsrŸ   s     €r!   rŒ   zYosoOnlyMLMHead.__init__O  s   ø€ Ü‰ÑÔÜ/°Ó7ˆÕr#   Úsequence_outputrí   c                 ó(   — | j                  |«      }|S r%   )r?  )r    r@  Úprediction_scoress      r!   rb   zYosoOnlyMLMHead.forwardS  s   € Ø ×,Ñ,¨_Ó=ÐØ Ð r#   rò   r®   s   @r!   r=  r=  N  s#   ø„ ô8ð! u§|¡|ð !¸¿¹÷ !r#   r=  c                   ót   ‡ — e Zd ZU eed<   dZdZ ej                  «       de	j                  fˆ fd„«       Zˆ xZS )ÚYosoPreTrainedModelr[   r    TÚmodulec                 óœ  •— t         ‰| �  |«       t        |t        «      r t	        j
                  |j                  «       yt        |t        «      r|t	        j                  |j                  t        j                  |j                  j                  d   «      j                  d«      dz   «       t	        j
                  |j                  «       yy)zInitialize the weightsr2   r†   r1   N)r‹   Ú_init_weightsr&   r6  ÚinitÚzeros_r¸   r€   Úcopy_r…   r>   r@   rÌ   rœ   rˆ   )r    rE  r¡   s     €r!   rG  z!YosoPreTrainedModel._init_weights^  s�   ø€ ô 	‰Ñ˜fÔ%Ü�fÔ2Ô3Ü�K‰K˜Ÿ™Õ$Ü˜¤Ô/Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÐjkÑ,kÔlÜ�K‰K˜×-Ñ-Õ.ð 0r#   )rl   rm   rn   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingr>   Úno_gradr   ÚModulerG  r­   r®   s   @r!   rD  rD  X  s:   ø… àÓØÐØ&*Ð#à€U‡]�]ƒ_ð/ B§I¡Iô /ó ô/r#   rD  c                   ó  ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	dz  de	dz  de	dz  de
ez  fd„«       Zˆ xZS )Ú	YosoModelc                 ó’   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        | j                  «        y r%   )r‹   rŒ   r[   r€   rª   r  ÚencoderÚ	post_initrŸ   s     €r!   rŒ   zYosoModel.__init__k  s;   ø€ Ü‰Ñ˜Ô ØˆŒä(¨Ó0ˆŒÜ" 6Ó*ˆŒð 	�‰Õr#   c                 ó.   — | j                   j                  S r%   ©rª   r‘   ©r    s    r!   Úget_input_embeddingszYosoModel.get_input_embeddingsu  s   € Ø�‰×.Ñ.Ð.r#   c                 ó&   — || j                   _        y r%   rV  )r    r_   s     r!   Úset_input_embeddingszYosoModel.set_input_embeddingsx  s   € Ø*/ˆ�‰Õ'r#   Nr¤   rÕ   rˆ   r…   r¥   rÖ   r)  r*  rí   c	                 óp  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|�|�t	        d«      ‚|�#| j                  ||«       |j                  «       }
n!|�|j                  «       d d }
nt	        d«      ‚|
\  }}|�|j                  n|j                  }|€t        j                  ||f|¬«      }|€pt        | j                  d«      r4| j                  j                  d d …d |…f   }|j                  ||«      }|}n&t        j                  |
t        j                  |¬«      }| j                  ||||¬«      }| j!                  |||||¬«      }|d	   }|s	|f|d
d  z   S t#        ||j$                  |j&                  |j(                  ¬«      S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer2   z5You have to specify either input_ids or inputs_embedsr8   rˆ   r‰   )r¤   r…   rˆ   r¥   )rÕ   rÖ   r)  r*  r   r   )r%  rÔ   r&  Úcross_attentions)r[   rÖ   r)  r*  r=   Ú%warn_if_padding_and_no_attention_maskr<   r9   r>   Úonesr£   rª   rˆ   rœ   r�   rž   rS  r   rÔ   r&  r\  )r    r¤   rÕ   rˆ   r…   r¥   rÖ   r)  r*  Úkwargsr¦   r×   r§   r9   r¨   r©   Úembedding_outputÚencoder_outputsr@  s                      r!   rb   zYosoModel.forward{  så  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
�JØ%.Ð%:�×!Ò!À×@TÑ@TˆàÐ!Ü"ŸZ™Z¨*°jÐ)AÈ6ÔRˆNàÐ!Ü�t—‘Ð(8Ô9Ø*.¯/©/×*HÑ*HÊÈKÈZÈKÈÑ*XÐ'Ø3J×3QÑ3QÐR\Ð^hÓ3iÐ0Ø!A‘ä!&§¡¨[ÄÇ
Á
ÐSYÔ!Z�àŸ?™?ØØ%Ø)Ø'ð	 +ó 
Ðð Ÿ,™,ØØ)Ø/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆáØ#Ð%¨¸¸Ð(;Ñ;Ð;ä1Ø-Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ô	
ð 	
r#   )NNNNNNNN)rl   rm   rn   rŒ   rX  rZ  r   r>   ró   Úboolr(  r   rb   r­   r®   s   @r!   rQ  rQ  i  sâ   ø„ ôò/ò0ð ð *.Ø.2Ø.2Ø,0Ø-1Ø)-Ø,0Ø#'ñA
à—<‘< $Ñ&ðA
ð Ÿ™ tÑ+ðA
ð Ÿ™ tÑ+ð	A
ð
 —l‘l TÑ)ðA
ð —|‘| dÑ*ðA
ð   $™;ðA
ð # T™kðA
ð ˜D‘[ðA
ð 
Ð3Ñ	3òA
ó ôA
r#   rQ  c                   ó6  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  de
dz  de
dz  de
dz  deez  fd„«       Zˆ xZS )ÚYosoForMaskedLMzcls.predictions.biasz&yoso.embeddings.word_embeddings.weight)zcls.predictions.decoder.biaszcls.predictions.decoder.weightc                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y r%   )r‹   rŒ   rQ  r    r=  ÚclsrT  rŸ   s     €r!   rŒ   zYosoForMaskedLM.__init__Ç  s4   ø€ Ü‰Ñ˜Ô ä˜fÓ%ˆŒ	Ü" 6Ó*ˆŒð 	�‰Õr#   c                 óB   — | j                   j                  j                  S r%   )rf  r?  r9  rW  s    r!   Úget_output_embeddingsz%YosoForMaskedLM.get_output_embeddingsÐ  s   € Ø�x‰x×#Ñ#×+Ñ+Ð+r#   c                 ó„   — || j                   j                  _        |j                  | j                   j                  _        y r%   )rf  r?  r9  r¸   )r    Únew_embeddingss     r!   Úset_output_embeddingsz%YosoForMaskedLM.set_output_embeddingsÓ  s,   € Ø'5ˆ�‰×ÑÔ$Ø$2×$7Ñ$7ˆ�‰×ÑÕ!r#   Nr¤   rÕ   rˆ   r…   r¥   ÚlabelsrÖ   r)  r*  rí   c
           
      ó˜  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }| j                  |«      }d}|�Ft	        «       } ||j                  d| j                   j                  «      |j                  d«      «      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a£  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N©rÕ   rˆ   r…   r¥   rÖ   r)  r*  r   r2   r   ©ÚlossÚlogitsrÔ   r&  )
r[   r*  r    rf  r   rÍ   rŽ   r   rÔ   r&  )r    r¤   rÕ   rˆ   r…   r¥   rl  rÖ   r)  r*  r_  rä   r@  rB  Úmasked_lm_lossÚloss_fctrø   s                    r!   rb   zYosoForMaskedLM.forward×  s÷   € ð( &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—)‘)ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð " !™*ˆØ ŸH™H _Ó5ÐàˆØÐÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNáØ'Ð)¨G°A°B¨KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r#   ©	NNNNNNNNN)rl   rm   rn   Ú_tied_weights_keysrŒ   rh  rk  r   r>   ró   rb  r(  r   rb   r­   r®   s   @r!   rd  rd  À  sü   ø„ ð )?Ø*RñÐô
ò,ò8ð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø)-Ø,0Ø#'ñ1
à—<‘< $Ñ&ð1
ð Ÿ™ tÑ+ð1
ð Ÿ™ tÑ+ð	1
ð
 —l‘l TÑ)ð1
ð —|‘| dÑ*ð1
ð —‘˜tÑ#ð1
ð   $™;ð1
ð # T™kð1
ð ˜D‘[ð1
ð 
�Ñ	ò1
ó ô1
r#   rd  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚYosoClassificationHeadz-Head for sentence-level classification tasks.c                 ó4  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _
        || _        y r%   )r‹   rŒ   r   rÀ   r�   rë   r˜   r™   rš   Ú
num_labelsÚout_projr[   rŸ   s     €r!   rŒ   zYosoClassificationHead.__init__  sg   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆŒÜŸ	™	 &×"4Ñ"4°f×6GÑ6GÓHˆŒàˆ�r#   c                 óê   — |d d …dd d …f   }| j                  |«      }| j                  |«      }t        | j                  j                     |«      }| j                  |«      }| j                  |«      }|S )Nr   )rš   rë   r	   r[   r   rz  )r    Úfeaturesr_  Úxs       r!   rb   zYosoClassificationHead.forward  se   € Ø’Q˜š1�WÑˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆÜ�4—;‘;×)Ñ)Ñ*¨1Ó-ˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆr#   r«   r®   s   @r!   rw  rw    s   ø„ Ù7ôör#   rw  zœ
    YOSO Model transformer with a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks.
    )Úcustom_introc                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  dee	z  fd„«       Z
ˆ xZS )ÚYosoForSequenceClassificationc                 ó¦   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        |«      | _        | j                  «        y r%   )r‹   rŒ   ry  rQ  r    rw  Ú
classifierrT  rŸ   s     €r!   rŒ   z&YosoForSequenceClassification.__init__(  sA   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ˜fÓ%ˆŒ	Ü0°Ó8ˆŒð 	�‰Õr#   Nr¤   rÕ   rˆ   r…   r¥   rl  rÖ   r)  r*  rí   c
           
      ó  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }| j                  |«      }d}|��‡| j                   j                  €�| j
                  dk(  rd| j                   _        nl| j
                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt        «       }| j
                  dk(  r& ||j                  «       |j                  «       «      }nŒ |||«      }n‚| j                   j                  dk(  r=t        «       } ||j                  d| j
                  «      |j                  d«      «      }n,| j                   j                  dk(  rt        «       } |||«      }|	s|f|dd z   }|�|f|z   S |S t        |||j                   |j"                  ¬	«      S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nrn  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr2   ro  )r[   r*  r    r‚  Úproblem_typery  rŠ   r>   rž   rC   r   Úsqueezer   rÍ   r   r   rÔ   r&  )r    r¤   rÕ   rˆ   r…   r¥   rl  rÖ   r)  r*  r_  rä   r@  rq  rp  rs  rø   s                    r!   rb   z%YosoForSequenceClassification.forward1  s×  € ð( &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—)‘)ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð " !™*ˆØ—‘ Ó1ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ÙØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r#   rt  )rl   rm   rn   rŒ   r   r>   ró   rb  r(  r   rb   r­   r®   s   @r!   r€  r€  !  sï   ø„ ôð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø)-Ø,0Ø#'ñB
à—<‘< $Ñ&ðB
ð Ÿ™ tÑ+ðB
ð Ÿ™ tÑ+ð	B
ð
 —l‘l TÑ)ðB
ð —|‘| dÑ*ðB
ð —‘˜tÑ#ðB
ð   $™;ðB
ð # T™kðB
ð ˜D‘[ðB
ð 
Ð)Ñ	)òB
ó ôB
r#   r€  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  dee	z  fd„«       Z
ˆ xZS )ÚYosoForMultipleChoicec                 ó  •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  «      | _        t	        j
                  |j                  d«      | _        | j                  «        y r  )
r‹   rŒ   rQ  r    r   rÀ   r�   Úpre_classifierr‚  rT  rŸ   s     €r!   rŒ   zYosoForMultipleChoice.__init__y  s_   ø€ Ü‰Ñ˜Ô ä˜fÓ%ˆŒ	Ü Ÿi™i¨×(:Ñ(:¸F×<NÑ<NÓOˆÔÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õr#   Nr¤   rÕ   rˆ   r…   r¥   rl  rÖ   r)  r*  rí   c
           
      ó�  — |	�|	n| j                   j                  }	|�|j                  d   n|j                  d   }|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�1|j                  d|j	                  d«      |j	                  d«      «      nd}| j                  ||||||||	¬«      }|d   }|dd…df   }| j                  |«      } t        j                  «       |«      }| j                  |«      }|j                  d|«      }d}|�t        «       } |||«      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r2   rU   rn  r   ro  )r[   r*  rÌ   rÍ   r<   r    rŒ  r   ÚReLUr‚  r   r   rÔ   r&  )r    r¤   rÕ   rˆ   r…   r¥   rl  rÖ   r)  r*  r_  Únum_choicesrä   Úhidden_stateÚpooled_outputrq  Úreshaped_logitsrp  rs  rø   s                       r!   rb   zYosoForMultipleChoice.forwardƒ  sý  € ðX &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ,5Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÐG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —)‘)ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð ˜q‘zˆØ$¢Q¨ TÑ*ˆØ×+Ñ+¨MÓ:ˆØ!œŸ™›	 -Ó0ˆØ—‘ Ó/ˆà Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDáØ%Ð'¨'°!°"¨+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r#   rt  )rl   rm   rn   rŒ   r   r>   ró   rb  r(  r   rb   r­   r®   s   @r!   rŠ  rŠ  w  sï   ø„ ôð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø)-Ø,0Ø#'ñY
à—<‘< $Ñ&ðY
ð Ÿ™ tÑ+ðY
ð Ÿ™ tÑ+ð	Y
ð
 —l‘l TÑ)ðY
ð —|‘| dÑ*ðY
ð —‘˜tÑ#ðY
ð   $™;ðY
ð # T™kðY
ð ˜D‘[ðY
ð 
Ð*Ñ	*òY
ó ôY
r#   rŠ  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  dee	z  fd„«       Z
ˆ xZS )ÚYosoForTokenClassificationc                 ó,  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r%   )r‹   rŒ   ry  rQ  r    r   r˜   r™   rš   rÀ   r�   r‚  rT  rŸ   s     €r!   rŒ   z#YosoForTokenClassification.__init__â  si   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä˜fÓ%ˆŒ	Ü—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr#   Nr¤   rÕ   rˆ   r…   r¥   rl  rÖ   r)  r*  rí   c
           
      óÂ  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }| j                  |«      }| j	                  |«      }d}|�Êt        «       }|�Œ|j                  d«      dk(  }|j                  d| j                  «      }t        j                  ||j                  d«      t        j                  |j                  «      j                  |«      «      } |||«      }n2 ||j                  d| j                  «      |j                  d«      «      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nrn  r   r2   r   ro  )r[   r*  r    rš   r‚  r   rÍ   ry  r>   Úwherer.   Úignore_indexÚtype_asr   rÔ   r&  )r    r¤   rÕ   rˆ   r…   r¥   rl  rÖ   r)  r*  r_  rä   r@  rq  rp  rs  Úactive_lossÚactive_logitsÚactive_labelsrø   s                       r!   rb   z"YosoForTokenClassification.forwardí  sh  € ð$ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—)‘)ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHàÐ)Ø,×1Ñ1°"Ó5¸Ñ:�Ø &§¡¨B°·±Ó @�Ü %§¡Ø §¡¨R£´%·,±,¸x×?TÑ?TÓ2U×2]Ñ2]Ð^dÓ2eó!�ñ   ¨}Ó=‘á §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR�áØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r#   rt  )rl   rm   rn   rŒ   r   r>   ró   rb  r(  r   rb   r­   r®   s   @r!   r”  r”  à  sã   ø„ ô	ð ð *.Ø.2Ø.2Ø,0Ø-1Ø&*Ø)-Ø,0Ø#'ñ:
à—<‘< $Ñ&ð:
ð Ÿ™ tÑ+ð:
ð Ÿ™ tÑ+ð	:
ð
 —l‘l TÑ)ð:
ð —|‘| dÑ*ð:
ð —‘˜tÑ#ð:
ð   $™;ð:
ð # T™kð:
ð ˜D‘[ð:
ð 
Ð&Ñ	&ò:
ó ô:
r#   r”  c                   ó@  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
edz  dedz  dedz  dee	z  fd„«       Z
ˆ xZS )ÚYosoForQuestionAnsweringc                 óò   •— t         ‰| �  |«       d|_        |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y )Nr1   )
r‹   rŒ   ry  rQ  r    r   rÀ   r�   Ú
qa_outputsrT  rŸ   s     €r!   rŒ   z!YosoForQuestionAnswering.__init__-  s[   ø€ Ü‰Ñ˜Ô àˆÔØ ×+Ñ+ˆŒä˜fÓ%ˆŒ	ÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr#   Nr¤   rÕ   rˆ   r…   r¥   Ústart_positionsÚend_positionsrÖ   r)  r*  rí   c           
      óî  — |
�|
n| j                   j                  }
| j                  |||||||	|
¬«      }|d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      }|j                  d«      }d }|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }|
s||f|dd  z   }|�|f|z   S |S t        ||||j                  |j                  ¬«      S )	Nrn  r   r   r2   r:   )r˜  r1   )rp  Ústart_logitsÚ
end_logitsrÔ   r&  )r[   r*  r    r   Úsplitrˆ  r;   r<   Úclampr   r   rÔ   r&  )r    r¤   rÕ   rˆ   r…   r¥   r¡  r¢  rÖ   r)  r*  r_  rä   r@  rq  r¤  r¥  Ú
total_lossÚignored_indexrs  Ú
start_lossÚend_lossrø   s                          r!   rb   z YosoForQuestionAnswering.forward9  s­  € ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—)‘)ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/ˆØ×'Ñ'¨Ó+ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r#   )
NNNNNNNNNN)rl   rm   rn   rŒ   r   r>   ró   rb  r(  r   rb   r­   r®   s   @r!   rž  rž  +  sù   ø„ ô
ð ð *.Ø.2Ø.2Ø,0Ø-1Ø/3Ø-1Ø)-Ø,0Ø#'ñ=
à—<‘< $Ñ&ð=
ð Ÿ™ tÑ+ð=
ð Ÿ™ tÑ+ð	=
ð
 —l‘l TÑ)ð=
ð —|‘| dÑ*ð=
ð Ÿ™¨Ñ,ð=
ð —|‘| dÑ*ð=
ð   $™;ð=
ð # T™kð=
ð ˜D‘[ð=
ð 
Ð-Ñ	-ò=
ó ô=
r#   rž  )rd  rŠ  rž  r€  r”  r
  rQ  rD  )Cr¬   rX   r>   r   Útorch.nnr   r   r   Ú r   rH  Úactivationsr	   Úmodeling_layersr
   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   r   Úconfiguration_yosor   Ú
get_loggerrl   r¼   r   r"   r/   r6   rP   ÚautogradÚFunctionrR   rr   rO  r€   r°   rè   rö   rý   r  r
  r  r1  r6  r=  rD  rQ  rd  rw  r€  rŠ  r”  rž  Ú__all__rp   r#   r!   ú<module>r¹     sT  ðñ ã ã Ý ß AÑ Aå &Ý !Ý 9÷÷ õ .Ý 6÷õ õ +ð 
ˆ×	Ñ	˜HÓ	%€ð €ò)òòCò.ô&B�U—^‘^×,Ñ,ô Bô>VB˜Ÿ™×/Ñ/ô VBôt6�R—Y‘Yô 6ôrM˜Ÿ	™	ô Môb�R—Y‘Yô ô
�B—I‘Iô 
ô�r—y‘yô ô �—‘ô ôÐ*ô ô:%
�"—)‘)ô %
ôR "§)¡)ô ô$˜2Ÿ9™9ô ô"!�b—i‘iô !ð ô/˜/ó /ó ð/ð  ôS
Ð#ó S
ó ðS
ðl ôH
Ð)ó H
ó ðH
ôV˜RŸY™Yô ñ* ðôôM
Ð$7ó M
óðM
ð` ôe
Ð/ó e
ó ðe
ðP ôG
Ð!4ó G
ó ðG
ðT ôK
Ð2ó K
ó ðK
ò\	�r#   