Ë
    ÛÓ^jÛ1 ã                   óf  — d Z ddlZddlmZ ddlmZ ddlmZ ddlm	Z	m
Z
mZmZmZmZmZ ddlmZmZmZ ddlmZ d	d
lmZmZmZmZ d	dlmZ  ej8                  e«      Zg d¢Zeg d¢z   Z d¨d„Z!de"de#fd„Z$d©dee#   dz  fd„Z% G d„ d«      Z& G d„ de&«      Z'de#de"fd„Z( G d„ d«      Z) G d„ de)«      Z* G d„ de)«      Z+ G d „ d!e)«      Z, G d"„ d#e)«      Z- G d$„ d%e)«      Z. G d&„ d'e)«      Z/ G d(„ d)e)«      Z0 G d*„ d+e)«      Z1 G d,„ d-e)«      Z2 G d.„ d/e)«      Z3 G d0„ d1e)«      Z4 G d2„ d3e)«      Z5 G d4„ d5e5«      Z6 G d6„ d7e5«      Z7 G d8„ d9e5«      Z8 G d:„ d;e5«      Z9 G d<„ d=e5«      Z: G d>„ d?e5«      Z; G d@„ dAe5«      Z< G dB„ dCe5«      Z= G dD„ dEe5«      Z> G dF„ dGe5«      Z? G dH„ dIe5«      Z@ G dJ„ dKe5«      ZA G dL„ dMe5«      ZB G dN„ dOe5«      ZC G dP„ dQe5«      ZD G dR„ dSe5«      ZE G dT„ dUeD«      ZF G dV„ dWe)«      ZG G dX„ dYe5«      ZH G dZ„ d[e)«      ZI G d\„ d]e)«      ZJ G d^„ d_e)«      ZK G d`„ dae5«      ZL G db„ dce5«      ZM G dd„ dee5«      ZN G df„ dge)«      ZO G dh„ die5«      ZP G dj„ dke5«      ZQ G dl„ dme5«      ZRdn„ ZS G do„ dp«      ZT G dq„ dr«      ZUi dse6“dte2“due7“dve*“dweH“dxeK“dye8“dzeI“d{e/“d|e*“d}e4“d~e9“de*“d€e*“d�e*“d‚e*“dƒe*“i d„e6“d…e,“d†e/“d‡e0“dˆe*“d‰e*“dŠe2“d‹e>“dŒe2“d�e2“dŽe*“d�eO“d�e:“d‘e;“d’e-“d“e*“d”e2“¥i d•e<“d–e.“d—eC“d˜e1“d™e@“dšeA“d›e2“dœe3“d�e=“dže*“dŸeD“d eE“d¡eF“d¢eG“d£e>“d¤e?“d¥e+“¥eLeNeNeMeNd¦œ¥ZVdªdefd§„ZWy)«zù
Utilities to convert slow tokenizers in their fast tokenizers counterparts.

All the conversions are grouped here to gather SentencePiece dependencies outside of the fast tokenizers files and
allow to make our dependency on SentencePiece optional.
é    N)Ú
Collection)Ú	lru_cache)Úversion)Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramÚ	WordPiece)Útqdmé   )Úis_protobuf_availableÚis_sentencepiece_availableÚloggingÚrequires_backends)ÚPROTOBUF_IMPORT_ERROR)Úar_ARÚcs_CZÚde_DEÚen_XXÚes_XXÚet_EEÚfi_FIÚfr_XXÚgu_INÚhi_INÚit_ITÚja_XXÚkk_KZÚko_KRÚlt_LTÚlv_LVÚmy_MMÚne_NPÚnl_XXÚro_ROÚru_RUÚsi_LKÚtr_TRÚvi_VNÚzh_CN)Úaf_ZAÚaz_AZÚbn_INÚfa_IRÚhe_ILÚhr_HRÚid_IDÚka_GEÚkm_KHÚmk_MKÚml_INÚmn_MNÚmr_INÚpl_PLÚps_AFÚpt_XXÚsv_SEÚsw_KEÚta_INÚte_INÚth_THÚtl_XXÚuk_UAÚur_PKÚxh_ZAÚgl_ESÚsl_SIc                 ó  — t        «       rddlm} |S t        «       rSdd l}t        j                  |j                  j                  «      t        j                  d«      k  rddl	m} |S ddl	m
} |S t        t        j                  | «      «      ‚)Nr   )Úsentencepiece_model_pb2z4.0.0)Úsentencepiece_model_pb2_new)r   ÚsentencepiecerL   r   Úgoogle.protobufr   ÚparseÚprotobufÚ__version__Útransformers.utilsrM   ÚImportErrorr   Úformat)Úerror_messagerL   Úgoogles      ún/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/convert_slow_tokenizer.pyÚimport_protobufrY   _   sl   € Ü!Ô#Ý9à&Ð&ÜÔÛä�=‰=˜Ÿ™×4Ñ4Ó5¼¿¹ÀgÓ8NÒNÝBð 'Ð&õ bØ&Ð&äÔ/×6Ñ6°}ÓEÓFÐFó    Úadd_prefix_spaceÚreturnc                 ó4   — | rd}t        |dd«      sd}|S d}|S )NÚalwaysÚlegacyTÚfirstÚnever)Úgetattr)r[   Úoriginal_tokenizerÚprepend_schemes      rX   Ú_get_prepend_schemere   p   s1   € ÙØ!ˆÜÐ)¨8°TÔ:Ø$ˆNð Ðð !ˆØÐrZ   Úskip_tokensc                 óÌ  ‡ — |�t        |«      n	t        «       }|d u}|rt        |«      n‰ }g }|j                  «       D ]x  \  }}||v rŒg }t        dt	        |«      «      D ]3  }|d | ||d  }
}	|	|v s|
|v rŒ|	‰ v sŒ|
‰ v sŒ |j                  |	|
|f«       Œ5 t        |ˆ fd„¬«      }|j                  |«       Œz t        |d„ |¬«      }|D �cg c]  }|d   |d   f‘Œ }}|S c c}w )Nr   c                 ó$   •— ‰| d      ‰| d      fS ©Nr   r   © )ÚxÚvocabs    €rX   ú<lambda>z!generate_merges.<locals>.<lambda>Š   s   ø€ ¨U°1°Q±4©[¸%ÀÀ!Á¹+Ð,F€ rZ   ©Úkeyc                 óB   — | d   t        | d   «      t        | d   «      fS )Né   r   r   )Úlen©Úvals    rX   rm   z!generate_merges.<locals>.<lambda>�   s!   € ¨S°©V´S¸¸Q¹³[Ä#ÀcÈ!ÁfÃ+Ð,N€ rZ   ©ro   Úreverser   )ÚsetÚdictÚitemsÚrangerr   ÚappendÚsortedÚextend)rl   Úvocab_scoresrf   rv   ÚmergesÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rrt   s   `           rX   Úgenerate_mergesr†   z   s  ø€ Ø&1Ð&=”#�kÔ"Ä3Ã5€KØ $Ð&€GÙ)0”4˜Ô%°e€Là€FØ*×0Ñ0Ó2ò Ñˆˆ{Ø�KÑØØˆÜ˜1œc %›jÓ)ò 	>ˆEØ$ V e˜}¨e°E°F¨m�WˆGØ˜+Ñ%¨°KÑ)?ØØ˜%Ò G¨uÒ$4Ø—‘˜g w°Ð<Õ=ð	>ô �uÓ"FÔGˆØ�‰�eÕðô �FÑ NÐX_Ô`€FØ*0Ö1 3ˆs�1‰v�s˜1‘vÒÐ1€FÐ1Ø€Mùò 2s   ÃC!c                   óB   — e Zd ZdZdefd„Zdeeeef   e	e   f   fd„Z
y)ÚSentencePieceExtractorzl
    Extractor implementation for SentencePiece trained models. https://github.com/google/sentencepiece
    Úmodelc                 ó  — t        | d«       t        | d«       t        «       }|j                  «       }t        |d«      5 }|j	                  |j                  «       «       d d d «       || _        y # 1 sw Y   || _        y xY w)NrN   rQ   Úrb)r   rY   Ú
ModelProtoÚopenÚParseFromStringÚreadÚproto)Úselfr‰   Ú	model_pb2ÚmÚfs        rX   Ú__init__zSentencePieceExtractor.__init__—   sl   € Ü˜$ Ô0Ü˜$ 
Ô+ô $Ó%ˆ	à× Ñ Ó"ˆÜ�%˜Óð 	( !Ø×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
ús   ¿ A/Á/A?r\   c           
      ó”  — | j                   j                  j                   |€/ddlm}m} | j                   j                  j                  dk(  r|n|}| j                   j                  D �cg c]  }|j                  |j                  f‘Œ }}|j                  dk7  r)| j                   j                  j                  |d<   ||d<   n;ddlm} t        |«      D ��	�
ci c]  \  }\  }	}
|	|“Œ }}	}}
 ||«      }||d<   ||d	<   t        | j                   j                  «      D ��cg c]0  \  }}|j                  d
v sŒ||j                  |j                  dk(  f‘Œ2 }}}t        |d„ ¬«      D ���cg c]  \  }}}t!        |d|¬«      ‘Œ c}}}|d<   t#        | j                   j$                  dd«      |d<   |S c c}w c c}
}	}w c c}}w c c}}}w )úÂ
        By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to
        order the merges with respect to the piece scores instead.
        Nr   )r   r   r   r   Úunk_idrl   )r†   r   ©é   é   rš   c                 ó   — | d   S ©Nr   rj   ©rk   s    rX   rm   z0SentencePieceExtractor.extract.<locals>.<lambda>Á   s
   € ÈQÈqÉT€ rZ   rn   F©Ú
normalizedÚspecialÚadditional_special_tokensÚprecompiled_charsmapÚ_spm_precompiled_charsmap)r�   Útrainer_specr˜   Útokenizers.modelsr   r   Ú
model_typeÚpiecesÚpieceÚscoreÚ__name__Útokenization_utils_baser†   Ú	enumerateÚtyper|   r   rb   Únormalizer_spec)r‘   r§   Úkwargsr   r   r©   rl   r†   ÚiÚwordrª   r   ÚidÚpÚspm_added_tokensÚtokenr¡   s                    rX   ÚextractzSentencePieceExtractor.extract£   s©  € ð
 	�
‰
×Ñ×&Ò&ØÐß6à$(§J¡J×$;Ñ$;×$FÑ$FÈ!Ò$K™ÐQTˆJØ9=¿¹×9JÑ9JÖK°�%—+‘+˜uŸ{™{Ò+ÐKˆÐKà×Ñ %Ò'Ø#Ÿz™z×6Ñ6×=Ñ=ˆF�8ÑØ#ˆF�7ŠOå@ä5>¸uÓ5E×FÐFÑ!1 ¡M T¨5�T˜1‘WÐFˆEÒFÙ$ UÓ+ˆFØ#ˆF�7‰OØ%ˆF�8Ñô ENÈdÏjÉj×N_ÑN_ÓD`×u¹5¸2¸qÐde×djÑdjÐntÒdt˜R §¡¨!¯&©&°A©+Ò6ÐuÐÑuô '-Ð-=Á>Ô&R÷/
ð /
á"��E˜7ô �u¨¸Ö@ô/
ˆÐ*Ñ+ô /6°d·j±j×6PÑ6PÐRhÐjnÓ.oˆÐ*Ñ+Øˆùò/ Lùô Gùó vùô/
s   Á*F1ÃF6Ä F=Ä7F=Å*GN)r«   Ú
__module__Ú__qualname__Ú__doc__Ústrr•   Útuplerx   ÚintÚlistr·   rj   rZ   rX   rˆ   rˆ   ’   s5   „ ñð
˜có 
ð!¨u°T¸#¸s¸(±^ÀTÈ%Á[Ð5PÑ/Qô !rZ   rˆ   c                   ó4   — e Zd Zddeeeef   ee   f   fd„Zy)ÚGemmaSentencePieceExtractorNr\   c                 óà   — | j                   }t        |j                  «       «      D �ci c]  }|j                  |«      |“Œ }}d|vr|j	                  d«      |d<   t        ||«      }||fS c c}w )r—   ú	ú<0x09>)Úsprz   ÚGetPieceSizeÚid_to_pieceÚgetr†   )r‘   r~   rÄ   rƒ   rl   r   s         rX   r·   z#GemmaSentencePieceExtractor.extractÈ   sr   € ð
 �W‰WˆÜ;@ÀÇÁÓARÓ;SÖT°%�—‘ Ó&¨Ñ-ÐTˆÐTð �uÑØŸ)™) HÓ-ˆE�$‰KÜ  ¨Ó5ˆØ�fˆ}Ðùò Us   ¨A+©N)	r«   r¸   r¹   r¼   rx   r»   r½   r¾   r·   rj   rZ   rX   rÀ   rÀ   Ç   s$   „ ñ¨E°$°s¸C°x±.À$ÀuÁ+Ð2MÑ,Nô rZ   rÀ   r©   c                 ó^   — t        | «      dk  xs | d   dk7  xs | d   j                  «        S )Nrq   éÿÿÿÿú,éþÿÿÿ)rr   Úisdigit)r©   s    rX   Úcheck_number_commarÎ   Ø   s3   € Üˆu‹:˜‰>ÒH˜U 2™Y¨#Ñ-ÒH°U¸2±Y×5FÑ5FÓ5HÐ1HÐHrZ   c                   ó   — e Zd Zd„ Zdefd„Zy)Ú	Converterc                 ó   — || _         y rÈ   )rc   )r‘   rc   s     rX   r•   zConverter.__init__Ý   s
   € Ø"4ˆÕrZ   r\   c                 ó   — t        «       ‚rÈ   )ÚNotImplementedError©r‘   s    rX   Ú	convertedzConverter.convertedà   s   € Ü!Ó#Ð#rZ   N)r«   r¸   r¹   r•   r   rÕ   rj   rZ   rX   rÐ   rÐ   Ü   s   „ ò5ð$˜9ô $rZ   rÐ   c                   ó   — e Zd Zdefd„Zy)ÚBertConverterr\   c           	      ól  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	�||f||	fg¬
«      |_        t1        j                  d¬«      |_        |S )N©Ú	unk_tokenFÚbasic_tokenizerT©Ú
clean_textÚhandle_chinese_charsÚstrip_accentsÚ	lowercaseú:0 $A:0 ú:0ú:0 $B:1 ú:1©ÚsingleÚpairÚspecial_tokensú##©Úprefix©rc   rl   r   r   r»   rÚ   ÚhasattrrÛ   Útokenize_chinese_charsrß   Údo_lower_caser
   ÚBertNormalizerÚ
normalizerr   ÚBertPreTokenizerÚpre_tokenizerÚ	cls_tokenÚ	sep_tokenÚcls_token_idÚsep_token_idr   ÚTemplateProcessingÚpost_processorr	   Údecoder©
r‘   rl   Ú	tokenizerrî   rß   rï   ÚclsÚseprö   r÷   s
             rX   rÕ   zBertConverter.convertedå   ó…  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐrZ   N©r«   r¸   r¹   r   rÕ   rj   rZ   rX   r×   r×   ä   ó   „ ð#˜9ô #rZ   r×   c                   ó   — e Zd Zdefd„Zy)ÚSplinterConverterr\   c           
      ó‚  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }t	        | j                   j&                  «      }d}	| j                   j(                  }
| j                   j*                  }| j                   j,                  }| j                   j/                  d«      }| j                   j0                  dk(  r|› d|› d	|	› d	|› d
|› d�
}n|› d|› d
|› d	|	› d	|› d�
}t3        j4                  |› d|› d�|||
f||f||f|	|fg¬«      |_        t9        j                  d¬«      |_        |S )NrÙ   FrÛ   TrÜ   ú.Úrightrá   ú rã   rä   râ   rå   ré   rê   )rc   rl   r   r   r»   rÚ   rí   rÛ   rî   rß   rï   r
   rð   rñ   r   rò   ró   rô   rõ   Úquestion_tokenrö   r÷   Úquestion_token_idÚconvert_tokens_to_idsÚpadding_sider   rø   rù   r	   rú   )r‘   rl   rü   rî   rß   rï   rý   rþ   ÚquestionÚdotrö   r÷   r	  Údot_token_idrç   s                  rX   rÕ   zSplinterConverter.converted  s"  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆÜ�t×.Ñ.×=Ñ=Ó>ˆØˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆØ ×3Ñ3×EÑEÐØ×.Ñ.×DÑDÀSÓIˆà×"Ñ"×/Ñ/°7Ò:Ø�U˜( 8 *¨A¨c¨U°!°C°5¸ÀÀÀRÐH‰Dà�U˜( 3 % x°¨z¸¸3¸%¸qÀÀÀRÐHˆDä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Øà�lÐ#Ø�lÐ#ØÐ,Ð-Ø�lÐ#ð	ô	$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐrZ   Nr   rj   rZ   rX   r  r    s   „ ð.˜9ô .rZ   r  c                   ó   — e Zd Zdefd„Zy)ÚFunnelConverterr\   c           	      ól  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	�||f||	fg¬
«      |_        t1        j                  d¬«      |_        |S )NrÙ   FrÛ   TrÜ   z:2 $A:0 râ   rã   rä   rå   ré   rê   rì   rû   s
             rX   rÕ   zFunnelConverter.converted>  rÿ   rZ   Nr   rj   rZ   rX   r  r  =  r  rZ   r  c                   ó   — e Zd Zdefd„Zy)ÚMPNetConverterr\   c                 ór  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	|› d
�||f||	fg¬«      |_        t1        j                  d¬«      |_        |S )NrÙ   FrÛ   TrÜ   rá   râ   z:0 rã   rä   rå   ré   rê   rì   rû   s
             rX   rÕ   zMPNetConverter.convertede  s‹  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   S¨¨¨X°c°U¸"Ð=à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐrZ   Nr   rj   rZ   rX   r  r  d  r  rZ   r  c                   ó   — e Zd Zdefd„Zy)ÚOpenAIGPTConverterr\   c           
      ó  — | j                   j                  }t        | j                   j                  j	                  «       «      }| j                   j
                  }t        t        ||d t        |«      dd¬«      «      }|j                  t        |«      «      �|j                  t        |«      g«       t        j                  d¬«      |_        t        j                  «       |_        t#        j$                  d¬«      |_        |S )Nú</w>F)rl   r   ÚdropoutrÚ   Úend_of_word_suffixÚfuse_unkT)rà   ©Úsuffix)rc   Úencoderr¾   Ú	bpe_ranksÚkeysrÚ   r   r   r»   Útoken_to_idÚadd_special_tokensr
   rð   rñ   r   rò   ró   r	   Ú
BPEDecoderrú   ©r‘   rl   r   rÚ   rü   s        rX   rÕ   zOpenAIGPTConverter.convertedŒ  sÔ   € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØÜ˜i›.Ø#)Øôó	
ˆ	ð × Ñ ¤ Y£Ó0Ð<Ø×(Ñ(¬#¨i«.Ð)9Ô:ä*×9Ñ9ÀDÔIˆ	ÔÜ"0×"AÑ"AÓ"Cˆ	ÔÜ$×/Ñ/°vÔ>ˆ	ÔàÐrZ   Nr   rj   rZ   rX   r  r  ‹  s   „ ð˜9ô rZ   r  c                   óH   — e Zd Zddeeef   dz  deeeef      dz  defd„Z	y)ÚGPT2ConverterNrl   r   r\   c           
      óN  — |s| j                   j                  }|st        | j                   j                  «      }t	        t        ||d ddd¬«      «      }t        | j                   dd«      }t        j                  |¬«      |_	        t        j                  «       |_        t        | j                   dd«      rT| j                   j                  }| j                   j                  }t        j                  |› d�|› d�||fg¬	«      |_        |S t        j                  d¬
«      |_        |S )NÚ F©rl   r   r  Úcontinuing_subword_prefixr  r  r[   ©r[   Úadd_bos_tokenz:0 $A:0z:0 $A:0 $B:1rå   ©Útrim_offsets)rc   r  r¾   r  r   r   rb   r   Ú	ByteLevelró   r	   rú   Ú	bos_tokenÚbos_token_idr   rø   rù   )r‘   rl   r   rü   r[   Úbosr1  s          rX   rÕ   zGPT2Converter.converted§  s  € ÙØ×+Ñ+×3Ñ3ˆEÙÜ˜$×1Ñ1×;Ñ;Ó<ˆFäÜØØØØ*,Ø#%Øôó	
ˆ	ô # 4×#:Ñ#:Ð<NÐPUÓVÐÜ"0×":Ñ":ÐL\Ô"]ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ�4×*Ñ*¨O¸UÔCØ×)Ñ)×3Ñ3ˆCØ×2Ñ2×?Ñ?ˆLÜ'1×'DÑ'DØ˜˜g�Ø�u˜LÐ)à˜,Ð'ð ô(ˆIÔ$ð Ðô (2×';Ñ';ÈÔ'OˆIÔ$ØÐrZ   ©NN©
r«   r¸   r¹   rx   r»   r½   r¾   r¼   r   rÕ   rj   rZ   rX   r&  r&  ¦  s@   „ ñ"˜t C¨ H™~°Ñ4ð "ÀTÈ%ÐPSÐUXÐPXÉ/ÑEZÐ]aÑEað "Ðmvô "rZ   r&  c                   ó   — e Zd Zdefd„Zy)ÚHerbertConverterr\   c           	      óŒ  — d}d}| j                   j                  }t        | j                   j                  j	                  «       «      }||d   d   v r|dd  }t        t        ||d | j                   j                  |¬«      «      }t        j                  dd¬«      |_
        t        j                  «       |_        t        j                  |¬«      |_        t#        j$                  | j                   j&                  | j                   j(                  f| j                   j*                  | j                   j,                  f¬	«      |_        |S )
Nz	#version:r  r   r   )r  rÚ   r  F)rà   rß   r  )rþ   rý   )rc   r  r¾   r  r   r   r   rÚ   r
   rð   rñ   r   rò   ró   r	   r#  rú   r   ÚBertProcessingrõ   r÷   rô   rö   rù   )r‘   Útokenizer_info_strÚtoken_suffixrl   r   rü   s         rX   rÕ   zHerbertConverter.convertedÍ  s  € Ø(ÐØˆà×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ ¨¡¨1¡Ñ-Ø˜A˜B�ZˆFäÜØØØØ×1Ñ1×;Ñ;Ø#/ôó
ˆ	ô  +×9Ñ9ÀEÐY^Ô_ˆ	ÔÜ"0×"AÑ"AÓ"Cˆ	ÔÜ$×/Ñ/°|ÔDˆ	ÔÜ#-×#<Ñ#<Ø×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYô$
ˆ	Ô ð
 ÐrZ   Nr   rj   rZ   rX   r6  r6  Ì  ó   „ ð˜9ô rZ   r6  c                   óH   — e Zd Zddeeef   dz  deeeef      dz  defd„Z	y)ÚQwen2ConverterNrl   r   r\   c                 ó0  — |s| j                   j                  }|s-t        | j                   j                  j	                  «       «      }t        t        ||d d dddd¬«      «      }t        j                  «       |_	        t        j                  t        j                  t        d«      dd¬«      t        j                  t        | j                   dd«      d¬«      g«      |_        t#        j                  «       |_        t'        j                  d¬	«      |_        |S )
Nr(  F)rl   r   r  rÚ   r*  r  r  Úbyte_fallbackzn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+Úisolated©ÚbehaviorÚinvertr[   ©r[   Ú	use_regexr-  )rc   r  r¾   r  r   r   r   r
   ÚNFCrñ   r   ÚSequenceÚSplitr   r/  rb   ró   r	   rú   r   rù   )r‘   rl   r   rü   s       rX   rÕ   zQwen2Converter.convertedì  s  € ÙØ×+Ñ+×3Ñ3ˆEÙÜ˜$×1Ñ1×;Ñ;×@Ñ@ÓBÓCˆFäÜØØØØØ*,Ø#%ØØ#ô	ó
ˆ	ô  +Ÿ™Ó0ˆ	Ôä"0×"9Ñ"9ä×$Ñ$Üð Nóð (Ø ôô ×(Ñ(Ü%,¨T×-DÑ-DÐFXÐZ_Ó%`Ø#ôðó#
ˆ	Ôô  %×.Ñ.Ó0ˆ	ÔÜ#-×#7Ñ#7ÀUÔ#Kˆ	Ô àÐrZ   r3  r4  rj   rZ   rX   r=  r=  ë  s@   „ ñ(˜t C¨ H™~°Ñ4ð (ÀTÈ%ÐPSÐUXÐPXÉ/ÑEZÐ]aÑEað (Ðmvô (rZ   r=  c                   ó   — e Zd Zdefd„Zy)ÚRobertaConverterr\   c           
      óÔ  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  |j                  ¬«      |_
        t        j                  «       |_        t        j                  |j                  |j                   f|j"                  |j$                  f|j                  d¬«      |_        |S )Nr(  Fr)  r+  T©rþ   rý   r[   r.  )rc   r  r¾   r  r   r   r   r   r/  r[   ró   r	   rú   r   ÚRobertaProcessingrõ   r÷   rô   rö   rù   ©r‘   Úotrl   r   rü   s        rX   rÕ   zRobertaConverter.converted  sÄ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#?Ñ#?Ø—‘˜rŸ™Ð/Ø—‘˜rŸ™Ð/Ø×0Ñ0Øô	$
ˆ	Ô ð ÐrZ   Nr   rj   rZ   rX   rJ  rJ    ó   „ ð˜9ô rZ   rJ  c                   ó   — e Zd Zdefd„Zy)ÚRoFormerConverterr\   c           	      óV  — ddl m} | j                  j                  }t	        t        |t        | j                  j                  «      ¬«      «      }d}d}t        | j                  d«      r@| j                  j                  j                  }| j                  j                  j                  }t        j                  dd||¬«      |_        t        j                   j#                   ||«      «      |_        t        | j                  j&                  «      }t        | j                  j(                  «      }| j                  j*                  }| j                  j,                  }	t/        j0                  |› d|› d	�|› d|› d
|› d�||f||	fg¬«      |_        t5        j
                  d¬«      |_        |S )Nr   )ÚJiebaPreTokenizerrÙ   FrÛ   TrÜ   rá   râ   rã   rä   rå   ré   rê   )Ú"models.roformer.tokenization_utilsrT  rc   rl   r   r   r»   rÚ   rí   rÛ   rß   rï   r
   rð   rñ   r   ÚPreTokenizerÚcustomró   rô   rõ   rö   r÷   r   rø   rù   r	   rú   )
r‘   rT  rl   rü   rß   rï   rý   rþ   rö   r÷   s
             rX   rÕ   zRoFormerConverter.converted5  sy  € ÝIà×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	àˆØˆÜ�4×*Ñ*Ð,=Ô>Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!&Ø'Ø#ô	 
ˆ	Ôô #1×"=Ñ"=×"DÑ"DÑEVÐW\ÓE]Ó"^ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐrZ   Nr   rj   rZ   rX   rR  rR  4  r  rZ   rR  c                   ó   — e Zd Zdefd„Zy)ÚDebertaConverterr\   c           
      óÖ  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  |j                  ¬«      |_
        t        j                  «       |_        t        j                  ddd| j                   j                  d«      fd| j                   j                  d«      fg¬	«      |_        |S )
Nr(  Fr)  r+  ú[CLS]:0 $A:0 [SEP]:0ú![CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1ú[CLS]ú[SEP]rå   )rc   r  r¾   r  r   r   r   r   r/  r[   ró   r	   rú   r   rø   r
  rù   rN  s        rX   rÕ   zDebertaConverter.converted\  sÕ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#@Ñ#@Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðô$
ˆ	Ô ð ÐrZ   Nr   rj   rZ   rX   rY  rY  [  r;  rZ   rY  c                   ó„   ‡ — e Zd ZdZeZi Zedd„«       Ze	dd„«       Z
ˆ fd„Zd„ Zd„ Zd„ Zd„ Zd	„ Zd
„ Zd„ Zdefd„Zˆ xZS )ÚSpmConverterFc           
      óZ  — | j                   j                  }| j                   j                  }| j                  j                  }t        |t        «      rt        t        ||xs g |d|d¬«      «      }nYt        |t        «      rH|rFt        |d   t        t        z  «      r,t        t        || j                   j                  |¬«      «      }nyt        j                  dd«      g}|r%|j                  dt        j                   |«      «       t        j"                  |«      |_        |rWt'        j"                  t'        j                  dd«      t'        j(                  «       t'        j*                  «       g«      |_        |S t'        j"                  t'        j                  dd«      g«      |_        |S )z÷
        Similar to convert_from_spm method, but used only when there is no `model_type` class, i.e. there is no matching class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracting stuff from the sentencepiece file
        TN)rl   r   rÚ   r  r?  r  r   )rl   r˜   r?  r  õ   â–�)r¥   r?  Ú	unk_piecer¯   r£   Ú
isinstancerx   r   r   r¾   r¼   r   r˜   r
   ÚReplaceÚinsertÚPrecompiledrG  rñ   r	   ÚByteFallbackÚFuserú   )r�   rl   r   r?  rc  r£   rü   Ú_normalizerss           rX   Úbuild_tokenizer_from_spm_protoz+SpmConverter.build_tokenizer_from_spm_proto  sr  € ð
 ×*Ñ*×8Ñ8ˆØ×&Ñ&×0Ñ0ˆ	Ø$×4Ñ4×IÑIÐô �eœTÔ"Ü!ÜØØ!š< RØ'Ø!Ø"/Ø ôó	‰Iô ˜œtÔ$©´:¸eÀA¹hÌÔPTÉÔ3UÜ!ÜØØ ×-Ñ-×4Ñ4Ø"/ôó‰Ið ô $×+Ñ+¨C°Ó7Ð8ˆÙØ×Ñ ¤;×#:Ñ#:Ð;OÓ#PÔQÜ*×3Ñ3°LÓAˆ	Ôñ Ü (× 1Ñ 1Ü×!Ñ! %¨Ó-¬x×/DÑ/DÓ/FÌÏÉËÐXó!ˆIÔð Ðô !)× 1Ñ 1´8×3CÑ3CÀEÈ3Ó3OÐ2PÓ QˆIÔàÐrZ   c                 ó   — |�||d<   |S )zŸ
        Hook used when converting directly from a SentencePiece model without a slow tokenizer instance.
        By default, return kwargs unchanged.
        rl   rj   )rý   rl   r°   s      rX   Úconvert_from_spmzSpmConverter.convert_from_spm¯  s   € ð ÐØ#ˆF�7‰OØˆrZ   c                 ó¦  •— t        | d«       t        ‰| �  |Ž  t        «       }|j	                  «       }t        | j                  j                  d«      5 }|j                  |j                  «       «       d d d «       || _
        | j                  j                  j                  r#| j                  st        j                  d«       y y y # 1 sw Y   ŒTxY w)NrQ   r‹   a  The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.)r   Úsuperr•   rY   rŒ   r�   rc   Ú
vocab_filerŽ   r�   r�   r¥   r?  Úhandle_byte_fallbackÚwarningsÚwarn)r‘   Úargsr’   r“   r”   Ú	__class__s        €rX   r•   zSpmConverter.__init__¹  s°   ø€ Ü˜$ 
Ô+ä‰Ñ˜$Ñô $Ó%ˆ	à× Ñ Ó"ˆÜ�$×)Ñ)×4Ñ4°dÓ;ð 	(¸qØ×Ñ˜aŸf™f›hÔ'÷	(àˆŒ
à�:‰:×"Ñ"×0Ò0¸×9RÒ9RÜ�M‰Mðeõð :SÐ0÷		(ð 	(ús   Á CÃCc                 ól   — |j                   D �cg c]  }|j                  |j                  f‘Œ c}S c c}w rÈ   ©r¨   r©   rª   ©r‘   r�   r©   s      rX   rl   zSpmConverter.vocabÎ  s'   € Ø8=¿¹ÖE¨u�—‘˜eŸk™kÒ*ÒEÐEùÒEs   �1c                 ó.   — |j                   j                  S rÈ   )r¥   r˜   ©r‘   r�   s     rX   r˜   zSpmConverter.unk_idÑ  s   € Ø×!Ñ!×(Ñ(Ð(rZ   c                 ó~  — |j                   j                  }| j                  |«      }|dk(  r1t        t	        || j                  |«      | j                  ¬«      «      }n£|dk(  r“| j                  | j                  j                  «      j                  |«      \  }}t        |«      D ���	ci c]  \  }\  }}	||“Œ }
}}}	t        t        |
||j                   j                  d| j                  d ¬«      «      }nt        d«      ‚t        |j                  «      D ��cg c]I  \  }}|j                   dv r6||j"                  |j                   dk(  xs |j"                  | j$                  v f‘ŒK }}}|j'                  t)        |d	„ ¬
«      D ���cg c]  \  }}}t+        |d|¬«      ‘Œ c}}}«       |S c c}	}}w c c}}w c c}}}w )Nr   ©r˜   r?  rq   T©rÚ   r  r?  r  z]You're trying to run a `Unigram` model but you're file was trained with a different algorithmr™   rš   c                 ó   — | d   S r�   rj   rž   s    rX   rm   z(SpmConverter.tokenizer.<locals>.<lambda>   ó   € ÐQRÐSTÑQU€ rZ   rn   FrŸ   )r¥   r§   rl   r   r   r˜   rq  ÚSpmExtractorrc   rp  r·   r­   r   rc  Ú	Exceptionr¨   r®   r©   rè   Ú
add_tokensr|   r   )r‘   r�   r§   r~   rü   Ú_r   r±   r²   rª   Ú	bpe_vocabr³   r´   rµ   r¶   r¡   s                   rX   rü   zSpmConverter.tokenizerÔ  s´  € Ø×'Ñ'×2Ñ2ˆ
Ø—z‘z %Ó(ˆà˜Š?Ü!ÜØ ØŸ;™; uÓ-Ø"&×";Ñ";ôó‰Ið ˜1Š_Ø×)Ñ)¨$×*AÑ*A×*LÑ*LÓM×UÑUÐVbÓc‰IˆAˆvÜ9BÀ<Ó9P×QÐQÑ%5 Q©¨¨u˜˜q™ÐQˆIÒQÜ!ÜØØØ#×0Ñ0×:Ñ:Ø!Ø"&×";Ñ";Ø ôó	‰Iô Øoóð ô # 5§<¡<Ó0÷
á��AØ�v‰v˜Ñð �—‘˜!Ÿ&™& A™+ÒG¨¯©°D×4GÑ4GÐ)GÒHð
Ðñ 
ð
 	×Ñô +1Ð1AÁ~Ô*V÷ð á&�B˜˜wô ˜5¨U¸GÖDôô	
ð ÐùôC Rùó*
ùôs   Â)F+ÄAF2ÆF8c                 ó   — |j                   j                  }t        j                  dd¬«      t        j                  t        d«      d«      g}|st        j                  |«      S t        j                  t        j                  |«      g|z   «      S )NFT)Úleftr  ú {2,}rb  )r¯   r£   r
   ÚStripre  r   rG  rg  ©r‘   r�   r£   rj  s       rX   rñ   zSpmConverter.normalizer  s{   € Ø$×4Ñ4×IÑIÐä×Ñ 5°Ô5Ü×Ñ¤ g£°Ó6ð
ˆñ $Ü×'Ñ'¨Ó5Ð5ä×'Ñ'¬×)@Ñ)@ÐAUÓ)VÐ(WÐZfÑ(fÓgÐgrZ   c                 ó\   — t        || j                  «      }t        j                  ||¬«      S ©N©Úreplacementrd   )re   rc   r   Ú	Metaspace©r‘   r�  r[   rd   s       rX   ró   zSpmConverter.pre_tokenizer  s)   € Ü,Ð-=¸t×?VÑ?VÓWˆÜ×'Ñ'°KÐP^Ô_Ð_rZ   c                  ó   — y rÈ   rj   rÔ   s    rX   rù   zSpmConverter.post_processor  ó   € ØrZ   c                 ó\   — t        || j                  «      }t        j                  ||¬«      S r‹  )re   rc   r	   rŽ  r�  s       rX   rú   zSpmConverter.decoder  s(   € Ü,Ð-=¸t×?VÑ?VÓWˆÜ×!Ñ!¨kÈ.ÔYÐYrZ   r\   c                 óz  — | j                  | j                  «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j
                  }| j                  ||«      }|�||_        | j                  ||«      |_        | j                  «       }|r||_        |S )Nrb  Tr[   )	rü   r�   rñ   rí   rc   r[   ró   rú   rù   )r‘   rü   rñ   r�  r[   ró   rù   s          rX   rÕ   zSpmConverter.converted  s¸   € Ø—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#à ŸL™L¨Ð6FÓGˆ	ÔØ×,Ñ,Ó.ˆÙØ'5ˆIÔ$àÐrZ   rÈ   )r«   r¸   r¹   rq  rˆ   r€  rè   Ústaticmethodrk  Úclassmethodrm  r•   rl   r˜   rü   rñ   ró   rù   rú   r   rÕ   Ú__classcell__)ru  s   @rX   r`  r`  z  ss   ø„ Ø ÐØ)€LØ€Nàò-ó ð-ð^ òó ðôò*Fò)ò0òd	hò`òòZð˜9÷ rZ   r`  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚAlbertConverterc                 óÌ   — |j                   D �cg c]J  }t        |j                  «      r|j                  |j                  fn|j                  |j                  dz
  f‘ŒL c}S c c}w ©Néd   ©r¨   rÎ   r©   rª   rx  s      rX   rl   zAlbertConverter.vocab6  ó^   € ð Ÿ™ö
àô +=¸U¿[¹[Ô*IˆU�[‰[˜%Ÿ+™+Ñ&ÐPU×P[ÑP[Ð]b×]hÑ]hÐknÑ]nÐOoÑoò
ð 	
ùò 
ó   �AA!c                 ó‚  — t        j                  dd«      t        j                  dd«      g}| j                  j                  sF|j	                  t        j
                  «       «       |j	                  t        j                  «       «       | j                  j                  r#|j	                  t        j                  «       «       |j                  j                  }|r$|j	                  t        j                  |«      «       |j	                  t        j                  t        d«      d«      «       t        j                  |«      S ©Nz``ú"z''r‡  r  ©r
   re  rc   Úkeep_accentsr{   ÚNFKDÚStripAccentsrï   Ú	Lowercaser¯   r£   rg  r   rG  ©r‘   r�   Úlist_normalizersr£   s       rX   rñ   zAlbertConverter.normalizer<  óï   € ä×Ñ  cÓ*Ü×Ñ  cÓ*ð
Ðð ×&Ñ&×3Ò3Ø×#Ñ#¤K×$4Ñ$4Ó$6Ô7Ø×#Ñ#¤K×$<Ñ$<Ó$>Ô?Ø×"Ñ"×0Ò0Ø×#Ñ#¤K×$9Ñ$9Ó$;Ô<à$×4Ñ4×IÑIÐáØ×#Ñ#¤K×$;Ñ$;Ð<PÓ$QÔRà×Ñ¤× 3Ñ 3´E¸'³NÀCÓ HÔIÜ×#Ñ#Ð$4Ó5Ð5rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S ©Nr[  r\  r]  r^  rå   ©r   rø   rc   r
  rÔ   s    rX   rù   zAlbertConverter.post_processorO  óR   € Ü×,Ñ,Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðô
ð 	
rZ   N©r«   r¸   r¹   rl   rñ   rù   rj   rZ   rX   r˜  r˜  5  ó   „ ò
ò6ó&
rZ   r˜  c                   ó   — e Zd Zd„ Zd„ Zy)ÚBarthezConverterc                 ó
   — d}|S ©Nrš   rj   ©r‘   r�   r˜   s      rX   r˜   zBarthezConverter.unk_id[  ó   € ØˆØˆrZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S ©Nz<s> $A </s>z<s> $A </s> </s> $B </s>ú<s>ú</s>rå   r¬  rÔ   s    rX   rù   zBarthezConverter.post_processor_  óR   € Ü×,Ñ,Ø Ø+à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
rZ   N)r«   r¸   r¹   r˜   rù   rj   rZ   rX   r±  r±  Z  s   „ òó
rZ   r±  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zedd„«       Zy)ÚCamembertConverterc                 ó�   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|dgz  }|S c c}w )N)©z
<s>NOTUSEDç        ©ú<pad>r¿  ©z</s>NOTUSEDr¿  ©ú<unk>r¿  )ú<unk>NOTUSEDiœÿÿÿr   ©ú<mask>r¿  rw  ©r‘   r�   rl   r©   s       rX   rl   zCamembertConverter.vocabk  sP   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØ�/Ð"Ñ"ˆØˆùò Ló   —Ac                  ó   — yr³  rj   rz  s     rX   r˜   zCamembertConverter.unk_idx  s   € àrZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S r·  r¬  rÔ   s    rX   rù   z!CamembertConverter.post_processor|  rº  rZ   Nc                 ó,  — t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }d|dfd	|dfd
g}|�|j                  t        |«      dd  «       |j	                  |df«       ||d<   |S )NÚ	pad_tokenrÁ  rÚ   rÄ  Ú
mask_tokenrÇ  r¾  r¿  rÂ  )rÅ  ç      YÀr   rl   ©r»   rÇ   r}   r¾   r{   )rý   rl   r°   rÍ  rÚ   rÎ  Ú
vocab_lists          rX   rm  z#CamembertConverter.convert_from_spm†  s¤   € ä˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð  Ø˜ÐØ Ø˜ÐØ$ð
ˆ
ð ÐØ×Ñœd 5›k¨!¨"˜oÔ.Ø×Ñ˜: sÐ+Ô,Ø$ˆˆw‰ØˆrZ   rÈ   ©r«   r¸   r¹   rl   r˜   rù   r•  rm  rj   rZ   rX   r¼  r¼  j  s%   „ òòò
ð òó ñrZ   r¼  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚDebertaV2Converterc                 ó  — g }| j                   j                  r%|j                  t        j                  d¬«      «       t        || j                   «      }|j                  t        j                  ||¬«      «       t        j                  |«      S )Nr@  )rB  rŒ  )rc   Úsplit_by_punctr{   r   ÚPunctuationre   rŽ  rG  )r‘   r�  r[   Úlist_pretokenizersrd   s        rX   ró   z DebertaV2Converter.pre_tokenizer›  sq   € ØÐØ×"Ñ"×1Ò1Ø×%Ñ%¤n×&@Ñ&@È*Ô&UÔVÜ,Ð-=¸t×?VÑ?VÓWˆØ×!Ñ!¤.×":Ñ":À{ÐcqÔ"rÔsÜ×&Ñ&Ð'9Ó:Ð:rZ   c                 ó¼  — g }| j                   j                  r#|j                  t        j                  «       «       |j                  t        j
                  «       «       |j                  j                  }|r$|j                  t        j                  |«      «       |j                  t        j                  t        d«      d«      «       t        j                  |«      S )Nr‡  r  )rc   rï   r{   r
   r¦  rˆ  r¯   r£   rg  re  r   rG  r§  s       rX   rñ   zDebertaV2Converter.normalizer£  s¥   € ØÐØ×"Ñ"×0Ò0Ø×#Ñ#¤K×$9Ñ$9Ó$;Ô<Ø×Ñ¤× 1Ñ 1Ó 3Ô4à$×4Ñ4×IÑIÐÙØ×#Ñ#¤K×$;Ñ$;Ð<PÓ$QÔRØ×Ñ¤× 3Ñ 3´E¸'³NÀCÓ HÔIä×#Ñ#Ð$4Ó5Ð5rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S r«  r¬  rÔ   s    rX   rù   z!DebertaV2Converter.post_processor°  r­  rZ   N)r«   r¸   r¹   ró   rñ   rù   rj   rZ   rX   rÔ  rÔ  š  s   „ ò;ò6ó
rZ   rÔ  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zedd„«       Zy)ÚMBartConverterc                 óž   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|g d¢z  }|dgz  }|S c c}w )N©©r¸  r¿  rÀ  ©r¹  r¿  rÃ  rš   )©r   r¿  ©r   r¿  ©r   r¿  ©r   r¿  ©r   r¿  ©r   r¿  ©r   r¿  ©r   r¿  ©r   r¿  ©r    r¿  ©r!   r¿  ©r"   r¿  ©r#   r¿  ©r$   r¿  ©r%   r¿  ©r&   r¿  ©r'   r¿  ©r(   r¿  ©r)   r¿  ©r*   r¿  ©r+   r¿  ©r,   r¿  ©r-   r¿  ©r.   r¿  ©r/   r¿  rÆ  rw  rÈ  s       rX   rl   zMBartConverter.vocab¼  sa   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØò 
ñ 	
ˆð6 	�/Ð"Ñ"ˆØˆùò; Ló   —A
c                  ó   — yr³  rj   rz  s     rX   r˜   zMBartConverter.unk_idâ  ó   € ØrZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz$A </s> en_XXz$A $B </s> en_XXr   r¹  rå   r¬  rÔ   s    rX   rù   zMBartConverter.post_processorå  óR   € Ü×,Ñ,Ø"Ø#à˜$×1Ñ1×GÑGÈÓPÐQØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
rZ   Nc                 óÖ  — t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  d	d
«      «      }|df|df|df|dfg}|�|j                  t        |«      dd  «       |j                  d„ t        D «       «       |j                  |df«       ||d<   |S )Nr0  r¸  rÍ  rÁ  Ú	eos_tokenr¹  rÚ   rÄ  rÎ  rÇ  r¿  rš   c              3   ó$   K  — | ]  }|d f–— Œ
 y­w©r¿  Nrj   ©Ú.0Ú	lang_codes     rX   ú	<genexpr>z2MBartConverter.convert_from_spm.<locals>.<genexpr>ÿ  s   è ø€ ÒL¨y˜9 cÔ*ÑLùó   ‚rl   )r»   rÇ   r}   r¾   ÚMBART_LANGUAGESr{   ©	rý   rl   r°   r0  rÍ  r   rÚ   rÎ  rÑ  s	            rX   rm  zMBartConverter.convert_from_spmï  sé   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÐØ×Ñœd 5›k¨!¨"˜oÔ.Ø×ÑÑL¼OÔLÔLØ×Ñ˜: sÐ+Ô,Ø$ˆˆw‰ØˆrZ   rÈ   rÒ  rj   rZ   rX   rÜ  rÜ  »  s&   „ ò$òLò
ð òó ñrZ   rÜ  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zedd„«       Zy)ÚMBart50Converterc                 óž   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|g d¢z  }|dgz  }|S c c}w )NrÞ  rš   )4rá  râ  rã  rä  rå  ræ  rç  rè  ré  rê  rë  rì  rí  rî  rï  rð  rñ  rò  ró  rô  rõ  rö  r÷  rø  rù  )r0   r¿  )r1   r¿  )r2   r¿  )r3   r¿  )r4   r¿  )r5   r¿  )r6   r¿  )r7   r¿  )r8   r¿  )r9   r¿  )r:   r¿  )r;   r¿  )r<   r¿  )r=   r¿  )r>   r¿  )r?   r¿  )r@   r¿  )rA   r¿  )rB   r¿  )rC   r¿  )rD   r¿  )rE   r¿  )rF   r¿  )rG   r¿  )rH   r¿  )rI   r¿  )rJ   r¿  rÆ  rw  rÈ  s       rX   rl   zMBart50Converter.vocab  sa   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØò  Rñ  	RˆØ�/Ð"Ñ"ˆØˆùò Lrú  c                  ó   — yr³  rj   rz  s     rX   r˜   zMBart50Converter.unk_id  rü  rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nzen_XX $A </s>zen_XX $A $B </s>r   r¹  rå   r¬  rÔ   s    rX   rù   zMBart50Converter.post_processor  rþ  rZ   Nc                 óÖ  — t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  d	d
«      «      }|df|df|df|dfg}|�|j                  t        |«      dd  «       |j                  d„ t        D «       «       |j                  |df«       ||d<   |S )Nrô   r¸  rÍ  rÁ  r   r¹  rÚ   rÄ  rÎ  rÇ  r¿  rš   c              3   ó$   K  — | ]  }|d f–— Œ
 y­wr  rj   r  s     rX   r  z4MBart50Converter.convert_from_spm.<locals>.<genexpr>/  s   è ø€ ÒN¨y˜9 cÔ*ÑNùr  rl   )r»   rÇ   r}   r¾   ÚMBART50_LANGUAGESr{   )	rý   rl   r°   rô   rÍ  r   rÚ   rÎ  rÑ  s	            rX   rm  z!MBart50Converter.convert_from_spm  sê   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÐØ×Ñœd 5›k¨!¨"˜oÔ.Ø×ÑÑNÔ<MÔNÔNØ×Ñ˜: sÐ+Ô,Ø$ˆˆw‰ØˆrZ   rÈ   rÒ  rj   rZ   rX   r  r    s%   „ ò
òò
ð òó ñrZ   r  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zedd„«       Zy)ÚNllbConverterc                 ó„   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w )NrÞ  rš   rw  rÈ  s       rX   rl   zNllbConverter.vocab6  óC   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØˆùò Ló   —=c                  ó   — yr³  rj   rz  s     rX   r˜   zNllbConverter.unk_id@  rü  rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nzeng_Latn $A </s>zeng_Latn $A $B </s>Úeng_Latnr¹  rå   r¬  rÔ   s    rX   rù   zNllbConverter.post_processorC  sR   € Ü×,Ñ,Ø%Ø&à˜T×4Ñ4×JÑJÈ:ÓVÐWØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
rZ   Nc                 óª  — t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }|d	|d
|d|di}|�Ot        |t        «      r|j	                  «       n|D ��	cg c]  \  }}	|‘Œ	 c}	}}
|
D ]  }||v rŒt        |«      ||<   Œ ||d<   |S c c}	}w )Nr0  r¸  rÍ  rÁ  r   r¹  rÚ   rÄ  r   r   rq   rš   rl   )r»   rÇ   rd  rx   r   rr   )rý   rl   r°   r0  rÍ  r   rÚ   Úreordered_vocabÚtokrƒ  Útokensr¶   s               rX   rm  zNllbConverter.convert_from_spmM  sâ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	ð �qØ�qØ�qØ�qð	
ˆð ÐÜ%/°´tÔ%<�U—Z‘Z”\ÐUZ×B[É6È3ÐPQÂ3ÓB[ˆFØò >�Ø˜OÑ+ØÜ),¨_Ó)=� Ò&ð>ð *ˆˆw‰Øˆùó C\s   ÂCrÈ   rÒ  rj   rZ   rX   r  r  5  s%   „ òòò
ð òó ñrZ   r  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚSeamlessM4TConverterc                 ó„   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w )N)rÀ  rÃ  rß  rà  rš   rw  rÈ  s       rX   rl   zSeamlessM4TConverter.vocabe  r  r  c                 ó.   — | j                   j                  S rÈ   )rc   Úunk_token_idrz  s     rX   r˜   zSeamlessM4TConverter.unk_ido  s   € Ø×&Ñ&×3Ñ3Ð3rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz__eng__ $A </s>z__eng__ $A $B </s>Ú__eng__r¹  rå   r¬  rÔ   s    rX   rù   z#SeamlessM4TConverter.post_processorr  sR   € Ü×,Ñ,Ø$Ø%à˜D×3Ñ3×IÑIÈ)ÓTÐUØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
rZ   N©r«   r¸   r¹   rl   r˜   rù   rj   rZ   rX   r  r  d  s   „ òò4ó
rZ   r  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zedd„«       Zy)ÚXLMRobertaConverterc                 ó�   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|dgz  }|S c c}w )NrÞ  rš   rÆ  rw  rÈ  s       rX   rl   zXLMRobertaConverter.vocab~  sP   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØ�/Ð"Ñ"ˆØˆùò LrÉ  c                 ó
   — d}|S r³  rj   r´  s      rX   r˜   zXLMRobertaConverter.unk_id‰  rµ  rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S r·  r¬  rÔ   s    rX   rù   z"XLMRobertaConverter.post_processor�  rº  rZ   Nc                 óž  — t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  d	d
«      «      }|df|df|df|dfg}|�|j                  t        |«      dd  «       |j	                  |df«       ||d<   |S )Nr0  r¸  rÍ  rÁ  r   r¹  rÚ   rÄ  rÎ  rÇ  r¿  rš   rl   rÐ  r	  s	            rX   rm  z$XLMRobertaConverter.convert_from_spm—  sÕ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÐØ×Ñœd 5›k¨!¨"˜oÔ.Ø×Ñ˜: sÐ+Ô,Ø$ˆˆw‰ØˆrZ   rÈ   rÒ  rj   rZ   rX   r'  r'  }  s%   „ ò	òò
ð òó ñrZ   r'  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚXLNetConverterc                 óÌ   — |j                   D �cg c]J  }t        |j                  «      r|j                  |j                  fn|j                  |j                  dz
  f‘ŒL c}S c c}w rš  rœ  rx  s      rX   rl   zXLNetConverter.vocab­  r�  rž  c                 ó‚  — t        j                  dd«      t        j                  dd«      g}| j                  j                  sF|j	                  t        j
                  «       «       |j	                  t        j                  «       «       | j                  j                  r#|j	                  t        j                  «       «       |j                  j                  }|r$|j	                  t        j                  |«      «       |j	                  t        j                  t        d«      d«      «       t        j                  |«      S r   r¢  r§  s       rX   rñ   zXLNetConverter.normalizer³  r©  rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz$A:0 <sep>:0 <cls>:2z!$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2z<sep>z<cls>rå   r¬  rÔ   s    rX   rù   zXLNetConverter.post_processorÆ  r­  rZ   Nr®  rj   rZ   rX   r-  r-  ¬  r¯  rZ   r-  c                   ó   — e Zd Zy)ÚReformerConverterN©r«   r¸   r¹   rj   rZ   rX   r2  r2  Ñ  ó   „ ØrZ   r2  c                   ó   — e Zd Zd„ Zd„ Zy)ÚRemBertConverterc                 ób  — t        j                  dd«      t        j                  dd«      t        j                  t        d«      d«      g}| j                  j                  sF|j                  t        j                  «       «       |j                  t        j                  «       «       | j                  j                  r#|j                  t        j                  «       «       |j                  j                  }|r$|j                  t        j                  |«      «       t        j                  |«      S r   )r
   re  r   rc   r£  r{   r¤  r¥  rï   r¦  r¯   r£   rg  rG  r§  s       rX   rñ   zRemBertConverter.normalizer×  sä   € ä×Ñ  cÓ*Ü×Ñ  cÓ*Ü×Ñ¤ g£°Ó4ð
Ðð
 ×&Ñ&×3Ò3Ø×#Ñ#¤K×$4Ñ$4Ó$6Ô7Ø×#Ñ#¤K×$<Ñ$<Ó$>Ô?Ø×"Ñ"×0Ò0Ø×#Ñ#¤K×$9Ñ$9Ó$;Ô<à$×4Ñ4×IÑIÐáØ×#Ñ#¤K×$;Ñ$;Ð<PÓ$QÔRä×#Ñ#Ð$4Ó5Ð5rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S r«  r¬  rÔ   s    rX   rù   zRemBertConverter.post_processorê  r­  rZ   N)r«   r¸   r¹   rñ   rù   rj   rZ   rX   r6  r6  Õ  s   „ ò6ó&
rZ   r6  c                   ó   — e Zd Zy)ÚBertGenerationConverterNr3  rj   rZ   rX   r:  r:  õ  r4  rZ   r:  c                   ó6   — e Zd Zd„ Zedd„«       Zd„ Zd„ Zd„ Zy)ÚPegasusConverterc                 óv  — | j                   j                  df| j                   j                  dfg}| j                   j                  �|| j                   j                  dfgz  }| j                   j                  �I| j                   j
                  | j                   j                  k  r|| j                   j                  dfgz  }|t        d| j                   j                  «      D �cg c]
  }d|› d�df‘Œ c}z  }||j                  dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w c c}w )Nr¿  rq   ú<unk_ú>rÏ  )rc   rÍ  r   Úmask_token_sentrÎ  Úmask_token_idÚoffsetrz   r¨   r©   rª   )r‘   r�   rl   r±   r©   s        rX   rl   zPegasusConverter.vocabú  s%  € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 ×"Ñ"×2Ñ2Ð>Ø�t×.Ñ.×>Ñ>ÀÐDÐEÑEˆEð ×#Ñ#×.Ñ.Ð:Ø×'Ñ'×5Ñ5¸×8OÑ8O×8VÑ8VÒVà�t×.Ñ.×9Ñ9¸3Ð?Ð@Ñ@ˆEà´%¸¸4×;RÑ;R×;YÑ;YÓ2ZÖ[¨Q�U˜1˜#˜Q�< Ò(Ò[Ñ[ˆØ¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØˆùò \ùÚKs   Ã%D1ÄD6Nc           	      ó(  — t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }|d	f|d	fg}|d
k7  r|j                  |d	f«       |d
k7  r||k7  r|j                  |d	f«       |j                  t	        d|j                  dd«      «      D �cg c]
  }d|› d�df‘Œ c}«       |�|j                  t        |«      dd  «       ||d<   |S c c}w )NrÍ  rÁ  r   r¹  rÎ  z<mask_1>r@  z<mask_2>r¿  ÚNonerq   rB  ég   r>  r?  rÏ  rl   )r»   rÇ   r{   r}   rz   r¾   )	rý   rl   r°   rÍ  r   rÎ  r@  rÑ  r±   s	            rX   rm  z!PegasusConverter.convert_from_spm  s  € ä˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ™ L°*Ó=Ó>ˆ
Ü˜fŸj™jÐ):¸JÓGÓHˆð ˜ÐØ˜Ðð
ˆ
ð ˜ÒØ×Ñ˜z¨3Ð/Ô0Ø˜fÒ$¨¸JÒ)FØ×Ñ˜°Ð4Ô5à×Ñ¼5ÀÀFÇJÁJÈxÐY\ÓD]Ó;^Ö_°a˜e A 3 a˜L¨&Ò1Ò_Ô`ØÐØ×Ñœd 5›k¨!¨"˜oÔ.Ø$ˆˆw‰Øˆùò	 `s   ÃDc                 ó\   — |j                   j                  | j                  j                  z   S rÈ   )r¥   r˜   rc   rB  rz  s     rX   r˜   zPegasusConverter.unk_id#  s%   € Ø×!Ñ!×(Ñ(¨4×+BÑ+B×+IÑ+IÑIÐIrZ   c                 óª   — t        || j                  «      }t        j                  t        j                  «       t        j
                  ||¬«      g«      S r‹  )re   rc   r   rG  ÚWhitespaceSplitrŽ  r�  s       rX   ró   zPegasusConverter.pre_tokenizer&  sJ   € Ü,Ð-=¸t×?VÑ?VÓWˆÜ×&Ñ&ä×.Ñ.Ó0Ü×(Ñ(°[ÐQ_Ô`ðó
ð 	
rZ   c                 óš   — | j                   j                  }|| j                   j                  fg}t        j                  d|gdd|g|¬«      S )Nú$Aú$Brå   )rc   r   Úeos_token_idr   rø   )r‘   Úeosrè   s      rX   rù   zPegasusConverter.post_processor/  sR   € Ø×%Ñ%×/Ñ/ˆà�$×)Ñ)×6Ñ6Ð7ð
ˆô ×,Ñ,°T¸3°KÀtÈTÐSVÐFWÐhvÔwÐwrZ   rÈ   )	r«   r¸   r¹   rl   r•  rm  r˜   ró   rù   rj   rZ   rX   r<  r<  ù  s,   „ òð& òó ðò*Jò
óxrZ   r<  c                   ó*   — e Zd Zd„ Zd„ Zedd„«       Zy)ÚT5Converterc                 óò   — | j                   j                  }|j                  D �cg c]  }|j                  |j                  f‘Œ }}|t        |dz
  dd«      D �cg c]
  }d|› d�df‘Œ c}z  }|S c c}w c c}w )Nr   rÊ   ú
<extra_id_r?  r¿  )rc   Ú
_extra_idsr¨   r©   rª   rz   )r‘   r�   Únum_extra_idsr©   rl   r±   s         rX   rl   zT5Converter.vocab8  sw   € Ø×/Ñ/×:Ñ:ˆØ9>¿¹ÖF°�%—+‘+˜uŸ{™{Ò+ÐFˆÐFØ´E¸-È!Ñ:KÈRÐQSÓ4TÖU¨q�Z ˜s !Ð$ cÒ*ÒUÑUˆØˆùò GùÚUs   ¥A/ÁA4c                 ór   — t        j                  ddgg d¢d| j                  j                  d«      fg¬«      S ©NrJ  r¹  )rJ  r¹  rK  r¹  rå   r¬  rÔ   s    rX   rù   zT5Converter.post_processor>  ó=   € Ü×,Ñ,Ø˜&�>Ú-à˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
rZ   Nc                 óò   — |j                  dd«      }t        |dz
  dd«      D �cg c]  }d|› d�‘Œ
 }}|�t        |«      ng }|j                  d„ |D «       «       |j	                  d|«       ||d	<   |S c c}w )
NÚ	extra_idsr›  r   rÊ   rQ  r?  c              3   ó$   K  — | ]  }|d f–— Œ
 y­wr  rj   )r  r¶   s     rX   r  z/T5Converter.convert_from_spm.<locals>.<genexpr>L  s   è ø€ ÒA¨5˜5 #œ,ÑAùr  r¢   rl   )rÇ   rz   r¾   r}   Ú
setdefault)rý   rl   r°   rX  r±   Úextra_tokensrÑ  s          rX   rm  zT5Converter.convert_from_spmG  s„   € à—J‘J˜{¨CÓ0ˆ	Ü38¸ÀQ¹ÈÈBÓ3OÖP¨a˜* Q C qÒ)ÐPˆÐPØ$)Ð$5”T˜%”[¸2ˆ
Ø×ÑÑA°LÔAÔAà×ÑÐ5°|ÔDØ$ˆˆw‰Øˆùò Qs   ¥A4rÈ   )r«   r¸   r¹   rl   rù   r•  rm  rj   rZ   rX   rO  rO  7  s    „ òò
ð òó ñrZ   rO  c                   ó   — e Zd Zd„ Zy)ÚUdopConverterc                 ór   — t        j                  ddgg d¢d| j                  j                  d«      fg¬«      S rU  r¬  rÔ   s    rX   rù   zUdopConverter.post_processorT  rV  rZ   N©r«   r¸   r¹   rù   rj   rZ   rX   r]  r]  S  s   „ ó
rZ   r]  c                   ó   — e Zd Zd„ Zy)ÚVideoPrismConverterc                  ó   — y rÈ   rj   rÔ   s    rX   rù   z"VideoPrismConverter.post_processor_  r‘  rZ   Nr_  rj   rZ   rX   ra  ra  ^  s   „ órZ   ra  c                   ó   — e Zd Zdefd„Zy)ÚWhisperConverterr\   c           
      óÈ  — | j                   j                  }t        | j                   j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  | j                   j                  ¬«      |_
        t        j                  «       |_        | j                   j                  }| j                   j                  |«      }| j                   j                  }| j                   j                   }dj#                  |D �cg c]  }|› d�‘Œ	 c}«      }	t%        j&                  |	› d|› d�|	› d|› d	�||fgt)        ||«      ¢¬
«      |_        |S c c}w )Nr(  Fr)  r+  r  râ   z $A:0 z $A:0 $B:1 rä   rå   )rc   r  r¾   r  r   r   r   r   r/  r[   ró   r	   rú   Úprefix_tokensÚconvert_ids_to_tokensr   rL  Újoinr   rø   Úziprù   )
r‘   rl   r   rü   Úprefix_token_idsÚprefixesrM  rL  r¶   Úprefix_templates
             rX   rÕ   zWhisperConverter.convertedd  sR  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈD×LcÑLc×LtÑLtÔ"uˆ	ÔÜ$×.Ñ.Ó0ˆ	Ôà×2Ñ2×@Ñ@ÐØ×*Ñ*×@Ñ@ÐAQÓRˆØ×%Ñ%×/Ñ/ˆØ×.Ñ.×;Ñ;ˆØŸ(™(¸hÖ#G°U u g¨R¢LÒ#GÓHˆÜ#-×#@Ñ#@Ø%Ð& f¨S¨E°Ð4Ø#Ð$ K°¨u°BÐ7à�lÐ#ðä�XÐ/Ó0ðô$
ˆ	Ô ð Ðùò $Hs   ÄENr   rj   rZ   rX   rd  rd  c  s   „ ð ˜9ô  rZ   rd  c                   ó   — e Zd Zd„ Zy)ÚBigBirdConverterc           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S r«  r¬  rÔ   s    rX   rù   zBigBirdConverter.post_processorˆ  r­  rZ   Nr_  rj   rZ   rX   rn  rn  ‡  s   „ ó
rZ   rn  c                   ó   — e Zd Zdefd„Zy)ÚCLIPConverterr\   c                 óp  — | j                   j                  }t        | j                   j                  j	                  «       «      }| j                   j
                  }t        t        ||d dddt        |«      ¬«      «      }t        j                  t        j                  «       t        j                  t        d«      d«      t        j                  «       g«      |_        t!        j                  t!        j"                  t        d«      dd	¬
«      t!        j$                  d¬«      g«      |_        t)        j$                  «       |_        t-        j.                  | j                   j0                  | j                   j2                  f| j                   j4                  | j                   j6                  fdd¬«      |_        |S )Nr(  r  F©rl   r   r  r*  r  r  rÚ   z\s+r  z9's|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+ÚremovedTrA  r+  rL  )rc   r  r¾   r  r   rÚ   r   r   r»   r
   rG  rF  re  r   r¦  rñ   r   rH  r/  ró   r	   rú   r   rM  r   rL  r0  r1  rù   r$  s        rX   rÕ   zCLIPConverter.converted”  sk  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØØ*,Ø#)ØÜ˜i›.ôó

ˆ	ô  +×3Ñ3Ü�_‰_Ó¤× 3Ñ 3´E¸&³MÀ3Ó GÌ×I^ÑI^ÓI`Ðaó 
ˆ	Ôô #1×"9Ñ"9ä×$Ñ$ÜÐZÓ[Ø&Øôô
 ×(Ñ(¸%Ô@ðó	#
ˆ	Ôô %×.Ñ.Ó0ˆ	Ôô $.×#?Ñ#?Ø×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ"Øô	$
ˆ	Ô ð ÐrZ   Nr   rj   rZ   rX   rq  rq  “  s   „ ð'˜9ô 'rZ   rq  c                   ó   — e Zd Zdefd„Zy)ÚLayoutLMv2Converterr\   c           	      ól  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	�||f||	fg¬
«      |_        t1        j                  d¬«      |_        |S )NrÙ   FTrÛ   rÜ   rá   râ   rã   rä   rå   ré   rê   rì   rû   s
             rX   rÕ   zLayoutLMv2Converter.converted¿  s…  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐrZ   Nr   rj   rZ   rX   rv  rv  ¾  r  rZ   rv  c                   ó   — e Zd Zdefd„Zy)ÚBlenderbotConverterr\   c           
      ó®  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  |j                  ¬«      |_
        t        j                  «       |_        t        j                  d|j                  › d�|j                  |j                   fg¬«      |_        |S )Nr(  Fr)  r+  z$A:0 râ   )ræ   rè   )rc   r  r¾   r  r   r   r   r   r/  r[   ró   r	   rú   r   rø   r   rL  rù   rN  s        rX   rÕ   zBlenderbotConverter.convertedæ  s½   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#@Ñ#@Ø˜2Ÿ<™<˜.¨Ð+à—‘˜rŸ™Ð/ðô$
ˆ	Ô ð ÐrZ   Nr   rj   rZ   rX   ry  ry  å  rP  rZ   ry  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚXGLMConverterc                 ó’   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|g d¢z  }|S c c}w )NrÞ  rš   ))z<madeupword0>r¿  )z<madeupword1>r¿  )z<madeupword2>r¿  )z<madeupword3>r¿  )z<madeupword4>r¿  )z<madeupword5>r¿  )z<madeupword6>r¿  rw  rÈ  s       rX   rl   zXGLMConverter.vocab  sT   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØò  zñ  	zˆØˆùò Ls   —Ac                 ó
   — d}|S r³  rj   r´  s      rX   r˜   zXGLMConverter.unk_id  rµ  rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz</s> $Az</s> $A </s> </s> $Br¸  r¹  rå   r¬  rÔ   s    rX   rù   zXGLMConverter.post_processor  sR   € Ü×,Ñ,ØØ'à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
rZ   Nr%  rj   rZ   rX   r|  r|    s   „ ò	òó
rZ   r|  c                   ó<   — e Zd ZdZeZddhZ	 d„ Zd„ Zd„ Z	d„ Z
d„ Zy	)
ÚGemmaConverterTz<start_of_turn>z<end_of_turn>c                 ó.   — t        j                  dd«      S ©Nr  rb  )r
   re  rz  s     rX   rñ   zGemmaConverter.normalizer-  s   € Ü×"Ñ" 3¨Ó.Ð.rZ   c                 ót  — | j                   j                  df| j                   j                  df| j                   j                  dfg}||j                  dd  D �cg c]  }|j
                  |j                  f‘Œ c}z  }t        d„ |D «       «      s#t        d„ t        |«      D «       d «      }|�d||<   |S c c}w )Nr¿  rš   c              3   ó,   K  — | ]  }|d    dk(  –— Œ y­w)r   rÂ   Nrj   )r  rk   s     rX   r  z'GemmaConverter.vocab.<locals>.<genexpr>9  s   è ø€ Ò/ A�1�Q‘4˜4•<Ñ/ùs   ‚c              3   ó8   K  — | ]  \  }}|d    dk(  sŒ|–— Œ y­w)r   rÃ   Nrj   )r  r±   rk   s      rX   r  z'GemmaConverter.vocab.<locals>.<genexpr>:  s   è ø€ Ò"V©¨¨AÀQÀqÁTÈXÓEU¤1Ñ"Vùs   ‚“)rÂ   r¿  )
rc   rÍ  r   r0  r¨   r©   rª   ÚanyÚnextr­   )r‘   r�   rl   r©   Úoverride_indexs        rX   rl   zGemmaConverter.vocab0  s·   € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆô Ñ/¨Ô/Ô/Ü!Ñ"V´¸5Ó1AÔ"VÐX\Ó]ˆNØÐ)Ø(3��nÑ%àˆùò Ls   ÁB5c                 ó.   — t        j                  dd«      S )Nr  Úmerged_with_previous)r   rH  ©r‘   r�  r[   s      rX   ró   zGemmaConverter.pre_tokenizer@  s   € Ü×#Ñ# CÐ)?Ó@Ð@rZ   c                 ó
   — d}|S r³  rj   r´  s      rX   r˜   zGemmaConverter.unk_idC  rµ  rZ   c                 ó¢   — t        j                  t        j                  dd«      t        j                  «       t        j                  «       g«      S )Nrb  r  )r	   rG  re  rh  ri  rŒ  s      rX   rú   zGemmaConverter.decoderG  s?   € Ü× Ñ ä× Ñ  ¨Ó,Ü×%Ñ%Ó'Ü—‘“ðó
ð 	
rZ   N)r«   r¸   r¹   rq  rÀ   r€  rè   rñ   rl   ró   r˜   rú   rj   rZ   rX   r�  r�    s6   „ ØÐØ.€Là'¨Ð9€Nðò/òò Aòó
rZ   r�  c                   ó4   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z	y)	ÚLlamaConverterTc                 ó(  — | j                   j                  d«      df| j                   j                  d«      df| j                   j                  d«      dfg}||j                  dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w )Nr   r¿  r   rq   rš   )rc   rg  r¨   r©   rª   rÈ  s       rX   rl   zLlamaConverter.vocabT  sŽ   € à×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCð
ˆð
 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØˆùò Ls   Á)Bc                 ó
   — d}|S r�   rj   r´  s      rX   r˜   zLlamaConverter.unk_id]  rµ  rZ   c                 óà   — t        j                  dd«      t        j                  «       t        j                  «       g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S ©Nrb  r  r   )Úcontentr†  ©r	   re  rh  ri  rˆ  rG  ©r‘   r�  r[   Úsequences       rX   rú   zLlamaConverter.decodera  ó\   € ä×Ñ˜U CÓ(Ü×!Ñ!Ó#Ü�M‰M‹Oð
ˆñ
 ØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*rZ   c                 óø   — t        | j                  dd«      rcg }t        | j                  dd«      r|t        j                  d¬«      gz  }|t        j                  dd¬«      gz  }t        j
                  |«      S y )Nr_   Tr[   rb  )Úprependr  )Úpatternr•  )rb   rc   r
   ÚPrependre  rG  )r‘   r�   r˜  s      rX   rñ   zLlamaConverter.normalizerk  sr   € Ü�4×*Ñ*¨H°dÔ;ØˆHÜ�t×.Ñ.Ð0BÀDÔIØœ[×0Ñ0¸Ô?Ð@Ñ@�Øœ×,Ñ,°SÀ%ÔHÐIÑIˆHÜ×'Ñ'¨Ó1Ð1ØrZ   c                 óŽ   — t        | j                  dd«      s.t        || j                  «      }t        j                  ||d¬«      S y )Nr_   TF©r�  rd   Úsplit)rb   rc   re   r   rŽ  r�  s       rX   ró   zLlamaConverter.pre_tokenizert  sA   € Ü�t×.Ñ.°¸$Ô?Ü0Ð1AÀ4×CZÑCZÓ[ˆNÜ!×+Ñ+¸ÐTbÐjoÔpÐpØrZ   c                  ó   — y rÈ   rj   rÔ   s    rX   rù   zLlamaConverter.post_processorz  s   € àrZ   N)
r«   r¸   r¹   rq  rl   r˜   rú   rñ   ró   rù   rj   rZ   rX   r�  r�  Q  s&   „ ØÐòòò+òòórZ   r�  c                   ó   — e Zd Zdefd„Zy)ÚMarkupLMConverterr\   c                 óŠ  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd| j                   j                  ¬«      «      }t        j                  |j                  ¬«      |_        t        j                  «       |_        t        | j                   j                  «      }t        | j                   j                   «      }| j                   j"                  }| j                   j$                  }t'        j(                  |› d|› �|› d|› d|› �||f||fg¬«      |_        |S )Nr(  Frs  r+  z $A z $B rå   )rc   r  r¾   r  r   r   r   rÚ   r   r/  r[   ró   r	   rú   r»   rô   rõ   rö   r÷   r   rø   rù   )	r‘   rO  rl   r   rü   rý   rþ   rö   r÷   s	            rX   rÕ   zMarkupLMConverter.converted€  s,  € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%ØØ×1Ñ1×;Ñ;ôó

ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜$˜s˜eÐ$Ø�5˜˜S˜E  c UÐ+à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ð ÐrZ   Nr   rj   rZ   rX   r£  r£    s   „ ð"˜9ô "rZ   r£  c                   ó(   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zy)ÚMoshiConverterTc                 ó  — t        | d«       t        j                  | |«       t        «       }|j	                  «       }t        |d«      5 }|j                  |j                  «       «       d d d «       || _        y # 1 sw Y   || _        y xY w©NrQ   r‹   ©	r   rÐ   r•   rY   rŒ   r�   rŽ   r�   r�   ©r‘   rp  r°   r’   r“   r”   s         rX   r•   zMoshiConverter.__init__¨  sr   € Ü˜$ 
Ô+ä×Ñ˜4 Ô,ô $Ó%ˆ	à× Ñ Ó"ˆÜ�*˜dÓ#ð 	( qØ×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
úó   Á	 A9Á9B	c                 óâ   — |j                   j                  }t        j                  dd«      g}|st        j                  |«      S t        j                  t        j
                  |«      g|z   «      S rƒ  )r¯   r£   r
   re  rG  rg  r‰  s       rX   rñ   zMoshiConverter.normalizerµ  sg   € Ø$×4Ñ4×IÑIÐä×Ñ  UÓ+ð
ˆñ $Ü×'Ñ'¨Ó5Ð5ä×'Ñ'¬×)@Ñ)@ÐAUÓ)VÐ(WÐZfÑ(fÓgÐgrZ   c                 óà   — t        j                  dd«      t        j                  «       t        j                  «       g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S r”  r–  r—  s       rX   rú   zMoshiConverter.decoder¿  r™  rZ   c                 ó6   — d}t        j                  ||d¬«      S )Nr`   FrŸ  )r   rŽ  r�  s       rX   ró   zMoshiConverter.pre_tokenizerÉ  s   € Ø ˆÜ×'Ñ'°KÐP^ÐfkÔlÐlrZ   N)r«   r¸   r¹   rq  r•   rñ   rú   ró   rj   rZ   rX   r¦  r¦  ¥  s   „ ØÐòòhò+ómrZ   r¦  c                   óB   — e Zd ZdZdd„Zd„ Zd„ Zd„ Zd„ Zd„ Z	d	„ Z
d
„ Zy)ÚHeliumConverterTNc                 ó  — t        | d«       t        j                  | |«       t        «       }|j	                  «       }t        |d«      5 }|j                  |j                  «       «       d d d «       || _        y # 1 sw Y   || _        y xY wr¨  r©  rª  s         rX   r•   zHeliumConverter.__init__Ñ  sp   € Ü˜$ 
Ô+ä×Ñ˜4 Ô,ä#Ó%ˆ	à× Ñ Ó"ˆÜ�*˜dÓ#ð 	( qØ×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
úr«  c                 óV  — | j                  |«      }t        t        || j                  |«      | j                  ¬«      «      }t        |j                  «      D ��cg c]I  \  }}|j                  dv r6||j                  |j                  dk(  xs |j                  | j                  v f‘ŒK }}}|j                  t        |d„ ¬«      D ���cg c]  \  }}}t        |d|d¬«      ‘Œ c}}}«       |j                  t        d	dd¬
«      g«       |j                  dd¬«       |S c c}}w c c}}}w )Nr|  r™   rš   c                 ó   — | d   S r�   rj   rž   s    rX   rm   z+HeliumConverter.tokenizer.<locals>.<lambda>ò  r  rZ   rn   FT)r    r¡   Úsingle_wordú
rŸ   rÁ  )rÍ  Úpad_id)rl   r   r   r˜   rq  r­   r¨   r®   r©   rè   r‚  r|   r   Úenable_padding)	r‘   r�   r~   rü   r³   r´   rµ   r¶   r¡   s	            rX   rü   zHeliumConverter.tokenizerÝ  s   € Ø—z‘z %Ó(ˆÜÜØØ—{‘{ 5Ó)Ø"×7Ñ7ôó
ˆ	ô # 5§<¡<Ó0÷
á��AØ�v‰v˜Ñð �—‘˜!Ÿ&™& A™+ÒG¨¯©°D×4GÑ4GÐ)GÒHð
Ðñ 
ð
 	×Ñô +1Ð1AÁ~Ô*V÷ð á&�B˜˜wô ˜5¨U¸GÐQUÖVôô	
ð 	×Ñœj¨¸%ÈÔOÐPÔQØ× Ñ ¨7¸1Ð Ô=ØÐùó
ùôs   ÁADÃD$c                 ó¨   — g }|j                   D ]@  }|j                  dk(  r|d|j                  fgz  }Œ%||j                  |j                  fgz  }ŒB |S )Nz<0x0A>rµ  rw  rÈ  s       rX   rl   zHeliumConverter.vocabù  s]   € ØˆØ—\‘\ò 	6ˆEØ�{‰{˜hÒ&Ø˜4 §¡Ð-Ð.Ñ.‘à˜5Ÿ;™;¨¯©Ð4Ð5Ñ5‘ð		6ð
 ˆrZ   c                 ó
   — d}|S r�   rj   r´  s      rX   r˜   zHeliumConverter.unk_id  rµ  rZ   c                 óÜ   — t        j                  dd«      t        j                  «       t        j                  «       g}|t        j                  dd¬«      gz  }t        j
                  |«      S r”  r–  r—  s       rX   rú   zHeliumConverter.decoder  sY   € ä×Ñ˜U CÓ(Ü×!Ñ!Ó#Ü�M‰M‹Oð
ˆð
 	”X—^‘^¨C°aÔ8Ð9Ñ9ˆÜ× Ñ  Ó*Ð*rZ   c                 ó~   — t        j                  t        j                  d«      t        j                  dd«      g«      S rƒ  )r
   rG  r�  re  rz  s     rX   rñ   zHeliumConverter.normalizer  s2   € Ü×#Ñ#¤[×%8Ñ%8¸Ó%=¼{×?RÑ?RÐSWÐY^Ó?_Ð$`ÓaÐarZ   c                 óV   — t        j                  t        j                  dd«      g«      S )Nrµ  Ú
contiguous)r   rG  rH  rŒ  s      rX   ró   zHeliumConverter.pre_tokenizer  s#   € Ü×&Ñ&¬×(<Ñ(<¸TÀ<Ó(PÐ'QÓRÐRrZ   c                 ó<   — t        j                  ddgg d¢dg¬«      S )Nr¸  rJ  )r¸  rJ  r¸  rK  )r¸  r   rå   )r   rø   rÔ   s    rX   rù   zHeliumConverter.post_processor  s/   € Ü×,Ñ,àØðòð ðô
ð 	
rZ   rÈ   )r«   r¸   r¹   rq  r•   rü   rl   r˜   rú   rñ   ró   rù   rj   rZ   rX   r°  r°  Î  s2   „ ØÐó
òò8òò+òbòSó
rZ   r°  c                   ó   — e Zd ZdZdd„Zd„ Zy)ÚParakeetConverterTNc                 ó&  — || _         t        | d«       t        j                  | |«       t	        «       }|j                  «       }t        |d«      5 }|j                  |j                  «       «       d d d «       || _	        y # 1 sw Y   || _	        y xY wr¨  )
rp  r   rÐ   r•   rY   rŒ   r�   rŽ   r�   r�   )r‘   rp  rt  r’   r“   r”   s         rX   r•   zParakeetConverter.__init__*  sw   € Ø$ˆŒä˜$ 
Ô+ä×Ñ˜4 Ô,ä#Ó%ˆ	Ø× Ñ Ó"ˆÜ�*˜dÓ#ð 	( qØ×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
ús   Á B Â Bc                 ón  — | j                  |«      }t        |«      D ���ci c]  \  }\  }}||“Œ }}}}t        ||«      }t        t	        |||j
                  j                  d| j                  d ¬«      «      }t        |j                  «      D �	�
cg c]I  \  }	}
|
j                  dv r6|	|
j                  |
j                  dk(  xs |
j                  | j                  v f‘ŒK }}	}
|j                  t        |d„ ¬«      D �	��cg c]  \  }	}}t        |d|¬«      ‘Œ c}}}	«       |S c c}}}w c c}
}	w c c}}}	w )	NTr}  r™   rš   c                 ó   — | d   S r�   rj   rž   s    rX   rm   z-ParakeetConverter.tokenizer.<locals>.<lambda>P  r  rZ   rn   FrŸ   )rl   r­   r†   r   r   r¥   rc  rq  r¨   r®   r©   rè   r‚  r|   r   )r‘   r�   r~   r±   r²   rª   r„  r   rü   r³   r´   rµ   r¶   r¡   s                 rX   rü   zParakeetConverter.tokenizer7  s6  € Ø—z‘z %Ó(ˆä5>¸|Ó5L×MÐMÑ!1 ¡M T¨5�T˜1‘WÐMˆ	ÒMÜ  ¨LÓ9ˆÜÜØØØ×,Ñ,×6Ñ6ØØ"×7Ñ7Øôó	
ˆ	ô # 5§<¡<Ó0÷
á��AØ�v‰v˜Ñð �—‘˜!Ÿ&™& A™+ÒG¨¯©°D×4GÑ4GÐ)GÒHð
Ðñ 
ð
 	×Ñô +1Ð1AÁ~Ô*V÷ð á&�B˜˜wô ˜5¨U¸GÖDôô	
ð Ðùô5 Nùó
ùôs   ¡D#ÂAD*Ã?D0rÈ   )r«   r¸   r¹   rq  r•   rü   rj   rZ   rX   rÀ  rÀ  '  s   „ ØÐóórZ   rÀ  c            	      óò  — t        t        t        d«      t        d«      dz   «      «      t        t        t        d«      t        d«      dz   «      «      z   t        t        t        d«      t        d«      dz   «      «      z   } | dd }d	}t        d
«      D ]1  }|| vsŒ| j                  |«       |j                  d
|z   «       |dz  }Œ3 |D �cg c]  }t	        |«      ‘Œ }}t        t        | |«      «      S c c}w )a8  
    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
    characters the bpe code barfs on.

    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
    tables between utf-8 bytes and unicode strings.
    ú!ú~r   õ   Â¡õ   Â¬õ   Â®õ   Ã¿Nr   é   )r¾   rz   Úordr{   Úchrrx   ri  )ÚbsÚcsÚnÚbs       rX   Úbytes_to_unicoderÒ  W  sæ   € ô 	ŒU”3�s“8œS ›X¨™\Ó*Ó+¬d´5¼¸T»ÄCÈÃIÐPQÁMÓ3RÓ.SÑSÔVZÔ[`ÔadÐeiÓajÔloÐptÓluÐxyÑlyÓ[zÓV{Ñ{ð ð 
‰Aˆ€BØ	€AÜ�4‹[ò ˆØ�BŠ;Ø�I‰I�aŒLØ�I‰I�d˜Q‘hÔØ�‰F‰Að	ð
 Ö	�QŒ#ˆa�&Ð	€BÐ	Ü”�B˜“ÓÐùò 
s   ÃC4c                   ó>   — e Zd ZdZ	 	 	 	 d	d„Zdefd„Zd„ Zdefd„Z	y)
ÚTikTokenConverterz'
    A general tiktoken converter.
    Nc                 óˆ   — || _         || _        || _        t        |t        «      r|j                  «       | _        y || _        y rÈ   )rp  rœ  r[   rd  rx   r   Úextra_special_tokens)r‘   rp  rœ  r[   rÖ  r°   s         rX   r•   zTikTokenConverter.__init__t  sD   € ð %ˆŒØˆŒØ 0ˆÔä+5Ð6JÌDÔ+QÐ ×%Ñ%Ó'ð 	Õ!ØWkð 	Õ!rZ   Útiktoken_urlc                 ó0  ‡‡— 	 ddl m}  ||«      Št	        «       Šˆfd„}g }i }‰j                  «       D ]�  \  }}|| ||«      <   t        |«      dk(  rŒ g }t        dt        |«      «      D ]2  }	|d |	 ||	d  }}
|
‰v sŒ|‰v sŒ|
|z   ‰v sŒ|j                  |
||f«       Œ4 t        |ˆfd„d¬«      }|j                  |«       Œ� t        |d	„ d¬«      }|D �cg c]  } ||d   «       ||d   «      f‘Œ }}||fS # t        $ r t        d«      ‚w xY wc c}w )
Nr   )Úload_tiktoken_bpezY`tiktoken` is required to read a `tiktoken` file. Install it with `pip install tiktoken`.c           	      ó‚   •— dj                  | j                  d«      D �cg c]  }‰t        |«         ‘Œ c}«      S c c}w ©Nr(  zlatin-1©rh  ÚdecoderÌ  ©rÑ  ÚcharÚbyte_encoders     €rX   Útoken_bytes_to_stringzPTikTokenConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_stringŽ  s2   ø€ Ø—7‘7ÀÇÁÈÓ@SÖT¸˜L¬¨T«Ó3ÒTÓUÐUùÒTó    <r   c                 ó$   •— ‰| d      ‰| d      fS ri   rj   )rk   r  s    €rX   rm   zCTikTokenConverter.extract_vocab_merges_from_model.<locals>.<lambda>œ  s   ø€ °¸1¸Q¹4±À)ÈAÈaÉDÁ/Ð0R€ rZ   Fru   c                 ó   — | d   S ©Nrq   rj   rs   s    rX   rm   zCTikTokenConverter.extract_vocab_merges_from_model.<locals>.<lambda>ž  ó
   € °°A±€ rZ   )Útiktoken.loadrÙ  r�  Ú
ValueErrorrÒ  ry   rr   rz   r{   r|   r}   )r‘   r×  rÙ  rá  r   rl   r¶   Úrankr‚   rƒ   r„   r…   rt   r  rà  s                @@rX   Úextract_vocab_merges_from_modelz1TikTokenConverter.extract_vocab_merges_from_modelƒ  sY  ù€ ð	Ý7ñ & lÓ3ˆ	Ü'Ó)ˆô	Vð ˆØˆØ$Ÿ?™?Ó,ò 
	!‰KˆE�4Ø26ˆEÑ'¨Ó.Ñ/Ü�5‹z˜QŠØØˆEÜ˜q¤# e£*Ó-ò ;�Ø#(¨¨% =°%¸¸°-˜�Ø˜iÒ'¨G°yÒ,@ÀgÐPWÑFWÐ\eÒEeØ—L‘L '¨7°DÐ!9Õ:ð;ô ˜5Ó&RÐ\aÔbˆEØ�M‰M˜%Õ ð
	!ô ˜Ñ$6ÀÔFˆØ\bÖcÐUXÑ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÒPÐcˆÐcØ�fˆ}Ðøô5 ò 	ÜØkóð ð	üò2 ds   „C; ÃDÃ;Dc                 ó¾   — | j                  | j                  «      \  }}t        t        ||d¬«      «      }t	        |j
                  d«      rd|j
                  _        |S ©NF)r  Úignore_mergesT©rê  rp  r   r   rí   r‰   rí  ©r‘   r~   r   rü   s       rX   rü   zTikTokenConverter.tokenizer¢  óN   € Ø#×CÑCÀDÇOÁOÓTÑˆ�fÜœc ,°ÀÔGÓHˆ	Ü�9—?‘? OÔ4Ø,0ˆI�O‰OÔ)ØÐrZ   r\   c           
      óâ  — | j                  «       }t        j                  t        j                  t	        | j
                  «      dd¬«      t        j                  | j                  d¬«      g«      |_        t        j                  «       |_
        | j                  �5|j                  | j                  D �cg c]  }t        |dd¬«      ‘Œ c}«       t        j                  d¬«      |_        |S c c}w )Nr@  FrA  rD  TrŸ   r-  )rü   r   rG  rH  r   rœ  r/  r[   ró   r	   rú   rÖ  r"  r   r   rù   )r‘   rü   r¶   s      rX   rÕ   zTikTokenConverter.converted©  sÄ   € Ø—N‘NÓ$ˆ	Ü"0×"9Ñ"9ä×$Ñ$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ñ(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ñ.Ó0ˆ	Ôà×$Ñ$Ð0Ø×(Ñ(ØPT×PiÑPiÖjÀu”˜E¨e¸TÖBÒjôô $.×#7Ñ#7ÀUÔ#Kˆ	Ô àÐùò ks   Â3C,©Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+FN)
r«   r¸   r¹   rº   r•   r»   rê  rü   r   rÕ   rj   rZ   rX   rÔ  rÔ  o  s:   „ ñð ð KØØ!ó
ð¸Có ò>ð˜9ô rZ   rÔ  c                   ó:   — e Zd Z	 	 	 	 dd„Zdefd„Zd„ Zdefd„Zy)	ÚMistralConverterNc                 óˆ   — || _         || _        || _        t        |t        «      r|j                  «       | _        y || _        y rÈ   )rp  rœ  r[   rd  rx   r   r¢   )r‘   rp  rœ  r[   r¢   r°   s         rX   r•   zMistralConverter.__init__¾  sJ   € ð %ˆŒØˆŒØ 0ˆÔô Ð3´TÔ:ð &×*Ñ*Ó,ð 	Õ&ð +ð 	Õ&rZ   r×  c                 ó  ‡‡— dd l }dd l}t        | j                  dd¬«      5 }|j	                  |«      }d d d «       d   d   | _        |d   D �cg c]  }t        |d   |d	   ¬
«      ‘Œ c}| _        |d   }t        «       Št        ˆfd„«       }g }	i }
t        | j                  «      D ]  \  }}||
|j                  <   Œ |D �cg c]  }|j                  |d   «      ‘Œ }}t        |«      }t        |«      D ��ci c]  \  }}||“Œ
 c}}Št        t        |d¬«      «      D ]�  \  }}||
 ||«      <   t        |«      dk(  rŒ g }t!        dt        |«      «      D ]2  }|d | ||d  }}||v sŒ||v sŒ||z   |v sŒ|j#                  |||f«       Œ4 t%        |ˆfd„d¬«      }|	j'                  |«       Œ� t%        |	d„ d¬«      }	|	D �cg c]  } ||d   «       ||d   «      f‘Œ }	}|
|	fS # 1 sw Y   �Œ´xY wc c}w c c}w c c}}w c c}w )Nr   Úrzutf-8)ÚencodingÚconfigrœ  rè   Ú	token_strÚ
is_control)r¡   rl   c           	      ó‚   •— dj                  | j                  d«      D �cg c]  }‰t        |«         ‘Œ c}«      S c c}w rÛ  rÜ  rÞ  s     €rX   rá  zOMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_stringÜ  s2   ø€ à—7‘7ÀÇÁÈÓ@SÖT¸˜L¬¨T«Ó3ÒTÓUÐUùÒTrâ  Útoken_bytesz(Converting tekken.json to tokenizer.json)Údescr   c                 ó$   •— ‰| d      ‰| d      fS ri   rj   )rk   Útoken_to_ranks    €rX   rm   zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>ð  s!   ø€ °¸qÀ¹tÑ1DÀmÐTUÐVWÑTXÑFYÐ0Z€ rZ   Fru   c                 ó   — | d   S rå  rj   rs   s    rX   rm   zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>ò  ræ  rZ   )Úbase64Újsonr�   rp  Úloadrœ  r   r¢   rÒ  r   r­   r•  Ú	b64decoderw   r   rr   rz   r{   r|   r}   )r‘   r×  r  r  r”   ÚuntypedÚkr  rá  r   rl   Úidxr¶   Úrank_setré  r‚   rƒ   r„   r…   rt   rà  r   s                       @@rX   rê  z0MistralConverter.extract_vocab_merges_from_modelÏ  s?  ù€ ÛÛä�$—/‘/ 3°Ô9ð 	#¸QØ—i‘i “lˆG÷	#à˜xÑ(¨Ñ3ˆŒàIPÐQaÑIbö*
ØDEŒJ�q˜‘~¨q°©Ö?ò*
ˆÔ&ð ˜GÑ$ˆ	Ü'Ó)ˆä	ó	Vó 
ð	Vð ˆØˆÜ# D×$BÑ$BÓCò 	'‰JˆC�Ø#&ˆE�%—-‘-Ò ð	'àAJÖK¸A�V×%Ñ% a¨Ñ&6Õ7ÐKˆ	ÐKÜ�y“>ˆÜ8AÀ)Ó8L×M©¨¨u˜ ™ÓMˆÜ$¤T¨)Ð:dÔ%eÓfò 
	!‰KˆD�%Ø26ˆEÑ'¨Ó.Ñ/Ü�5‹z˜QŠØØˆEÜ˜q¤# e£*Ó-ò ;�Ø#(¨¨% =°%¸¸°-˜�Ø˜hÒ&¨7°hÒ+>ÀGÈgÑDUÐZbÒCbØ—L‘L '¨7°DÐ!9Õ:ð;ô ˜5Ó&ZÐdiÔjˆEØ�M‰M˜%Õ ð
	!ô ˜Ñ$6ÀÔFˆØ\bÖcÐUXÑ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÒPÐcˆÐcØ�fˆ}Ð÷C	#ñ 	#üò*
ùò LùãMùò ds#   £G'ÁG4ÃG9Ã:G>ÇHÇ'G1c                 ó¾   — | j                  | j                  «      \  }}t        t        ||d¬«      «      }t	        |j
                  d«      rd|j
                  _        |S rì  rî  rï  s       rX   rü   zMistralConverter.tokenizerö  rð  rZ   r\   c                 óŒ  — | j                  «       }t        j                  t        j                  t	        | j
                  «      dd¬«      t        j                  | j                  d¬«      g«      |_        t        j                  «       |_
        |j                  | j                  «       t        j                  d¬«      |_        |S )Nr@  FrA  rD  r-  )rü   r   rG  rH  r   rœ  r/  r[   ró   r	   rú   r‚  r¢   r   rù   )r‘   rü   s     rX   rÕ   zMistralConverter.convertedý  s™   € Ø—N‘NÓ$ˆ	Ü"0×"9Ñ"9ä×$Ñ$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ñ(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ñ.Ó0ˆ	Ôà×Ñ˜T×;Ñ;Ô<Ü#-×#7Ñ#7ÀUÔ#Kˆ	Ô àÐrZ   rò  )	r«   r¸   r¹   r•   r»   rê  rü   r   rÕ   rj   rZ   rX   rô  rô  ½  s6   „ ð ð KØØ"&ó
ð"%¸Có %òNð˜9ô rZ   rô  ÚAlbertTokenizerÚBartTokenizerÚBarthezTokenizerÚBertTokenizerÚBigBirdTokenizerÚBlenderbotTokenizerÚCamembertTokenizerÚCLIPTokenizerÚCodeGenTokenizerÚConvBertTokenizerÚDebertaTokenizerÚDebertaV2TokenizerÚDistilBertTokenizerÚDPRReaderTokenizerÚDPRQuestionEncoderTokenizerÚDPRContextEncoderTokenizerÚElectraTokenizerÚFNetTokenizerÚFunnelTokenizerÚGPT2TokenizerÚHerbertTokenizerÚLayoutLMTokenizerÚLayoutLMv2TokenizerÚLayoutLMv3TokenizerÚLayoutXLMTokenizerÚLongformerTokenizerÚLEDTokenizerÚLxmertTokenizerÚMarkupLMTokenizerÚMBartTokenizerÚMBart50TokenizerÚMPNetTokenizerÚMobileBertTokenizerÚMvpTokenizerÚNllbTokenizerÚOpenAIGPTTokenizerÚPegasusTokenizerÚQwen2TokenizerÚReformerTokenizerÚRemBertTokenizerÚRobertaTokenizerÚRoFormerTokenizerÚSeamlessM4TTokenizerÚSqueezeBertTokenizerÚT5TokenizerÚUdopTokenizerÚVideoPrismTokenizerÚWhisperTokenizerÚXLMRobertaTokenizerÚXLNetTokenizerÚSplinterTokenizer)ÚXGLMTokenizerÚLlamaTokenizerÚCodeLlamaTokenizerÚGemmaTokenizerÚPhi3Tokenizerc                 ó*  — | j                   j                  }|t        v r!|st        |   } || «      j                  «       S | j                  j                  d«      r?| | _        t        j                  d«       t        | j                  «      j                  «       S 	 t        j                  d«       t        | j                  | j                  ¬«      j                  «       S # t        $ r* t        dt        t        j                  «       «      › �«      ‚w xY w)aˆ  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].
       from_tiktoken (bool, optional): Whether to use the `tiktoken` library to convert the tokenizer instead of sentencepiece.
            Defaults to False.

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    ztekken.jsonz#Converting from Mistral tekken.jsonzConverting from Tiktoken)rp  rÖ  zÌConverting from SentencePiece and Tiktoken failed, if a converter for SentencePiece is available, provide a model path with a SentencePiece tokenizer.model file.Currently available slow->fast converters: )ru  r«   ÚSLOW_TO_FAST_CONVERTERSrÕ   rp  Úendswithrc   ÚloggerÚinforô  rÔ  rÖ  r�  rè  r¾   r   )Útransformer_tokenizerÚfrom_tiktokenÚtokenizer_class_nameÚconverter_classs       rX   Úconvert_slow_tokenizerrM  I  sþ   € ð  1×:Ñ:×CÑCÐØÔ6Ñ6¹}Ü1Ð2FÑGˆÙÐ4Ó5×?Ñ?ÓAÐAØ	×	)Ñ	)×	2Ñ	2°=Ô	AØ3HÐÔ0Ü�‰Ð9Ô:ÜÐ 5× @Ñ @ÓA×KÑKÓMÐMð	Ü�K‰KÐ2Ô3Ü$Ø0×;Ñ;Ø%:×%OÑ%Oô÷ ‰i‹kðøô ò 	Üð>ä>BÔCZ×C_ÑC_ÓCaÓ>bÐ=cðeóð ð	ús   ÂAC Ã3D)r(  rÈ   )F)Xrº   rr  Úcollections.abcr   Ú	functoolsr   Ú	packagingr   Ú
tokenizersr   r   r   r	   r
   r   r   r¦   r   r   r   r   Úutilsr   r   r   r   Úutils.import_utilsr   Ú
get_loggerr«   rG  r  r  rY   Úboolr»   re   r†   rˆ   rÀ   rÎ   rÐ   r×   r  r  r  r  r&  r6  r=  rJ  rR  rY  r`  r˜  r±  r¼  rÔ  rÜ  r  r  r  r'  r-  r2  r6  r:  r<  rO  r]  ra  rd  rn  rq  rv  ry  r|  r�  r�  r£  r¦  r°  rÀ  rÒ  rÔ  rô  rE  rM  rj   rZ   rX   ú<module>rV     sÅ  ðñó Ý &Ý å ß f× fÑ fß 5Ñ 5Ý ç `Ó `Ý 5ð 
ˆ×	Ñ	˜HÓ	%€ò€ð8 $ò 'ñ Ð ó>Gð"¨$ð Àsó ñ°jÀ±oÈÑ6Ló ÷02ñ 2ôjÐ"8ô ð"I˜cð I dó I÷$ñ $ô$�Iô $ôN/˜	ô /ôd$�iô $ôN$�Yô $ôN˜ô ô6#�Iô #ôL�yô ô>)�Yô )ôX�yô ô:$˜	ô $ôN�yô ô>x�9ô xôv"
�lô "
ôJ
�|ô 
ô -˜ô -ô`
˜ô 
ôBG�\ô GôT-�|ô -ô`,�Lô ,ô^
˜<ô 
ô2,˜,ô ,ô^"
�\ô "
ôJ	˜ô 	ô
�|ô 
ô@	˜lô 	ô;x�|ô ;xô|�,ô ô8
�Lô 
ô˜+ô ô
!�yô !ôH	
�|ô 	
ô(�Iô (ôV$˜)ô $ôN˜)ô ô:
�Lô 
ô61
�\ô 1
ôh+�\ô +ô\#˜	ô #ôL&m�\ô &môRV
�lô V
ôr-˜ô -ò`÷0Kñ K÷\Mñ Mð`9Ø�ð9àÐ%ð9ð Ð(ð9ð �]ð	9ð
 Ð(ð9ð Ð.ð9ð Ð,ð9ð �]ð9ð ˜ð9ð ˜ð9ð Ð(ð9ð Ð,ð9ð ˜=ð9ð ˜-ð9ð " =ð9ð  ! -ð!9ð" ˜ñ#9ð$ �_ð%9ð& �ð'9ð( �]ð)9ð* Ð(ð+9ð, ˜ð-9ð. ˜=ð/9ð0 Ð+ð19ð2 Ð-ð39ð4 Ð+ð59ð6 Ð$ð79ð8 �}ð99ð: Ð*ð;9ð< �nð=9ð> Ð(ð?9ð@ �nðA9ðB ˜=ðC9ðD Ð$òE9ðF �]ðG9ðH Ð,ðI9ðJ Ð(ðK9ðL �nðM9ðN Ð*ðO9ðP Ð(ðQ9ðR Ð(ðS9ðT Ð*ðU9ðV Ð0ðW9ðX ˜MðY9ðZ �;ð[9ð\ �]ð]9ð^ Ð.ð_9ð` Ð(ða9ðb Ð.ðc9ðd �nðe9ðf Ð*ñg9ðh #Ø$Ø(Ø$Ø#òq9Ð ñx$È)ô $rZ   