Ë
    ÛÓ^jæ~  ã                   óP  — d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZmZmZ d dlZddlmZ  e«       rd dlZd dlmZ d d	lmZ  G d
„ de«      Z ej0                  «       j2                  Z e
e«      j8                  d   Zde› d�j=                  «       ZdddddœZ 	 d9dee!dz   ejD                  d¬«      f   fd„Z# G d„ d«      Z$de!ejJ                  z  de!de!fd„Z&de
de$de!d e!d!e'e(e!e)f      f
d"„Z*d#e!d$e'e!   fd%„Z+de
d&e!de!d e!fd'„Z,d&e!de!d!e'e(e!e)f      fd(„Z-d)e!d*e.fd+„Z/d)e
de$d,e!d*e(e!e!e'f   fd-„Z0de
de$de!d!e'e(e!e)f      d*e!f
d.„Z1de
de$d!e'e(e!e)f      fd/„Z2de
de$de!d e!d!e'e(e!e)f      f
d0„Z3	 	 	 d:d1e!d2e!dz  d3edz  d4e!dz  d*ef
d5„Z4d6e!d*e)fd7„Z5d8„ Z6y);é    N)ÚCallable)Údate)ÚPath)Ú	AnnotatedÚAnyÚcasté   )Úis_libcst_available)Ú
CSTVisitor)Úmatchersc                   óz   — e Zd ZdZd„ Zdej                  ddfd„Zdej                  fd„Zdej                  fd„Z
y)	ÚClassFinderzC
        A visitor to find all classes in a python module.
        c                 ó.   — g | _         g | _        d| _        y ©NF)ÚclassesÚpublic_classesÚis_in_class)Úselfs    ún/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/cli/add_new_model_like.pyÚ__init__zClassFinder.__init__)   s   € Ø!#ˆDŒLØ(*ˆDÔØ$ˆDÕó    ÚnodeÚreturnNc                 óp   — | j                   j                  |j                  j                  «       d| _        y)zwRecord class names. We assume classes always only appear at top-level (i.e. no class definition in function or similar)TN)r   ÚappendÚnameÚvaluer   ©r   r   s     r   Úvisit_ClassDefzClassFinder.visit_ClassDef.   s$   € à�L‰L×Ñ §	¡	§¡Ô0Ø#ˆDÕr   c                 ó   — d| _         y r   )r   r   s     r   Úleave_ClassDefzClassFinder.leave_ClassDef3   s
   € Ø$ˆDÕr   c                 ó¨  — t        j                  t        j                  t        j                  t        j                  «       ¬«      g¬«      g¬«      }| j
                  sït        j                  ||«      rØt        t        j                  |j                  d   «      }t        t        j                  |j                  d   j                  «      j                  }|dk(  rpt        t        j                  |j                  «      j                  }|D �cg c]0  }t        t        j                  |j                  «      j                  ‘Œ2 c}| _        yyyyc c}w )z:Record all public classes inside the `__all__` assignment.)Útarget)Útargets)Úbodyr   Ú__all__N)ÚmÚSimpleStatementLineÚAssignÚAssignTargetÚNamer   Úmatchesr   Úcstr%   r$   r#   r   ÚTupleÚelementsÚSimpleStringr   )r   r   Ú!simple_top_level_assign_structureÚstmtÚassigned_variabler/   Úelements          r   Úvisit_SimpleStatementLinez%ClassFinder.visit_SimpleStatementLine6   së   € ä01×0EÑ0EÜ—h‘h¬¯©¼a¿f¹f»hÔ(GÐ'HÔIÐJô1Ð-ð ×#Ò#¬¯	©	°$Ð8YÔ(ZÜœCŸJ™J¨¯	©	°!©Ó5�Ü$(¬¯©°4·<±<À±?×3IÑ3IÓ$J×$PÑ$PÐ!Ø$¨	Ò1Ü#¤C§I¡I¨t¯z©zÓ:×CÑC�HØfnÖ*oÐ[b¬4´×0@Ñ0@À'Ç-Á-Ó+P×+VÓ+VÒ*o�DÕ'ð 2ð )[Ð#ùò
 +ps   Ä5E)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r-   ÚClassDefr   r!   r(   r5   © r   r   r   r   $   sG   „ ñ	ò	%ð
	$ s§|¡|ð 	$¸ó 	$ð
	% s§|¡|ó 	%ð
	p°#×2IÑ2Iô 
	pr   r   é   z
# coding=utf-8
# Copyright aN   the HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ÚIMAGE_PROCESSOR_MAPPING_NAMESÚVIDEO_PROCESSOR_MAPPING_NAMESÚPROCESSOR_MAPPING_NAMESÚFEATURE_EXTRACTOR_MAPPING_NAMES)zimage_processing_auto.pyzvideo_processing_auto.pyzprocessing_auto.pyzfeature_extraction_auto.pyÚ	repo_pathzFWhen not using an editable install, the path to the Transformers repo.)Úhelpc                 ód   — t        «       \  }}}}t        | �t        | «      nt        ||||¬«       y)zC
    Add a new model to the library, based on an existing one.
    N©rA   Úold_model_infosÚnew_lowercase_nameÚnew_model_paper_nameÚfilenames_to_add)Úget_user_inputÚ_add_new_model_like_internalr   Ú	REPO_PATHrD   s        r   Úadd_new_model_likerL   b   s?   € ô 	ÓñØØØØô !Ø%.Ð%:”$�y”/Ä	Ø'Ø-Ø1Ø)ör   c                   ó   — e Zd ZdZdefd„Zy)Ú
ModelInfoszI
    Retrieve the basic information about an existing model classes.
    Úlowercase_namec                 ój  — ddl m} ddlm} ddlm} ddlm} ddlm	} ddl
m} |j                  «       j                  dd	«      j                  d
d	«      | _        | j                  |vr| j                  j                  d	d
«       | j                  |vrt        |› d�«      ‚|| j                     | _        | j                   j                  dd«      | _        | j                  |v r>d | _        || j                     | _        | j&                  dk(  rd n| j&                  | _        nd\  | _        | _        |j)                  | j                  d «      | _        |j)                  | j                  d «      | _        |j)                  | j                  d «      | _        |j)                  | j                  d «      | _        y )Nr	   ©ÚCONFIG_MAPPING_NAMES)r@   )r=   )r?   )ÚTOKENIZER_MAPPING_NAMES)r>   ú Ú_ú-z is not a valid model nameÚConfigÚ ÚPreTrainedTokenizerFast)NN)Úmodels.auto.configuration_autorR   Ú#models.auto.feature_extraction_autor@   Ú!models.auto.image_processing_autor=   Úmodels.auto.processing_autor?   Úmodels.auto.tokenization_autorS   Ú!models.auto.video_processing_autor>   ÚlowerÚreplacerO   Ú
ValueErrorÚconfig_classÚcamelcase_nameÚtokenizer_classÚfast_tokenizer_classÚgetÚimage_processor_classesÚvideo_processor_classÚfeature_extractor_classÚprocessor_class)r   rO   rR   r@   r=   r?   rS   r>   s           r   r   zModelInfos.__init__‚   s‰  € ÝIÝYÝUÝIÝKÝUð -×2Ñ2Ó4×<Ñ<¸SÀ#ÓF×NÑNÈsÐTWÓXˆÔØ×ÑÐ&:Ñ:Ø×Ñ×'Ñ'¨¨SÔ1Ø×ÑÐ&:Ñ:Ü Ð/Ð/IÐJÓKÐKà0°×1DÑ1DÑEˆÔØ"×/Ñ/×7Ñ7¸À"ÓEˆÔð ×ÑÐ"9Ñ9Ø#'ˆDÔ Ø(?À×@SÑ@SÑ(TˆDÔ%à×1Ñ1Ð5NÒN‘ÐTX×TmÑTmð Õ%ð ?IÑ;ˆDÔ  $Ô";à'D×'HÑ'HÈ×I\ÑI\Ð^bÓ'cˆÔ$Ø%B×%FÑ%FÀt×GZÑGZÐ\`Ó%aˆÔ"Ø'F×'JÑ'JÈ4×K^ÑK^Ð`dÓ'eˆÔ$Ø6×:Ñ:¸4×;NÑ;NÐPTÓUˆÕr   N)r6   r7   r8   r9   Ústrr   r;   r   r   rN   rN   }   s   „ ñðV sô Vr   rN   Ú	file_nameÚnew_contentÚ	add_afterc                 ó  — t        | dd¬«      5 }|j                  «       }ddd«       j                  |d«      \  }}||z   |z   |z   }t        | dd¬«      5 }|j                  |«       ddd«       y# 1 sw Y   ŒRxY w# 1 sw Y   yxY w)ac  
    A utility to add some content inside a given file.

    Args:
        file_name (`str` or `os.PathLike`):
            The name of the file in which we want to insert some content.
        new_content (`str`):
            The content to add.
       add_after (`str`):
           The new content is added just after the first instance matching it.
    Úrúutf-8©ÚencodingNé   Úw)ÚopenÚreadÚsplitÚwrite)rm   rn   ro   ÚfÚold_contentÚbeforeÚafters          r   Úadd_content_to_filer   ¤   s’   € ô 
ˆi˜ wÔ	/ð °1Ø—f‘f“hˆ÷ð  ×%Ñ% i°Ó3�M€FˆEØ˜9Ñ$ {Ñ2°UÑ:€Kä	ˆi˜ wÔ	/ð °1Ø	�‰�Ô÷ð ÷ð ú÷ð ús   �A1ÁA=Á1A:Á=BrE   rF   rG   rH   c           	      ó<  — dj                  d„ |j                  dd«      j                  d«      D «       «      }|j                  }|j                  }|dd D ��	cg c]  \  }}	|j                  |d«      |	f‘Œ }}}	g }
|D ]0  \  }}	d|v r|
d	   \  }}||xs |	f|
d	<   Œ|
j                  ||	f«       Œ2 t        | d
z  dz  dz  dz  dz  d|› d|› d�d¬«       | d
z  dz  dz  dz  dz  j                  «       }|
D �]  \  }}	|	sŒ
|t        v rot        |   }d}t        j                  d|› d�|t        j                  «      }|j                  d«      }t        j                  d|› d�|t        j                  «      }nZ|j                  dd«      }| d
z  dz  dz  dz  |z  j                  «       }t        j                  d|› d�|t        j                  «      }|D ]?  }t        | d
z  dz  dz  dz  |z  |j                  ||«      j                  ||«      |¬«       ŒA �Œ! yc c}	}w )aÎ  
    Add a model to all the relevant mappings in the auto module.

    Args:
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_lowercase_name (`str`):
            The new lowercase model name.
        new_model_paper_name (`str`):
            The fully cased name (as in the official paper name) of the new model.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    rX   c              3   ó<   K  — | ]  }|j                  «       –— Œ y ­w©N©Útitle©Ú.0Úxs     r   ú	<genexpr>z-add_model_to_auto_mappings.<locals>.<genexpr>Ï   ó   è ø€ Ò`¨1˜QŸW™WŸYÑ`ùó   ‚rV   rU   ru   NÚautoztokenization_auto_fast.pyéÿÿÿÿÚsrcÚtransformersÚmodelszauto_mappings.pyz("z", "zConfig"),
        z2CONFIG_MAPPING_NAMES = OrderedDict(
    [
        ©rn   ro   z[^\w_]z%\s*=\s*OrderedDict\(\s*\[(.*?)\]\s*\)z( {8,12}\(\s*"z",.*?\),\n)(?: {4,12}\(|\])ú_fast.pyú.py)Újoinra   ry   rO   rd   r   r   Ú	read_textÚ_AUTO_MAPPING_NAMESÚreÚsearchÚDOTALLÚgroupÚfindall)rA   rE   rF   rG   rH   Únew_cased_nameÚold_lowercase_nameÚold_cased_nameÚfilenameÚto_addÚcorrected_filenames_to_addÚfileÚprevious_fileÚprevious_to_addÚautofileÚmapping_nameÚblock_matchÚblockÚmatching_linesÚmatchs                       r   Úadd_model_to_auto_mappingsrª   º   sž  € ð* —W‘WÑ`Ð0B×0JÑ0JÈ3ÐPSÓ0T×0ZÑ0ZÐ[^Ó0_Ô`Ó`€NØ(×7Ñ7ÐØ$×3Ñ3€NàWgÐhiÐhjÐWk÷ÙCSÀ8ÈVˆ×	Ñ	Ð,¨fÓ	5°vÒ>ðÐñ ð "$ÐØ(ò >‰ˆˆfØ&¨$Ñ.Ø-GÈÑ-KÑ*ˆM˜?Ø.;¸_Ò=VÐPVÐ-WÐ& rÒ*à&×-Ñ-¨t°V¨nÕ=ð>ô Ø�EÑ˜NÑ*¨XÑ5¸Ñ>ÐASÑSØÐ+Ð,¨D°Ð0@Ð@SÐTØHõð
 ˜EÑ! NÑ2°XÑ=ÀÑFÐI[Ñ[×fÑfÓh€Hà6ó Ñˆ�&ÚØÔ.Ñ.ä2°8Ñ<�Ø-�Ü Ÿi™iØ˜l˜^Ð+PÐQÐS[Ô]_×]fÑ]fó�ð $×)Ñ)¨!Ó,�Ü!#§¡Ø'Ð(:Ð';Ð;XÐYÐ[`Ôbd×bkÑbkó"‘ð
 $×+Ñ+¨J¸Ó>�Ø! EÑ)¨NÑ:¸XÑEÈÑNÐQYÑY×dÑdÓf�ä!#§¡Ø'Ð(:Ð';Ð;XÐYÐ[_Ôac×ajÑajó"�ð (ò �Ü#Ø Ñ%¨Ñ6¸ÑAÀFÑJÈXÑUØ %§¡Ð.@ÐBTÓ U× ]Ñ ]Ø&¨ó!ð $öòñ-ùó)s   ÁHÚnew_paper_namer   c                 ó"  — d}t        j                  ddt        «      j                  dd«      |z   }t	        j
                  d| › d| › d�«      }g }|D ]$  }d	|› d
|› �}d|v r|dz  }|j                  |«       Œ& dj                  |«      }||z   |z   S )a4  
    Create a new doc file to fill for the new model.

    Args:
        new_paper_name (`str`):
            The fully cased name (as in the official paper name) of the new model.
        public_classes (`list[str]`):
            A list of all the public classes that the model will have in the library.
    u®   

âš ï¸� Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be rendered properly in your Markdown viewer.

-->

z# ?rX   zcoding=utf-8
z<!--z
        # z#

        ## Overview

        The a<   model was proposed in [<INSERT PAPER NAME HERE>](<INSERT PAPER LINK HERE>) by <INSERT AUTHORS HERE>.
        <INSERT SHORT SUMMARY HERE>

        The abstract from the paper is the following:

        <INSERT PAPER ABSTRACT HERE>

        Tips:

        <INSERT TIPS ABOUT MODEL HERE>

        This model was contributed by [INSERT YOUR HF USERNAME HERE](https://huggingface.co/<INSERT YOUR HF USERNAME HERE>).
        The original code can be found [here](<INSERT LINK TO GITHUB REPO HERE>).

        ## Usage examples

        <INSERT SOME NICE EXAMPLES HERE>

        z## z

[[autodoc]] ÚModelz
    - forwardú

)r–   ÚsubÚ	COPYRIGHTra   ÚtextwrapÚdedentr   r“   )	r«   r   Ú
added_noteÚcopyright_for_markdownÚdoc_templateÚdoc_for_classesÚclass_ÚdocÚ	class_docs	            r   Úcreate_doc_filerº     sÑ   € ð	Kð ô  ŸV™V F¨B´	Ó:×BÑBÐCSÐU[Ó\Ð_iÑiÐä—?‘?ðØÐ
ð ð Ðð 	ð	ó€Lð8 €OØ ò $ˆØ�F�8Ð+¨F¨8Ð4ˆØ�fÑØÐ$Ñ$ˆCØ×Ñ˜sÕ#ð	$ð —‘˜OÓ,€Ià! LÑ0°9Ñ<Ð<r   rœ   c                 ó>  — | dz  dz  dz  dz  }t        |d«      5 }|j                  «       }ddd«       t        j                  d|› d�«      }|€t	        d	|› �«      ‚|j                  d
«      }d|› d|› d�}	t        | dz  dz  dz  dz  |	|¬«       y# 1 sw Y   ŒhxY w)a€  
    Insert the new model in the doc `_toctree.yaml`, in the same section as the old model.

    Args:
        old_lowercase_name (`str`):
            The old lowercase model name.
        new_lowercase_name (`str`):
            The new lowercase model name.
        new_model_paper_name (`str`):
            The fully cased name (as in the official paper name) of the new model.
    ÚdocsÚsourceÚenz_toctree.ymlrq   Nz- local: model_doc/z\n {8}title: .*?\nzCould not find TOC entry for r   z      - local: model_doc/z
        title: ú
r�   )rw   rx   r–   r—   rb   r™   r   )
rA   rœ   rF   rG   Útoc_filer{   ÚcontentÚ	toc_matchÚold_model_tocÚnew_tocs
             r   Úinsert_model_in_doc_tocrÅ   >  sÐ   € ð ˜6Ñ! HÑ,¨tÑ3°nÑD€HÜ	ˆh˜Ó	ð  Ø—&‘&“(ˆ÷ô —	‘	Ð0Ð1CÐ0DÐDXÐYÐ[bÓc€IØÐÜÐ8Ð9KÐ8LÐMÓNÐNØ—O‘O AÓ&€MØ)Ð*<Ð)=Ð=NÐOcÐNdÐdfÐg€GÜØ�FÑ˜XÑ%¨Ñ,¨~Ñ=È7Ð^kö÷ð ús   ›BÂBc                 óî   — |D ��cg c])  \  }}|j                  | |«      j                  dd«      |f‘Œ+ }}}dj                  d„ |D «       «      }t        t        j                  d|› d�«      z   }|S c c}}w )a/  
    Create the `__init__.py` file to add in the new model folder.

    Args:
        old_lowercase_name (`str`):
            The old lowercase model name.
        new_lowercase_name (`str`):
            The new lowercase model name.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    r’   rX   ú
            c              3   ó4   K  — | ]  \  }}|sŒ	d |› d�–— Œ y­w)zfrom .z	 import *Nr;   )r†   r¡   rŸ   s      r   rˆ   z#create_init_file.<locals>.<genexpr>k  s    è ø€ Ò#lÁÀÀvÒek f¨T¨F°)Ô$<Ñ#lùs   ‚
�z¾
        from typing import TYPE_CHECKING

        from ...utils import _LazyModule
        from ...utils.import_utils import define_import_structure


        if TYPE_CHECKING:
            zÐ
        else:
            import sys

            _file = globals()["__file__"]
            sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)
        )ra   r“   r°   r±   r²   )rœ   rF   rH   rž   rŸ   ÚimportsÚ	init_files          r   Úcreate_init_filerË   Z  sš   € ð !1÷áˆH�fð 
×	Ñ	Ð,Ð.@Ó	A×	IÑ	IÈ%ÐQSÓ	TÐV\Ò]ðÐñ ð ×#Ñ#Ñ#lÐQaÔ#lÓl€GÜœHŸO™Oðð ˆIð 	ð	óñ €Ið" Ðùó-s   †.A1Úmodule_namer   c                 óø   — t        | dd¬«      5 }|j                  «       }ddd«       t        j                  «      }t	        «       }|j                  |«       |j                  |j                  fS # 1 sw Y   ŒQxY w)zä
    Find the name of all classes defined in `module_name`, including public ones (defined in `__all__`).

    Args:
        module_name (`str`):
            The full path to the python module from which to extract classes.
    rq   rr   rs   N)rw   rx   r-   Úparse_moduler   Úvisitr   r   )rÌ   r¡   Úsource_codeÚmoduleÚvisitors        r   Úfind_all_classes_from_filerÓ   €  sj   € ô 
ˆk˜3¨Ô	1ð "°TØ—i‘i“kˆ÷"ä×Ñ˜kÓ*€FÜ‹m€GØ
‡L�L�ÔØ�?‰?˜G×2Ñ2Ð2Ð2÷"ð "ús   �A0Á0A9r›   c                 óZ  ‡‡	— t        | «      \  }}dj                  | j                  dd «      j                  dd«      }|j                  Š	d|› ddj                  d	„ |D «       «      › �}d
j                  ˆˆ	fd„|D «       «      }|D �cg c]  }|j                  ‰	‰«      ‘Œ }}|||fS c c}w )a�  
    Extract the modular structure that will be needed to copy a file `module_name` using modular.

    Args:
        module_name (`str`):
            The full path to the python module to copy with modular.
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_cased_name (`str`):
            The new cased model name.
    ú.éþÿÿÿNr’   rX   zfrom ..z import z, c              3   ó    K  — | ]  }|–— Œ y ­wr‚   r;   )r†   r·   s     r   rˆ   z)find_modular_structure.<locals>.<genexpr>¡  s   è ø€ Ò:\Àf¼6Ñ:\ùs   ‚r®   c              3   óP   •K  — | ]  }d |j                  ‰‰«      › d|› d�–— Œ y­w)zclass ú(z):
    passN)ra   )r†   r·   r›   r�   s     €€r   rˆ   z)find_modular_structure.<locals>.<genexpr>¢  s1   øè ø€ ò "Ø[aˆ&�—‘ °Ó?Ð@ÀÀ&ÀÈÔVñ"ùs   ƒ#&)rÓ   r“   Úpartsra   rd   )
rÌ   rE   r›   Úall_classesr   Úimport_locationrÉ   Úmodular_classesr·   r�   s
     `      @r   Úfind_modular_structurerÞ   �  s»   ù€ ô #=¸[Ó"IÑ€K�Ø—h‘h˜{×0Ñ0°°Ð5Ó6×>Ñ>¸uÀbÓI€OØ$×3Ñ3€NØ˜Ð(¨°·±Ñ:\ÐP[Ô:\Ó1\Ð0]Ð^€GØ—k‘kô "Øepô"ó €Oð TbÖbÈ�f—n‘n ^°^ÕDÐb€NÐbØ�O ^Ð3Ð3ùò cs   ÂB(c                 ó  — dj                  d„ |j                  dd«      j                  d«      D «       «      }|j                  }| dz  dz  dz  |z  }d}d}g }	|D ]=  \  }
}|sŒ	t	        ||
z  ||«      \  }}}|d|› �z  }|d	|› �z  }|	j                  |«       Œ? d
j                  d„ |	D «       «      }t        j                  d|› d�«      }t        |z   |z   |z   }|	D �cg c]  }|j                  dd«      ‘Œ }	}||	fS c c}w )a™  
    Create a new modular file which will copy the old model, based on the new name and the different filenames
    (modules) to add.

    Args:
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_lowercase_name (`str`):
            The new lowercase model name.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    rX   c              3   ó<   K  — | ]  }|j                  «       –— Œ y ­wr‚   rƒ   r…   s     r   rˆ   z&create_modular_file.<locals>.<genexpr>¼  r‰   rŠ   rV   rU   r�   rŽ   r�   r¿   r®   rÇ   c              3   ó&   K  — | ]	  }|› d �–— Œ y­w)ú,Nr;   )r†   Úpublic_classs     r   rˆ   z&create_modular_file.<locals>.<genexpr>Î  s   è ø€ Ò4oÈL¸°~ÀQÔ5GÑ4oùs   ‚z"

        __all__ = [
            z
        ]
        ú")	r“   ra   ry   rO   rÞ   Úextendr±   r²   r°   )rA   rE   rF   rH   r›   rœ   Úold_folder_rootÚall_importsÚ
all_bodiesÚall_public_classesrž   rŸ   rÉ   r%   r   Úpublic_classes_formattedÚall_statementÚmodular_filerã   s                      r   Úcreate_modular_filerí   ©  sW  € ð& —W‘WÑ`Ð0B×0JÑ0JÈ3ÐPSÓ0T×0ZÑ0ZÐ[^Ó0_Ô`Ó`€NØ(×7Ñ7ÐØ %Ñ'¨.Ñ8¸8ÑCÐFXÑX€Oð €KØ€JØÐØ,ò 6Ñˆ�&ÚÜ,BØ (Ñ*¨O¸^ó-Ñ)ˆG�T˜>ð ˜R ˜y˜>Ñ)ˆKØ˜D  ˜-Ñ'ˆJØ×%Ñ% nÕ5ð6ð  0×4Ñ4Ñ4oÐ\nÔ4oÓoÐÜ—O‘Oðð &Ð&ð '	ð	ó€Mô ˜{Ñ*¨ZÑ7¸-ÑG€LàL^Ö_¸L˜,×.Ñ.¨s°BÕ7Ð_ÐÐ_ØÐ+Ð+Ð+ùò `s   ÃC=c           	      óf  — dj                  d„ |j                  dd«      j                  d«      D «       «      }|j                  }|j                  }|dd D ��cg c]  \  }}d|j                  ||«      z   |f‘Œ }}}g }	|D ]F  \  }
}t        j                  d|› d	�|
«      r|	d
   \  }}||xs |f|	d
<   Œ4|	j                  |
|f«       ŒH i }|	D ]Ï  \  }}|sŒ	|j                  ||«      }| dz  dz  |z  |z  }|j                  «       sŒ:t        |d«      5 }|j                  «       }ddd«       j                  d«      }d}||   j                  d«      r|dz  }||   j                  d«      rŒt        dj                  ||d «      z   }|j                  ||«      ||<   ŒÑ |S c c}}w # 1 sw Y   Œ„xY w)a‰  
    Create the test files for the new model. It basically copies over the old test files and adjust the class names.

    Args:
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_lowercase_name (`str`):
            The new lowercase model name.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    rX   c              3   ó<   K  — | ]  }|j                  «       –— Œ y ­wr‚   rƒ   r…   s     r   rˆ   z$create_test_files.<locals>.<genexpr>í  r‰   rŠ   rV   rU   ru   NÚtest_z+test_(?:tokenization)|(?:image_processing)_r‘   rŒ   Útestsr�   rq   r¿   r   ú#)r“   ra   ry   rO   rd   r–   r—   r   Úis_filerw   rx   Ú
startswithr°   )rA   rE   rF   rH   r›   rœ   r�   rž   rŸ   r    r¡   r¢   r£   Ú
test_filesÚnew_fileÚoriginal_test_fileÚoriginal_test_pathr{   Ú	test_codeÚ
test_linesÚidxs                        r   Úcreate_test_filesrü   Þ  s  € ð —W‘WÑ`Ð0B×0JÑ0JÈ3ÐPSÓ0T×0ZÑ0ZÐ[^Ó0_Ô`Ó`€NØ(×7Ñ7ÐØ$×3Ñ3€Nð !1°°Ð 4÷áˆH�fð 
�8×#Ñ#Ð$6Ð8JÓKÑ	KÈVÒTðÐñ ð
 "$ÐØ(ò >‰ˆˆfÜ�9‰9ÐCÐDVÐCWÐW_Ð`ÐbfÔgØ-GÈÑ-KÑ*ˆM˜?Ø.;¸_Ò=VÐPVÐ-WÐ& rÒ*à&×-Ñ-¨t°V¨nÕ=ð>ð €JØ6ò UÑˆ�&ÚØ!)×!1Ñ!1Ð2DÐFXÓ!YÐØ!*¨WÑ!4°xÑ!?ÐBTÑ!TÐWiÑ!iÐà%×-Ñ-Ô/ØÜÐ(¨#Ó.ð %°!ØŸF™F›H�	÷%ð #Ÿ™¨Ó.ˆJØˆCØ˜S‘/×,Ñ,¨SÔ1Ø�q‘�ð ˜S‘/×,Ñ,¨SÕ1ä! D§I¡I¨j¸¸Ð.>Ó$?Ñ?ˆIØ#,×#4Ñ#4°^À^Ó#TˆJ�xÒ ðUð" Ðùó?÷*%ð %ús   Á!F!ÄF'Æ'F0	c                 óÈ  — t        «       st        d«      ‚|j                  }| dz  dz  dz  |z  }t        j                  |d¬«       t        | |||«      \  }}t        |d|› d�z  d	«      5 }	|	j                  |«       d
d
d
«       t        |||«      }
t        |dz  d	«      5 }	|	j                  |
«       d
d
d
«       t        | dz  dz  dz  dz  d|› d�d¬«       t        | ||||«       | dz  dz  |z  }t        j                  |d¬«       t        |dz  d	«      5  	 d
d
d
«       t        | |||«      }|j                  «       D ].  \  }}t        ||z  d	«      5 }	|	j                  |«       d
d
d
«       Œ0 t        ||«      }t        | dz  dz  dz  dz  |› d�z  d	«      5 }	|	j                  |«       d
d
d
«       t        | |||«       | dz  dz  dz  dz  }t        j                   dd|||dg| t        j"                  ¬«       t        j                   dd|||g| t        j"                  ¬«       t        j                   g d¢| t        j"                  ¬«       t        j                   ddg| t        j"                  ¬«       t        j                   dd|g| t        j"                  ¬«       y
# 1 sw Y   �Œ(xY w# 1 sw Y   �ŒÿxY w# 1 sw Y   �Œ¤xY w# 1 sw Y   �Œ“xY w# 1 sw Y   �Œ+xY w)a9  
    Creates a new model module like a given model of the Transformers library.

    Args:
        repo_path (`Path`):
            The path to the root of the Transformers repository.
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_lowercase_name (`str`):
            The new lowercase model name.
        new_model_paper_name (`str`):
            The fully cased name (as in the official paper name) of the new model.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    zHYou need to install `libcst` to run this command -> `pip install libcst`r�   rŽ   r�   T)Úexist_okÚmodular_r’   rv   Nz__init__.pyz
    from .z
 import *
zif TYPE_CHECKING:
r�   rñ   r¼   r½   r¾   Ú	model_docz.mdÚruffÚcheckz--fix)ÚcwdÚstdoutÚformat)Úpythonzutils/check_doc_toc.pyz--fix_and_overwriter  zutils/sort_auto_mappings.pyz utils/modular_model_converter.py)r
   rb   rO   ÚosÚmakedirsrí   rw   rz   rË   r   rª   rü   Úitemsrº   rÅ   Ú
subprocessÚrunÚDEVNULL)rA   rE   rF   rG   rH   rœ   Únew_module_folderrì   r   r{   rÊ   Útests_folderrõ   rž   rÁ   Údoc_fileÚmodel_init_files                    r   rJ   rJ     s*  € ô0 Ô ÜÐcÓdÐdà(×7Ñ7Ðð " EÑ)¨NÑ:¸XÑEÐHZÑZÐÜ‡K�KÐ!¨DÕ1ô $7Ø�?Ð$6Ð8Hó$Ñ €L�.ô 
Ð HÐ-?Ð,@ÀÐ"DÑDÀcÓ	Jð ÈaØ	�‰�Ô÷ô !Ð!3Ð5GÐIYÓZ€IÜ	Ð -Ñ/°Ó	5ð ¸Ø	�‰�	Ô÷ô Ø�EÑ˜NÑ*¨XÑ5¸ÑEØ Ð!3Ð 4°KÐ@Ø'õô ˜y¨/Ð;MÐOcÐeuÔvð ˜wÑ&¨Ñ1Ð4FÑF€LÜ‡K�K� tÕ,ä	ˆl˜]Ñ*¨CÓ	0ñ Ø÷ä" 9¨oÐ?QÐScÓd€JØ'×-Ñ-Ó/ò Ñˆ�'Ü�, Ñ)¨3Ó/ð 	°1Ø�G‰G�GÔ÷	ð 	ðô
 Ð3°^ÓD€HÜ	ˆi˜&Ñ  8Ñ+¨dÑ2°[Ñ@ÐFXÐEYÐY\ÐC]Ñ]Ð_bÓ	cð ÐghØ	�‰�Ô÷ä˜IÐ'9Ð;MÐOcÔdð   %Ñ'¨.Ñ8¸8ÑCÀmÑS€OÜ‡N�NØ	�Ð+¨\¸?ÈGÐTØÜ×!Ñ!õô
 ‡N�NØ	�Ð,¨l¸OÐLØÜ×!Ñ!õô
 ‡N�NÚCÈÔ[e×[mÑ[mõô ‡N�N�HÐ;Ð<À)ÔT^×TfÑTfÕgô ‡N�NØ	Ð5Ð7IÐJÐPYÔbl×btÑbtö÷kñ ú÷
ñ ú÷"ñ ú÷	ñ 	ú÷
ñ ús<   Á+J#Â!J0ÄJ=ÅK
ÆKÊ#J-Ê0J:Ê=KË
K	ËK!ÚquestionÚdefault_valueÚ
convert_toÚfallback_messagec                 óê   — | j                  d«      s| dz   } |�| › d|› d�} d}|s=t        | «      }|�t        |«      dk(  r|}|�	  ||«      }d}nd}|st	        |«       |sŒ=S # t        $ r d}Y Œw xY w)a4  
    A utility function that asks a question to the user to get an answer, potentially looping until it gets a valid
    answer.

    Args:
        question (`str`):
            The question to ask the user.
        default_value (`str`, *optional*):
            A potential default value that will be used when the answer is empty.
        convert_to (`Callable`, *optional*):
            If set, the answer will be passed to this function. If this function raises an error on the provided
            answer, the question will be asked again.
        fallback_message (`str`, *optional*):
            A message that will be displayed each time the question is asked again to the user.

    Returns:
        `Any`: The answer provided by the user (or the default), passed through the potential conversion function.
    rT   z [z] Fr   T)ÚendswithÚinputÚlenÚ	ExceptionÚprint)r  r  r  r  Úvalid_answerÚanswers         r   Úget_user_fieldr  q  s©   € ð0 ×Ñ˜SÔ!Ø˜c‘>ˆØÐ Ø�Z˜r - °Ð3ˆà€LÙÜ�x“ˆØÐ$¬¨V«¸Ò)9Ø"ˆFØÐ!ð%Ù# FÓ+�Ø#‘ð  ˆLáÜÐ"Ô#ò ð  €Møô ò %Ø$’ð%ús   Á
A$ Á$A2Á1A2r‡   c                 ój   — | j                  «       dv ry| j                  «       dv ryt        | › d�«      ‚)z&
    Converts a string to a bool.
    )Ú1ÚyÚyesÚtrueT)Ú0ÚnÚnoÚfalseFz0 is not a value that can be converted to a bool.)r`   rb   )r‡   s    r   Úconvert_to_boolr'  ¢  s;   € ð 	‡w�wƒyÐ-Ñ-ØØ‡w�wƒyÐ-Ñ-ØÜ
˜�sÐJÐKÓ
LÐLr   c            
      ó
  — ddl m}  t        | j                  «       «      }d}|sut	        d«      }||v rd}n`t        |› d�«       t        j                  ||«      }t        |«      dk\  r.t        |«      dkD  rdj                  |«      }t        d	|› d
�«       |sŒut        «      }t        d«      }t        ddj                  d„ |j                  d«      D «       «      ¬«      }d}d}	d}
d}d}d}|j                  �t        d|› d�t        d¬«      }|j                  �t        d|› d�t        d¬«      }	|j                   �t        d|› d�t        d¬«      }
|j"                  �t        d|› d�t        d¬«      }|j$                  �t        d|› d�t        d¬«      }|j&                  �t        d|› d�t        d¬«      }|j(                  }d|› d�dfd|› d�dfd|› d�|fd|› d�|	fd|› d�|
fd |› d�|fd!|› d�|fd"|› d�|ff}||||fS )#zE
    Ask the user for the necessary inputs to add the new model.
    r   rQ   FzWWhat model would you like to duplicate? Please provide it as lowercase, e.g. `llama`): Tz is not a valid model type.ru   z or zDid you mean ú?zSWhat is the new model name? Please provide it as snake lowercase, e.g. `new_model`?zUWhat is the fully cased name you would like to appear in the doc (e.g. `NeW ModEl`)? rX   c              3   ó<   K  — | ]  }|j                  «       –— Œ y ­wr‚   rƒ   r…   s     r   rˆ   z!get_user_input.<locals>.<genexpr>Í  s   è ø€ ÒO¨A˜aŸg™gŸiÑOùrŠ   rU   )r  zHDo you want to create a new tokenizer? If `no`, it will use the same as z (y/n)?z.Please answer yes/no, y/n, true/false or 1/0. )r  r  zMDo you want to create a new fast tokenizer? If `no`, it will use the same as zNDo you want to create a new image processor? If `no`, it will use the same as zNDo you want to create a new video processor? If `no`, it will use the same as zPDo you want to create a new feature extractor? If `no`, it will use the same as zHDo you want to create a new processor? If `no`, it will use the same as Úconfiguration_r’   Ú	modeling_Útokenization_r‘   Úimage_processing_Úvideo_processing_Úfeature_extraction_Úprocessing_)Ú+transformers.models.auto.configuration_autorR   ÚlistÚkeysr  r  ÚdifflibÚget_close_matchesr  r“   rN   r  ry   re   r'  rf   rh   ri   rj   rk   rO   )rR   Úmodel_typesÚvalid_model_typeÚold_model_typeÚnear_choicesrE   rF   rG   Úadd_tokenizerÚadd_fast_tokenizerÚadd_image_processorÚadd_video_processorÚadd_feature_extractorÚadd_processorrœ   rH   s                   r   rI   rI   ­  së  € õ QäÐ+×0Ñ0Ó2Ó3€Kð ÐÙÜØeó
ˆð ˜[Ñ(Ø#Ñä�^Ð$Ð$?Ð@ÔAÜ"×4Ñ4°^À[ÓQˆLÜ�<Ó  AÒ%Ü�|Ó$ qÒ(Ø#)§;¡;¨|Ó#<�LÜ˜ l ^°1Ð5Ô6ò ô ! Ó0€Oô (Ø]óÐô *Ø_Ø—g‘gÑOÐ1C×1IÑ1IÈ#Ó1NÔOÓOôÐð €MØÐØÐØÐØ!ÐØ€MØ×&Ñ&Ð2Ü&ØVÐWeÐVfÐfmÐnÜ&ØMô
ˆð
 ×+Ñ+Ð7Ü+Ø[Ð\jÐ[kÐkrÐsÜ&ØMô
Ðð
 ×.Ñ.Ð:Ü,Ø\Ð]kÐ\lÐlsÐtÜ&ØMô
Ðð
 ×,Ñ,Ð8Ü,Ø\Ð]kÐ\lÐlsÐtÜ&ØMô
Ðð
 ×.Ñ.Ð:Ü .Ø^Ð_mÐ^nÐnuÐvÜ&ØMô!
Ðð
 ×&Ñ&Ð2Ü&ØVÐWeÐVfÐfmÐnÜ&ØMô
ˆð )×7Ñ7Ðð Ð,Ð-¨SÐ	1°4Ð8ØÐ'Ð(¨Ð	,¨dÐ3ØÐ+Ð,¨CÐ	0°-Ð@ØÐ+Ð,¨HÐ	5Ð7IÐJØÐ/Ð0°Ð	4Ð6IÐJØÐ/Ð0°Ð	4Ð6IÐJØÐ1Ð2°#Ð	6Ð8MÐNØÐ)Ð*¨#Ð	.°Ð>ð	Ðð Ð.Ð0DÐFVÐVÐVr   r‚   )NNN)7r5  r  r–   r
  r±   Úcollections.abcr   Údatetimer   Úpathlibr   Útypingr   r   r   ÚtyperÚutilsr
   Úlibcstr-   r   r   r'   r   ÚtodayÚyearÚCURRENT_YEARÚ__file__ÚparentsrK   Úlstripr°   r•   rl   ÚArgumentrL   rN   ÚPathLiker   r3  ÚtupleÚboolrª   rº   rÅ   rË   ÚsetrÓ   rÞ   rí   rü   rJ   r  r'  rI   r;   r   r   ú<module>rS     s$  ðó Û 	Û 	Û Û Ý $Ý Ý ß 'Ñ 'ã å 'ñ
 ÔÛÝ!Ý$ôp�jô pð> ˆt�z‰z‹|× Ñ €Ù�‹N×"Ñ" 1Ñ%€	ðàˆNð ð÷ �FƒHð 
ð& !@Ø ?Ø3Ø"Cñ	Ð ð 	ñØØˆd‰
�N�E—N‘NÐ(pÔqÐqñó÷6$Vñ $VðN 3¨¯©Ñ#4ð À3ð ÐSVó ð,IØðIàðIð ðIð ð	Ið
 ˜5  d Ñ+Ñ,óIðX5= Cð 5=¸¸c¹ó 5=ðpØðØ),ðØBEðØ]`óð8#¨ð #À#ð #ÐY]Ð^cÐdgÐimÐdmÑ^nÑYoó #ðL3¨Cð 3°Có 3ð 4Øð4Ø(2ð4ØDGð4à
ˆ3��Tˆ>Ñó4ð22,Øð2,àð2,ð ð2,ð ˜5  d Ñ+Ñ,ð	2,ð
 	ó2,ðj1Øð1Ø&0ð1ØX\Ð]bÐcfÐhlÐclÑ]mÑXnó1ðh\Øð\àð\ð ð\ð ð	\ð
 ˜5  d Ñ+Ñ,ó\ðB !%Ø"&Ø#'ñ	.Øð.à˜‘:ð.ð ˜4‘ð.ð ˜D‘jð	.ð
 	ó.ðbM�sð M˜tó Mó\Wr   