
    ^j~                     P   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZmZmZ d dlZddlmZ  e       rd dlZd dlmZ d d	lmZ  G d
 de      Z ej0                         j2                  Z e
e      j8                  d   Zde dj=                         ZdddddZ 	 d9dee!dz   ejD                  d      f   fdZ# G d d      Z$de!ejJ                  z  de!de!fdZ&de
de$de!d e!d!e'e(e!e)f      f
d"Z*d#e!d$e'e!   fd%Z+de
d&e!de!d e!fd'Z,d&e!de!d!e'e(e!e)f      fd(Z-d)e!d*e.fd+Z/d)e
de$d,e!d*e(e!e!e'f   fd-Z0de
de$de!d!e'e(e!e)f      d*e!f
d.Z1de
de$d!e'e(e!e)f      fd/Z2de
de$de!d e!d!e'e(e!e)f      f
d0Z3	 	 	 d:d1e!d2e!dz  d3edz  d4e!dz  d*ef
d5Z4d6e!d*e)fd7Z5d8 Z6y);    N)Callable)date)Path)	AnnotatedAnycast   )is_libcst_available)
CSTVisitor)matchersc                   z    e Zd ZdZd Zdej                  ddfdZdej                  fdZdej                  fdZ
y)	ClassFinderzC
        A visitor to find all classes in a python module.
        c                 .    g | _         g | _        d| _        y NF)classespublic_classesis_in_class)selfs    n/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/cli/add_new_model_like.py__init__zClassFinder.__init__)   s    !#DL(*D$D    nodereturnNc                 p    | j                   j                  |j                  j                         d| _        y)zwRecord class names. We assume classes always only appear at top-level (i.e. no class definition in function or similar)TN)r   appendnamevaluer   r   r   s     r   visit_ClassDefzClassFinder.visit_ClassDef.   s$    LL		0#Dr   c                     d| _         y r   )r   r   s     r   leave_ClassDefzClassFinder.leave_ClassDef3   s
    $Dr   c                    t        j                  t        j                  t        j                  t        j                               g      g      }| j
                  st        j                  ||      rt        t        j                  |j                  d         }t        t        j                  |j                  d   j                        j                  }|dk(  rpt        t        j                  |j                        j                  }|D cg c]0  }t        t        j                  |j                        j                  2 c}| _        yyyyc c}w )z:Record all public classes inside the `__all__` assignment.)target)targets)bodyr   __all__N)mSimpleStatementLineAssignAssignTargetNamer   matchesr   cstr%   r$   r#   r   TupleelementsSimpleStringr   )r   r   !simple_top_level_assign_structurestmtassigned_variabler/   elements          r   visit_SimpleStatementLinez%ClassFinder.visit_SimpleStatementLine6   s    010E0Ehhaffh(G'HIJ1- ##		$8Y(ZCJJ		!5$(4<<?3I3I$J$P$P!$	1#CIItzz:CCHfn*o[b40@0@'--+P+V+V*oD' 2 )[#
 +ps   5E)__name__
__module____qualname____doc__r   r-   ClassDefr   r!   r(   r5    r   r   r   r   $   sG    		%
	$s|| 	$ 	$
	%s|| 	%
	p#2I2I 
	pr   r      z
# coding=utf-8
# Copyright aN   the HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
IMAGE_PROCESSOR_MAPPING_NAMESVIDEO_PROCESSOR_MAPPING_NAMESPROCESSOR_MAPPING_NAMESFEATURE_EXTRACTOR_MAPPING_NAMES)zimage_processing_auto.pyzvideo_processing_auto.pyzprocessing_auto.pyzfeature_extraction_auto.py	repo_pathzFWhen not using an editable install, the path to the Transformers repo.)helpc                 d    t               \  }}}}t        | t        |       nt        ||||       y)zC
    Add a new model to the library, based on an existing one.
    NrA   old_model_infosnew_lowercase_namenew_model_paper_namefilenames_to_add)get_user_input_add_new_model_like_internalr   	REPO_PATHrD   s        r   add_new_model_likerL   b   s?     	 !%.%:$y/	'-1)r   c                       e Zd ZdZdefdZy)
ModelInfoszI
    Retrieve the basic information about an existing model classes.
    lowercase_namec                 j   ddl m} ddlm} ddlm} ddlm} ddlm	} ddl
m} |j                         j                  dd	      j                  d
d	      | _        | j                  |vr| j                  j                  d	d
       | j                  |vrt        | d      || j                     | _        | j                   j                  dd      | _        | j                  |v r>d | _        || j                     | _        | j&                  dk(  rd n| j&                  | _        nd\  | _        | _        |j)                  | j                  d       | _        |j)                  | j                  d       | _        |j)                  | j                  d       | _        |j)                  | j                  d       | _        y )Nr	   CONFIG_MAPPING_NAMES)r@   )r=   )r?   )TOKENIZER_MAPPING_NAMES)r>    _-z is not a valid model nameConfig PreTrainedTokenizerFast)NN)models.auto.configuration_autorR   #models.auto.feature_extraction_autor@   !models.auto.image_processing_autor=   models.auto.processing_autor?   models.auto.tokenization_autorS   !models.auto.video_processing_autor>   lowerreplacerO   
ValueErrorconfig_classcamelcase_nametokenizer_classfast_tokenizer_classgetimage_processor_classesvideo_processor_classfeature_extractor_classprocessor_class)r   rO   rR   r@   r=   r?   rS   r>   s           r   r   zModelInfos.__init__   s   IYUIKU -224<<S#FNNsTWX&::''S1&:://IJKK01D1DE"//77"E "99#'D (?@S@S(TD%115NNTXTmTm % ?I;D $";'D'H'HI\I\^b'c$%B%F%FtGZGZ\`%a"'F'J'J4K^K^`d'e$6::4;N;NPTUr   N)r6   r7   r8   r9   strr   r;   r   r   rN   rN   }   s    Vs Vr   rN   	file_namenew_content	add_afterc                    t        | dd      5 }|j                         }ddd       j                  |d      \  }}||z   |z   |z   }t        | dd      5 }|j                  |       ddd       y# 1 sw Y   RxY w# 1 sw Y   yxY w)ac  
    A utility to add some content inside a given file.

    Args:
        file_name (`str` or `os.PathLike`):
            The name of the file in which we want to insert some content.
        new_content (`str`):
            The content to add.
       add_after (`str`):
           The new content is added just after the first instance matching it.
    rutf-8encodingN   w)openreadsplitwrite)rm   rn   ro   fold_contentbeforeafters          r   add_content_to_filer      s     
iw	/ 1ffh  %%i3MFE9${2U:K	iw	/ 1	   s   A1A=1A:=BrE   rF   rG   rH   c           	      <   dj                  d |j                  dd      j                  d      D              }|j                  }|j                  }|dd D 	cg c]  \  }}	|j                  |d      |	f }}}	g }
|D ]0  \  }}	d|v r|
d	   \  }}||xs |	f|
d	<   |
j                  ||	f       2 t        | d
z  dz  dz  dz  dz  d| d| dd       | d
z  dz  dz  dz  dz  j                         }|
D ]  \  }}	|	s
|t        v rot        |   }d}t        j                  d| d|t        j                        }|j                  d      }t        j                  d| d|t        j                        }nZ|j                  dd      }| d
z  dz  dz  dz  |z  j                         }t        j                  d| d|t        j                        }|D ]?  }t        | d
z  dz  dz  dz  |z  |j                  ||      j                  ||      |       A ! yc c}	}w )a  
    Add a model to all the relevant mappings in the auto module.

    Args:
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_lowercase_name (`str`):
            The new lowercase model name.
        new_model_paper_name (`str`):
            The fully cased name (as in the official paper name) of the new model.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    rX   c              3   <   K   | ]  }|j                           y wNtitle.0xs     r   	<genexpr>z-add_model_to_auto_mappings.<locals>.<genexpr>        `1QWWY`   rV   rU   ru   Nautoztokenization_auto_fast.pysrctransformersmodelszauto_mappings.pyz("z", "zConfig"),
        z2CONFIG_MAPPING_NAMES = OrderedDict(
    [
        rn   ro   z[^\w_]z%\s*=\s*OrderedDict\(\s*\[(.*?)\]\s*\)z( {8,12}\(\s*"z",.*?\),\n)(?: {4,12}\(|\])_fast.py.py)joinra   ry   rO   rd   r   r   	read_text_AUTO_MAPPING_NAMESresearchDOTALLgroupfindall)rA   rE   rF   rG   rH   new_cased_nameold_lowercase_nameold_cased_namefilenameto_addcorrected_filenames_to_addfileprevious_fileprevious_to_addautofilemapping_nameblock_matchblockmatching_linesmatchs                       r   add_model_to_auto_mappingsr      s   * WW`0B0J0J3PS0T0Z0Z[^0_``N(77$33NWghihjWkCS8V		,f	5v>  "$( >f&$.-G-K*M?.;_=VPV-W&r*&--tVn=> EN*X5>ASS+,D0@@STH
 E!N2X=FI[[ffhH6 &..28<- iil^+PQS[]_]f]f $))!,!#'(:';;XY[`bdbkbk"
 $++J>!E)N:XENQYYddf!#'(:';;XY[_acajaj" ( #%6AFJXU %.@BT U ] ]&! $-)s   Hnew_paper_namer   c                 "   d}t        j                  ddt              j                  dd      |z   }t	        j
                  d|  d|  d      }g }|D ]$  }d	| d
| }d|v r|dz  }|j                  |       & dj                  |      }||z   |z   S )a4  
    Create a new doc file to fill for the new model.

    Args:
        new_paper_name (`str`):
            The fully cased name (as in the official paper name) of the new model.
        public_classes (`list[str]`):
            A list of all the public classes that the model will have in the library.
    u   

⚠️ Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be rendered properly in your Markdown viewer.

-->

z# ?rX   zcoding=utf-8
z<!--z
        # z#

        ## Overview

        The a<   model was proposed in [<INSERT PAPER NAME HERE>](<INSERT PAPER LINK HERE>) by <INSERT AUTHORS HERE>.
        <INSERT SHORT SUMMARY HERE>

        The abstract from the paper is the following:

        <INSERT PAPER ABSTRACT HERE>

        Tips:

        <INSERT TIPS ABOUT MODEL HERE>

        This model was contributed by [INSERT YOUR HF USERNAME HERE](https://huggingface.co/<INSERT YOUR HF USERNAME HERE>).
        The original code can be found [here](<INSERT LINK TO GITHUB REPO HERE>).

        ## Usage examples

        <INSERT SOME NICE EXAMPLES HERE>

        z## z

[[autodoc]] Modelz
    - forward

)r   sub	COPYRIGHTra   textwrapdedentr   r   )	r   r   
added_notecopyright_for_markdowndoc_templatedoc_for_classesclass_doc	class_docs	            r   create_doc_filer     s    	K   VVFB	:BBCSU[\_ii??
   		L8 O  $F8+F84f$$Cs#	$ O,I!L09<<r   r   c                 >   | dz  dz  dz  dz  }t        |d      5 }|j                         }ddd       t        j                  d| d      }|t	        d	|       |j                  d
      }d| d| d}	t        | dz  dz  dz  dz  |	|       y# 1 sw Y   hxY w)a  
    Insert the new model in the doc `_toctree.yaml`, in the same section as the old model.

    Args:
        old_lowercase_name (`str`):
            The old lowercase model name.
        new_lowercase_name (`str`):
            The new lowercase model name.
        new_model_paper_name (`str`):
            The fully cased name (as in the official paper name) of the new model.
    docssourceenz_toctree.ymlrq   Nz- local: model_doc/z\n {8}title: .*?\nzCould not find TOC entry for r   z      - local: model_doc/z
        title: 
r   )rw   rx   r   r   rb   r   r   )
rA   r   rF   rG   toc_filer{   content	toc_matchold_model_tocnew_tocs
             r   insert_model_in_doc_tocr   >  s     6!H,t3nDH	h	 &&( 		01C0DDXY[bcI89K8LMNNOOA&M)*<)==NOcNddfgGFX%,~=7^k s   BBc                     |D cg c])  \  }}|j                  | |      j                  dd      |f+ }}}dj                  d |D              }t        t        j                  d| d      z   }|S c c}}w )a/  
    Create the `__init__.py` file to add in the new model folder.

    Args:
        old_lowercase_name (`str`):
            The old lowercase model name.
        new_lowercase_name (`str`):
            The new lowercase model name.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    r   rX   
            c              3   4   K   | ]  \  }}|s	d | d  yw)zfrom .z	 import *Nr;   )r   r   r   s      r   r   z#create_init_file.<locals>.<genexpr>k  s      #lvekfTF)$<#ls   
z
        from typing import TYPE_CHECKING

        from ...utils import _LazyModule
        from ...utils.import_utils import define_import_structure


        if TYPE_CHECKING:
            z
        else:
            import sys

            _file = globals()["__file__"]
            sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)
        )ra   r   r   r   r   )r   rF   rH   r   r   imports	init_files          r   create_init_filer   Z  s     !1Hf 
		,.@	A	I	I%QS	TV\]  ###lQa#llGHOO I 		 I" -s   .A1module_namer   c                     t        | dd      5 }|j                         }ddd       t        j                        }t	               }|j                  |       |j                  |j                  fS # 1 sw Y   QxY w)z
    Find the name of all classes defined in `module_name`, including public ones (defined in `__all__`).

    Args:
        module_name (`str`):
            The full path to the python module from which to extract classes.
    rq   rr   rs   N)rw   rx   r-   parse_moduler   visitr   r   )r   r   source_codemodulevisitors        r   find_all_classes_from_filer     sj     
k3	1 "Tiik"k*FmG
LL??G2222" "s   A00A9r   c                 Z  	 t        |       \  }}dj                  | j                  dd       j                  dd      }|j                  	d| ddj                  d	 |D               }d
j                  	fd|D              }|D cg c]  }|j                  	       }}|||fS c c}w )a  
    Extract the modular structure that will be needed to copy a file `module_name` using modular.

    Args:
        module_name (`str`):
            The full path to the python module to copy with modular.
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_cased_name (`str`):
            The new cased model name.
    .Nr   rX   zfrom ..z import z, c              3       K   | ]  }|  y wr   r;   )r   r   s     r   r   z)find_modular_structure.<locals>.<genexpr>  s     :\f6:\s   r   c              3   P   K   | ]  }d |j                         d| d  yw)zclass (z):
    passN)ra   )r   r   r   r   s     r   r   z)find_modular_structure.<locals>.<genexpr>  s1      "[a&?@&V"s   #&)r   r   partsra   rd   )
r   rE   r   all_classesr   import_locationr   modular_classesr   r   s
     `      @r   find_modular_structurer     s     #=["IKhh{0056>>ubIO$33N(:\P[:\1\0]^Gkk "ep" O Tbbfnn^^DbNbO^33 cs   B(c                    dj                  d |j                  dd      j                  d      D              }|j                  }| dz  dz  dz  |z  }d}d}g }	|D ]=  \  }
}|s	t	        ||
z  ||      \  }}}|d| z  }|d	| z  }|	j                  |       ? d
j                  d |	D              }t        j                  d| d      }t        |z   |z   |z   }|	D cg c]  }|j                  dd       }	}||	fS c c}w )a  
    Create a new modular file which will copy the old model, based on the new name and the different filenames
    (modules) to add.

    Args:
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_lowercase_name (`str`):
            The new lowercase model name.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    rX   c              3   <   K   | ]  }|j                           y wr   r   r   s     r   r   z&create_modular_file.<locals>.<genexpr>  r   r   rV   rU   r   r   r   r   r   r   c              3   &   K   | ]	  }| d   yw),Nr;   )r   public_classs     r   r   z&create_modular_file.<locals>.<genexpr>  s     4oL~Q5G4os   z"

        __all__ = [
            z
        ]
        ")	r   ra   ry   rO   r   extendr   r   r   )rA   rE   rF   rH   r   r   old_folder_rootall_imports
all_bodiesall_public_classesr   r   r   r%   r   public_classes_formattedall_statementmodular_filer   s                      r   create_modular_filer     sW   & WW`0B0J0J3PS0T0Z0Z[^0_``N(77%'.88CFXXO KJ, 6&,B(*O^-)GT> Ry>)KD-'J%%n56  0444o\n4ooOO && '		M {*Z7-GLL^_L,..sB7__+++ `s   C=c           	      f   dj                  d |j                  dd      j                  d      D              }|j                  }|j                  }|dd D cg c]  \  }}d|j                  ||      z   |f }}}g }	|D ]F  \  }
}t        j                  d| d	|
      r|	d
   \  }}||xs |f|	d
<   4|	j                  |
|f       H i }|	D ]  \  }}|s	|j                  ||      }| dz  dz  |z  |z  }|j                         s:t        |d      5 }|j                         }ddd       j                  d      }d}||   j                  d      r|dz  }||   j                  d      rt        dj                  ||d       z   }|j                  ||      ||<    |S c c}}w # 1 sw Y   xY w)a  
    Create the test files for the new model. It basically copies over the old test files and adjust the class names.

    Args:
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_lowercase_name (`str`):
            The new lowercase model name.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    rX   c              3   <   K   | ]  }|j                           y wr   r   r   s     r   r   z$create_test_files.<locals>.<genexpr>  r   r   rV   rU   ru   Ntest_z+test_(?:tokenization)|(?:image_processing)_r   r   testsr   rq   r   r   #)r   ra   ry   rO   rd   r   r   r   is_filerw   rx   
startswithr   )rA   rE   rF   rH   r   r   r   r   r   r   r   r   r   
test_filesnew_fileoriginal_test_fileoriginal_test_pathr{   	test_code
test_linesidxs                        r   create_test_filesr     s    WW`0B0J0J3PS0T0Z0Z[^0_``N(77$33N !1 4Hf 
8##$68JK	KVT 
 "$( >f99CDVCWW_`bfg-G-K*M?.;_=VPV-W&r*&--tVn=> J6 U&!)!1!12DFX!Y!*W!4x!?BT!TWi!i%--/(#. %!FFH	% #.JCS/,,S1q S/,,S1!DIIj.>$??I#,#4#4^^#TJx U" ?*% %s   !F!F''F0	c                    t               st        d      |j                  }| dz  dz  dz  |z  }t        j                  |d       t        | |||      \  }}t        |d| dz  d	      5 }	|	j                  |       d
d
d
       t        |||      }
t        |dz  d	      5 }	|	j                  |
       d
d
d
       t        | dz  dz  dz  dz  d| dd       t        | ||||       | dz  dz  |z  }t        j                  |d       t        |dz  d	      5  	 d
d
d
       t        | |||      }|j                         D ].  \  }}t        ||z  d	      5 }	|	j                  |       d
d
d
       0 t        ||      }t        | dz  dz  dz  dz  | dz  d	      5 }	|	j                  |       d
d
d
       t        | |||       | dz  dz  dz  dz  }t        j                   dd|||dg| t        j"                         t        j                   dd|||g| t        j"                         t        j                   g d| t        j"                         t        j                   ddg| t        j"                         t        j                   dd|g| t        j"                         y
# 1 sw Y   (xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   +xY w)a9  
    Creates a new model module like a given model of the Transformers library.

    Args:
        repo_path (`Path`):
            The path to the root of the Transformers repository.
        old_model_infos (`ModelInfos`):
            The structure containing the class information of the old model.
        new_lowercase_name (`str`):
            The new lowercase model name.
        new_model_paper_name (`str`):
            The fully cased name (as in the official paper name) of the new model.
        filenames_to_add (`list[tuple[str, bool]]`):
            A list of tuples of all potential filenames to add for a new model, along a boolean flag describing if we
            should add this file or not. For example, [(`modeling_xxx.px`, True), (`configuration_xxx.py`, True), (`tokenization_xxx.py`, False),...]
    zHYou need to install `libcst` to run this command -> `pip install libcst`r   r   r   T)exist_okmodular_r   rv   Nz__init__.pyz
    from .z
 import *
zif TYPE_CHECKING:
r   r   r   r   r   	model_docz.mdruffcheckz--fix)cwdstdoutformat)pythonzutils/check_doc_toc.pyz--fix_and_overwriter  zutils/sort_auto_mappings.pyz utils/modular_model_converter.py)r
   rb   rO   osmakedirsr   rw   rz   r   r   r   r   itemsr   r   
subprocessrunDEVNULL)rA   rE   rF   rG   rH   r   new_module_folderr   r   r{   r   tests_folderr   r   r   doc_filemodel_init_files                    r   rJ   rJ     s*   0  cdd(77 "E)N:XEHZZKK!D1 $7?$68H$ L. 
H-?,@"DDc	J a	 !!35GIYZI	-/	5 		 EN*X5E !3 4K@' y/;MOceuv w&14FFLKKt,	l]*C	0 "9o?QScdJ'--/ ',)3/ 	1GGG	 	
 3^DH	i& 8+d2[@FXEYY\C]]_b	c gh	I'9;MOcd  %'.88CmSONN	+\?GT!!
 NN	,lOL!!
 NNC[e[m[m NNH;<)T^TfTfg NN	57IJPYblbtbtk 
 " 	 	
 s<   +J#!J0J=K
K#J-0J:=K
K	K!questiondefault_value
convert_tofallback_messagec                     | j                  d      s| dz   } ||  d| d} d}|s=t        |       }|t        |      dk(  r|}|	  ||      }d}nd}|st	        |       |s=S # t        $ r d}Y w xY w)a4  
    A utility function that asks a question to the user to get an answer, potentially looping until it gets a valid
    answer.

    Args:
        question (`str`):
            The question to ask the user.
        default_value (`str`, *optional*):
            A potential default value that will be used when the answer is empty.
        convert_to (`Callable`, *optional*):
            If set, the answer will be passed to this function. If this function raises an error on the provided
            answer, the question will be asked again.
        fallback_message (`str`, *optional*):
            A message that will be displayed each time the question is asked again to the user.

    Returns:
        `Any`: The answer provided by the user (or the default), passed through the potential conversion function.
    rT   z [z] Fr   T)endswithinputlen	Exceptionprint)r  r  r  r  valid_answeranswers         r   get_user_fieldr  q  s    0 S!c> Zr-3Lx$V)9"F!%#F+#  L"#   M  %$%s   
A$ $A21A2r   c                 j    | j                         dv ry| j                         dv ryt        |  d      )z&
    Converts a string to a bool.
    )1yyestrueT)0nnofalseFz0 is not a value that can be converted to a bool.)r`   rb   )r   s    r   convert_to_boolr'    s;     	wwy--wwy--
sJK
LLr   c            
      
   ddl m}  t        | j                               }d}|sut	        d      }||v rd}n`t        | d       t        j                  ||      }t        |      dk\  r.t        |      dkD  rdj                  |      }t        d	| d
       |sut              }t        d      }t        ddj                  d |j                  d      D                    }d}d}	d}
d}d}d}|j                  t        d| dt        d      }|j                  t        d| dt        d      }	|j                   t        d| dt        d      }
|j"                  t        d| dt        d      }|j$                  t        d| dt        d      }|j&                  t        d| dt        d      }|j(                  }d| ddfd| ddfd| d|fd| d|	fd| d|
fd | d|fd!| d|fd"| d|ff}||||fS )#zE
    Ask the user for the necessary inputs to add the new model.
    r   rQ   FzWWhat model would you like to duplicate? Please provide it as lowercase, e.g. `llama`): Tz is not a valid model type.ru   z or zDid you mean ?zSWhat is the new model name? Please provide it as snake lowercase, e.g. `new_model`?zUWhat is the fully cased name you would like to appear in the doc (e.g. `NeW ModEl`)? rX   c              3   <   K   | ]  }|j                           y wr   r   r   s     r   r   z!get_user_input.<locals>.<genexpr>  s     OAaggiOr   rU   )r  zHDo you want to create a new tokenizer? If `no`, it will use the same as z (y/n)?z.Please answer yes/no, y/n, true/false or 1/0. )r  r  zMDo you want to create a new fast tokenizer? If `no`, it will use the same as zNDo you want to create a new image processor? If `no`, it will use the same as zNDo you want to create a new video processor? If `no`, it will use the same as zPDo you want to create a new feature extractor? If `no`, it will use the same as zHDo you want to create a new processor? If `no`, it will use the same as configuration_r   	modeling_tokenization_r   image_processing_video_processing_feature_extraction_processing_)+transformers.models.auto.configuration_autorR   listkeysr  r  difflibget_close_matchesr  r   rN   r  ry   re   r'  rf   rh   ri   rj   rk   rO   )rR   model_typesvalid_model_typeold_model_typenear_choicesrE   rF   rG   add_tokenizeradd_fast_tokenizeradd_image_processoradd_video_processoradd_feature_extractoradd_processorr   rH   s                   r   rI   rI     s    Q+0023K e
 [(#^$$?@A"44^[QL< A%|$q(#);;|#<Ll^156  !0O (] *_ggO1C1I1I#1NOO M!M&&2&VWeVffmn&M

 ++7+[\j[kkrs&M

 ..:,\]k\llst&M

 ,,8,\]k\llst&M

 ..: .^_m^nnuv&M!

 &&2&VWeVffmn&M
 )77 ,-S	148'(	,d3+,C	0-@+,H	57IJ/0	46IJ/0	46IJ12#	68MN)*#	.>	 .0DFVVVr   r   )NNN)7r5  r  r   r
  r   collections.abcr   datetimer   pathlibr   typingr   r   r   typerutilsr
   libcstr-   r   r   r'   r   todayyearCURRENT_YEAR__file__parentsrK   lstripr   r   rl   ArgumentrL   rN   PathLiker   r3  tupleboolr   r   r   r   setr   r   r   r   rJ   r  r'  rI   r;   r   r   <module>rS     s$    	 	   $   ' '  '
 !$pj p> tzz|  N""1%	N  FH 
& !@ ?3"C	  	d
NENN(pqq6$V $VN3#4 3 SV ,III I 	I
 5d+,IX5=C 5=c 5=p),BE]`8# ## #Y]^cdgimdm^nYo #L3C 3C 3 44(24DG4
3T>422,2,2, 2, 5d+,	2,
 	2,j11&01X\]bcfhlcl]mXn1h\\\ \ 	\
 5d+,\B !%"&#'	..:. 4. Dj	.
 	.bMs Mt M\Wr   