^j,dZddlZddlZddlZddlZddlZddlmZmZddl m Z ddl m Z ddl m Z mZddlmZmZmZddlmZdd lmZmZmZmZmZmZdd lmZdd lmZdd l m!Z!dd l"m#Z#ddl$m%Z%ddl&m'Z'm(Z(ddl)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6ddl7m8Z8m9Z9ddl:m;Z;mZ?de@de@dzdeAe@e@dzffdZBdeCe@dzdede@dzdeCedzfdZDdZEee@e8dfZFee@dze9dfZGee@dze9de.e fZHeeCe@dze9d!d"d#fZIee@dze9d$d%fZJee@dze9d&fZKeeLe9d'd(d)fZMee@dze9d*fZNeeCeOdze9d+d,fZPeeLe9d-d.fZQeeCe@dze9d/d0fZRee@dze9d1d2d3fZSeeLdze9d4fZTeeLdze9d5fZUee@e8d6fZVee@e8d7fZWeeCe@dze8d8fZXeeCe@e8d9fZYee@e8d:fZZeeCe@dze8d;fZ[ee@e8d<fZ\eeCe@dze9d=d>d?fZ]e6d@Z^dedAeddfdBZ_e^jdCdDdEigdFG ddHeFdIeYdJe+dKe-dLeJdMeIdNe]dOe*dPe,dQeHdReKdSeMdTePdUeQdeNdVe1ddf"dWZae^jdXgdYZ ddeZd[eeLe9d\d]d^fd_eeOdze9d`dadbfdeNdVe1ddf dcZbddece@e@fdeeCeAe@e@e@fdeLfdfZddgeOddfdhZede@dieece@efdjeCe@deeAeLe@eCeCe@eOzffdkZfe^jdldmgZ ddne[deNdVe1ddfdoZge^jdpgdqZ ddreeLe9dsdtdufdveeCe@dze9dwe.edxyfdMeeCe@dze9d!d"dzfd{eeOe9d|d}fdeNdVe1d~eeCe@dze9d\ddfddfdZhe^jddgZddZie^jddgZ ddneeCe@e8dfdeNdVe1ddfdZje^jddgZ ddeZdeNdVe1ddfdZke^jdgdZ ddneeCe@e8dfdRee@dze9dfdeNdVe1ddf dZle^jdgdZ ddeZdLeJdMeIdeeLe9ddfdeNdVe1ddfdZme^jdUgdZ ddeZde0de/deNdVe1ddf dZne6dZoe^jeodeojdCdDdEigdG ddeWdeXdHeGdQeHdJe+dKe-dLeJdMeIdNe]dOe*dPe,dReKdSeMdTePdUeQdeNdVe1deRdeSddf(dZqe6dZre^jerderjdCdDdEidgG ddeVdHeFdIeYdeTdeUdJe+dKe-dLeJdMeIdNe]dOe*dPe,dQeHdReKdTePdeNdVe1ddf$dZserjdpdgZ ddreeLe9dsdtdfdeNdVe1d~eeCe@dze9d\ddfddf dZterjddgZ ddeeCe@e8dfdeNdVe1ddfdZuerjddgZ dde\deNdVe1ddfdZverjddgZ dde\deNdVe1ddfdZwerjddgZ dde\deNdVe1ddfdZxerjddgZ dde\deNdVe1ddfdZyerjdgdZ dde\dLeJdMeIdeeLe9ddfdeNdVe1ddfdZze6d¬Z{erje{de{jdCdDdEiddgG ddeVdeWdeXdeTdeUdHeGdQeHdJe+dKe-dLeJdMeIdNe]dOe*dPe,dReKdTePdeNdVe1deRdeSddf*dńZ|dddeCe@dzdLe@dzdece@e@fdzfdƄZ}deCeCe@eOzdeCe@de@fdɄZ~edʫZdeededeefdecddfdτZdejjdedeefdeCecdeefd҄Zy)z@Contains commands to interact with jobs on the Hugging Face Hub.N)CallableIterable)fnmatch)Path)EmptyQueue) AnnotatedAnyTypeVar)urlsplit)HfApi JobHardwareJobInfoJobStageVolume constants)TERMINAL_JOB_STAGES)CLIError)logging) _format_size) _split_mount)format_durationparse_duration) EnvFileOptEnvOptSecretsFileOpt SecretsOpt SoftChoice SshDryRunOptSshIdentityFileOptTokenOptexec_ssh get_hf_api parse_env_map parse_volumes typer_factory)ArgumentOption)_dataclass_to_dictoutjob_id namespacereturnc|s td|jddkDrtd|dd|vr||fS|jdd\}}|r|std|d|||k7rtd|d|d||fS) a&Extract namespace from job_id if provided in 'namespace/job_id' format. Allows users to pass job IDs copied from the Hub UI (e.g. 'username/job_id') instead of only bare job IDs. If the namespace is also provided explicitly via --namespace and conflicts, a CLIError is raised. zJob ID cannot be empty./rz`. Requires an SSH public key registered on https://huggingface.co/settings/keys.z--withz%Run with the given packages installedz-pz--pythonz5The Python interpreter to use for the run environmentz!Suspend (pause) the scheduled Jobz8Allow multiple instances of this Job to run concurrentlyzWOne of annually, yearly, monthly, weekly, daily, hourly, or a CRON schedule expression.z$UV script to run (local file or URL)zArguments for the scriptzThe command to run.zJob ID (or 'namespace/job_id')zJob IDs (or 'namespace/job_id')z2Scheduled Job ID (or 'namespace/scheduled_job_id')z-vz--volumeaMount one or more volumes. Format: hf://[TYPE/]SOURCE:/MOUNT_PATH[:ro|:rw] or LOCAL_DIR:/MOUNT_PATH[:ro|:rw]. TYPE is one of: models, datasets, spaces, buckets. TYPE defaults to models if omitted. models, datasets and spaces are always mounted read-only. buckets are read+write by default. A local directory source is first synced to a bucket and mounted read-only by default. E.g. -v hf://datasets/org/ds:/data or -v hf://buckets/org/b:/mnt:ro or -v ./inputs:/inputszRun and manage Jobs on the Hub.jobcX|j|j|jjdD]}t j ||j |j|jj}|jjtjk7rc|jjrd|jjnd}td|jd|jjd|t j d|jd y ) zcStream Job logs until the Job ends, then fail the command if the Job did not complete successfully.T)r,r-followr,r-z: zJob z finished with stage 'r1z completedN) fetch_job_logsidownernamer+text wait_for_jobstatusstager COMPLETEDmessager)r:rPlogfinalr^s r6_stream_logs_and_check_statusra6s!!399>>RV!W     CFFciinn  EE ||X///161E1EBu||++,-2ehhZ'=ell>P>P=QQRSZR[\]]HHtEHH:Z ()r8runignore_unknown_optionsT)zFhf jobs run --name hello-world python:3.12 python -c 'print("Hello!")'z1hf jobs run --detach python:3.12 python script.pyz3hf jobs run -e FOO=foo python:3.12 python script.pyz;hf jobs run --secrets HF_TOKEN python:3.12 python script.pyz^hf jobs run -v hf://org/my-model:/data -v hf://buckets/org/b:/mnt python:3.12 python script.py)context_settingsexamplesimagecommandenvsecretsrXlabelrLenv_file secrets_fileflavortimeoutdetachexposesshtokenc t||}t||}t|}|j||||t||t |||| | | | | }t j d|j|j|:t jd|jjd |jd t|jjtrDd j!d |jjD}t jd |t|jj"t$r:t jd|jjd |jd| r[|jjd |j}t jd|d|dt jd|dyt'||y)z Run a Job.rrrXr:r-) rfrgrhrilabelsr9rmrnrprqr- Job startedrVurlN#Name this Job with `hf jobs labels r0 --name NAME`. c3&K|] }d| ywz N.0rzs r6 zjobs_run..uF2cUFSExposed ports are reachable at (requires an HF token with read access to the job): Use `hf jobs ssh &` to open an SSH session into the job.Use `hf jobs logs -f &` to stream logs, or `hf jobs inspect ` to check status.Use `hf jobs wait ` to block until it finishes.)r%r$run_job_parse_labels_maprMr+rIrVrzrErWrX isinstancer[ expose_urlslistjoinssh_urlstrra)rfrgrhrirXrjrLrkrlrmrnrorprqr-rrenv_map secrets_mapr:rPurlsjob_refs r6jobs_runrBs<C*G6K 5 !C ++  T2+FyQ   CJJ}SWW5 | 6syy~~6Fax~^_#**(($/yyFszz/E/EFF ghlgmno#**$$c* $SYY^^$4AcffX=cde YY^^$AcffX. ( 1WX_W``rst %gY.KLM!#s+r8logs)zhf jobs logs zhf jobs logs -f zhf jobs logs --tail 20 z#hf jobs logs -f --tail 100 )rerRz-fz--followzqFollow log output (stream until the job completes). Without this flag, only currently available logs are printed.tailz-nz--tailzvNumber of lines to show from the end of the logs. When combined with --follow, starts streaming from the last N lines.ct||\}}t|}|j||||}|D]}tj||r%|r|d|n|}tj d|dyy)aFetch the logs of a Job. By default, prints currently available logs and exits (non-blocking). Use --follow/-f to stream logs in real-time until the job completes. Use --tail/-n to limit the number of lines returned (server-side when supported). Note: following exits when the log stream ends, regardless of whether the Job succeeded or failed. Run `hf jobs inspect ` to check the final status. rt)r,r-rRrr0z#Stream ended. Run `hf jobs inspect z6` to check the final status (e.g. COMPLETED or ERROR).N)r7r$rUr+rYrE) r,rRrr-rrr:rr_rs r6 jobs_logsrsN5VYGFI 5 !C   VyVZ  [D   -6YKq)F 6wi?uvwr8job_propertiesfiltersc|D]Z\}}}|j|}||dk(r yt|j|j}|dk(r|r |dk(sW|sZyy)z5Check if scheduled job matches all specified filters.!=F=T)getrlower)rrkeyop_strpatternvaluematchs r6_matches_filtersrsk 'VW""3' =~ w}}7 cM%Vt^ r8ncFd}d}t|D]}t||y)Nzz)end)rangeprint)rLINE_UP LINE_CLEARis r6 _clear_liners)GJ 1X' g:&'r8metrics_stream table_headersc #K|D]J}||ddt|ddz dtd|dz|dz d dt|dd t|dd t|d d t|ddg}|drt|dtr|gdgt |zgt |ddz zz}t |t |dD]R\}}|d|}||ddtd|dz|dz d dt|dd t|dd gz }Tn!|dgt |t |z zz }|g}d||fMd|gfyw)N cpu_usage_pct%cpu_millicoresg@@rdmemory_used_bytesmemory_total_byteszB / Brx_bpszbps / tx_bpsbpsgpusrT utilizationN/AFT)roundrrdictlenzipsorted)r,rrmetricsrowrowsgpu_idgpus r6_get_jobs_stats_rowsrs"" '( * '*+f4a 8S7#677'BV:WWYZ[\\] ^G$789:$|GThLi?j>kkl mGH-./vl78CT6U5VVY Z   6?z'&/4@5RD3s8O,GFO0Dq0HIID"4)@A  Vfof-=)*!,S3':#;;cBV>WWYZ[\\]^#C(;$<=>djob_idsc |+g}|D]"}t||\}}|j|$|}t|}||jd}|O|j |Dcgc]4}|j r|j j nddvr |j6}}t|dk(rtjdygd }tjjt|5}i} |D]+}|g} | |d dD cgc]} d | vsd | vrd ndc} z } | g| |<-tj} | D cgc]}| |D]} | } }} tt!| ||Dcgc]}||j#|||d}}t%|t&|D]\}}}|r| j)|dn|| |<tj}|| z t*k\sBt-dt| z| D cgc]}| |D]} | } }} tt!| ||}  dddycc}wcc} wcc} }wcc}wcc} }w#1swYyxYw)z7Fetch the resource usage statistics and metrics of JobsNrtrXr-UNKNOWN)RUNNINGUPDATINGrzNo running jobs found) zJOB IDzCPU %zNUM CPUzMEM %z MEM USAGEzNET I/Oz GPU UTIL %z GPU MEM %z GPU MEM USAGErr0USAGEz-- / --z--)headersrS)r,rr) kwargs_listr)r7rHr$whoami list_jobsr[r\rVrr+rYmultiprocessingpool ThreadPooltimer _tabulatefetch_job_metricsiflatmap_unorderedrpopSTATS_UPDATE_MIN_INTERVALr)rr-rr parsed_idsr,r:rPrrrows_per_job_idrheaderlast_update_time total_rowsrdonernows r6 jobs_statsrs  &F  ,F$*8C ^klmln^opTZ#-7f3DI4Op pC'*eOF # , 99;(7[f?SYCZ[Cc[c[ [ i M:;"   !"%"7"7vQZ"7"[!.   #5T;O]h"i ' D&$##FD1*.'))+C%%)BBAJ/00?cf?[aKbcCcccc ci MBC#&  ')''' .q\  d9''sO%9H60IH; &I<II,I AI I<I I;IIzlist | ls | ps)z hf jobs lsz hf jobs ls -az&hf jobs ls --status running,schedulingz+hf jobs ls --label env=prod --label team=mlz%hf jobs ls --all --label hf-sandbox=1allz-az--allzWShow all Jobs (default shows running and scheduling). Cannot be combined with --status.r[z--statuszfOnly show Jobs with the given status. Comma-separated or repeated, e.g. `--status running,scheduling`.)rOrNz{Only show Jobs with the given `key=value` label. Repeat to require several labels, e.g. `--label env=prod --label team=ml`.limitz--limitzCMaximum number of Jobs to display. Set to 0 to show all (no limit).filterz--filterz2(Deprecated) Use `--status` and `--label` instead.ct|}|rtjd|d|r |r tdg}|xsgD])} |j d| j dD+|r|} n9|rd} n4t jjt jjg} i} |xsgD].} d| vrtd | d | j d\} } | | | <0|j|| | xsd }d }|d kDr8ttj||dz}t||kDrd}|d|}n t|}g}|D]}t|}|j!dxsi}|j!dxsg}|j!dd|d<|j!dxsd|d<|rdj#|nd|d<|j!dr|dddj%ddnd|d<|j!dxsij!dd|d<t'|j!d |d!<|j)|tj*|gd"d#|rtj,d$|d%|sg|s| rJd&j#gd'|Dd(| j/D}tj0d)|y|stj,d*yyy)+zList Jobs. Use `--status` to filter by status (see [`JobStage`] for possible values) and `--label` to filter by `key=value` labels. A Job must match every filter to be listed. rtzIgnoring filter 'zc'. `-f`/`--filter` is deprecated and will be removed in a future release. Use `--status`/`--label`.z0`-a`/`--all` cannot be combined with `--status`.c3^K|]%}|js|j'ywN)strip)rparts r6rzjobs_ps..qsVTDJJLVs--,NrzInvalid label filter 'z"': must be in the form 'key=value')r-r[rwFrrT durationsrgrVrTr, docker_imager image/spacer> created_atTcreatedr[r\r running_secsruntime)r,rrgrr[rrid_keyzOutput truncated to z4 Jobs. Use `--limit 0` to show all (or `--limit N`)., c3&K|] }d| yw)zstatus=Nr)rss r6rzjobs_ps..s7QWQC=7rc32K|]\}}d|d|yw)zlabel=rNr)rkvs r6rzjobs_ps..s$:gPTPQSTVA3as;K:gzNo jobs matched filters: zHNo running jobs. Use `-a`/`--all` to include finished (and failed) jobs.)r$r+warningrrAr3rrr SCHEDULINGrr itertoolsislicerr*rrreplacerrHtablerEitemsrY)rr[rjrr-rrrr: raw_statusesrserver_statusesrwitemr jobs_iter truncatedjobs job_itemsrPjob_itemrcmd filters_msgs r6jobs_psr)st 5 !C  x(p p  vIJJ!L2WVU[[5EVVW& #++1183F3F3L3LM F  d?3D69[\] ]ZZ_ Us   /RXR`\` aII qyI$$Y :; t9u I"G%/2chhsmOW||\hOih|4Sb9AA#sKot&ll84:??S-immN.KL" #IIT  'w.bcd  6))h7,7h:gX^XdXdXf:ghK HH0 > ? HH_ ` r8hardwarezhf jobs hardwarec ^t}|j}g}|D]}d}|jrG|jjd|jjd|jj d}|j rd|j dnd}|j rd|j dzd nd}|j|j|j|j|j|j|||d tj|tjd y ) z(List available hardware options for JobsrTzx  ()$z.4ffree<z.2f)rXz pretty namecpuramstorage acceleratorzcost/minz cost/hourzLUse `hf jobs run --flavor ...` to request a specific hardware flavor.N)r$list_jobs_hardwarer%quantitymodelvram unit_cost_usdrHrX pretty_namer"r#ephemeral_storager+rrE)r: hardware_listrhwaccelerator_infocost_min cost_hours r6 jobs_hardwarer2s ,C**,M E  >>"$.."9"9!:"R^^=Q=Qz*Job IDs to inspect (or 'namespace/job_id')cg}|D]"}t||\}}|j|$|}t|}|Dcgc]}|j||}}t j |Dcgc] }t |c}ycc}wcc}w)z0Display detailed information on one or more JobsrtrSN)r7rHr$ inspect_jobr+rr*)rr-rrrr,r:rrPs r6 jobs_inspectr6sJ"8K &!"G 5 !CNU VFCOO6YO ? VD VII$73!#&78 W7 B+B cancelzhf jobs cancel ct||\}}t|}|j||tjd|y)z Cancel a JobrtrSz Job cancelledrVN)r7r$ cancel_jobr+rI)r,r-rrr:s r6 jobs_cancelr<s=5VYGFI 5 !CNN&IN6JJ6*r8wait)zhf jobs wait z"hf jobs wait z"hf jobs ls -q | xargs hf jobs waitz,Job IDs to wait for (or 'namespace/job_id').zTMax time to wait: int with s (seconds, default), m (minutes), h (hours) or d (days).c *g}t}|D]3}t||\}}|j||j|5t |dkDr3t ddj dt|tDz|j}| t|nd} t|} tjdt |d } | j|| | } | j!t | dtj"| D cgc]C} | j$t| jj&| jj(dEc} | D cgc],} | jj&t*j,k7s+| .}} |r.sFBBFr)rNrtz Waiting for z Job(s) to finish...)rnr-z Timed out.zTimed out after z waiting for Job(s) to finish.z Job(s) finished.)rVr\r^z of z' Job(s) did not complete successfully: c3hK|]*}|jd|jjd,yw)rrN)rVr[r\)rrPs r6rzjobs_wait..(s,KS366("SZZ%5%5$6a8Ks02)setr7rHaddrrrrrrrr$r+r[rZ TimeoutErrorrrrVr\r^rr])rrnr-rrr namespacesr, parsed_idparsed_namespace timeout_secsr:r[rrPfaileds r6 jobs_waitrJs>JJ)&B69&U# #)$'() : >iiF&*EFF G   I.5.A>'*tL 5 !C ZZ,s:&77KL MF] LIV KK3t9+./0IIhlmadcffs3::+;+;'<I[I[\mn! LcSZZ%5%59K9K%Kc LF L 6{m4D {*Q RiiKFKK L   ] L!)'2PQRX\\] n LsGAH +,HH+Hrw)z*hf jobs labels --name training-v2z8hf jobs labels --label env=prod --label team=mlzhf jobs labels --clearclearz--clearzRemove all labels from the job.cf|s| |s td|s| |r tdt||\}}t|}|,|s*|j||jxsi}i|d|i}nt ||xsi}|j |||} tjd | j y) zfUpdate labels on a Job. Passing --label replaces all existing labels; passing --name alone keeps them.NePlease set a name with --name or at least one label with --label. To remove all labels, pass --clear.lCannot set a name or labels and clear them at the same time. Please use --name/--label or --clear, not both.rtrSrXru)r,rwr-Labels updatedr:) rr7r$r5rwrupdate_job_labelsr+rIrV) r,rXrjrKr-rrr:current_labelsrwrPs r6 jobs_labelsrR,s" T\% s   !u z  5VYGFI 5 !C )LSSYWY1N1FD1"5t4:  vf  RCJJCFF+r8)zhf jobs ssh zhf jobs ssh --dry-runz)hf jobs ssh -i ~/.ssh/id_ed25519 identity_filedry_runct||\}}t|}|j||}|jj t d|jj tvr0t d|jd|jj d|jj tjk7rtjd|jd |jj d }|j|j|tjg }|jj tjk7rA|jd t d|jd |jj d|jdt|jj}t|j d|j"|j$||y)a SSH into a running Job. If the Job is not yet running, waits until it reaches the RUNNING state before connecting. Requires the Job to be started with SSH enabled (`hf jobs run --ssh ...`) and your SSH public key to be registered at https://huggingface.co/settings/keys. rtrSNz[SSH is not enabled on this job. Start a job with SSH support using `hf jobs run --ssh ...`.zCannot SSH into job 'z%': job has already finished (stage: 'z').zWaiting for job 'z' to be running (stage: 'z')...)r,r-stagesz Job finished.z1': job finished before reaching RUNNING (stage: 'zJob is running.@)portrSrT)r7r$r5r[rrr\rrVrrr+rZrr r#usernamehostnamerX) r,rSrTr-rrr:rPr[rs r6jobs_sshr[Qs*5VYGFI 5 !C //9/ =C zz!tuu zz...svvh6[\_\f\f\l\l[mmpqrr zz8+++/x7PQTQ[Q[QaQaPbbghicff 8K[K[J\] ::  x// / KK ('x/`adakakaqaq`rruv   %&szz))*G    Ag../0 \\# r8zFRun UV scripts (Python with inline dependencies) on HF infrastructure.uvru)z,hf jobs uv run --name my-script my_script.pyz$hf jobs uv run --detach my_script.pyz1hf jobs uv run ml_training.py --flavor a10g-smallz+hf jobs uv run --with transformers train.pyzNhf jobs uv run -v hf://org/my-model:/data -v hf://buckets/org/b:/mnt script.pyscript script_argswith_pythonct|| }t|| }t|}|j||xsg|||||t||t ||||| | ||}t j d|j|j|:t jd|jjd |jd t|jjtrDd j!d |jjD}t jd |t|jj"t$r:t jd|jjd |jd| r[|jjd |j}t jd|d|dt jd|dyt'||y)8Run a UV script (local file or URL) on HF infrastructurertrurv)r]r^ dependenciesr`rfrhrirwr9rmrnrprqr-rxryNr{r0r|r}c3&K|] }d| ywrrrs r6rzjobs_uv_run..rrrrrrrrrr)r%r$ run_uv_jobrrMr+rIrVrzrErWrXrr[rrrrrra)r]r^rfrmrhrirXrjrLrkrlrnrorprqr-rrr_r`rrr:rPrrs r6 jobs_uv_runrfsBC*G6K 5 !C ..%2  T2+FyQ   C JJ}SWW5 | 6syy~~6Fax~^_#**(($/yyFszz/E/EFF ghlgmno#**$$c* $SYY^^$4AcffX=cde YY^^$AcffX. ( 1WX_W``rst %gY.KLM!#s+r8z,Create and manage scheduled Jobs on the Hub. scheduledzRhf jobs scheduled run "0 0 * * *" --name daily-script python:3.12 python script.pyschedulesuspend concurrencyct|| }t|| }t|}|j|||||||t||t | ||| | || }t j d|j|:t jd|jjd |jd t jd |jjd |jd y) zSchedule a Job.rtrurv) rfrgrhrirjrhrirwr9rmrnrpr-Scheduled Job createdr:N7Name this scheduled Job with `hf jobs scheduled labels r0r|Use `hf jobs scheduled inspect ` to view its details.) r%r$create_scheduled_jobrrMr+rIrVrErWrX)rhrfrgrirjrhrirXrjrLrkrlrmrnrpr-rrrrr: scheduled_jobs r6 scheduled_runrrs2C*G6K 5 !C,,  T2+FyQ-MJJ&=+;+;< | EmFYFYF^F^E__`anaqaq`rsA B HH.}/B/B/G/G.H-JZJZI[[q rsr8zhf jobs scheduled lsz1Show all scheduled Jobs (default hides suspended)z>Filter output based on conditions provided (format: key=value)ct|}|j|}g}|xsgD]}d|vra|jdd\}} |jdrd} |dd}nd} |j |j | | j fht jd |d g} |D]} | jxsd } |s| r| jjxsd }| jjxsg}|rd j|nd }| j|t| |d}t||s| j | g}| D]}t!|}|j#dxsi}|j#dxsi}|j#d}|j#dxsg}|j#dxsd |d<|rd j|nd |d<|r)|j#dr|dddj%dd nd |d<|j#dr|dddj%dd nd |d<|j#dxsd |d<|j |t j&|gdd|s2|r0djd |D}t j(d!||r;|d"d}t j*d#|d$t j*d%|d&yy)'zList scheduled Jobsrtrrr!rNz Ignoring invalid filter format 'z'. Use key=value format.Frr>)rVrfrirgjob_specr[last_jobrgrratrrlast_runnext_job_run_atnext_runri)rVrhrrgryr{rirVrrc34K|]\}}}|||ywrr)rrors r6rzscheduled_ps..Ks"DAq1#aS Dsz#No scheduled jobs matched filters: rrnz(` to view details about a scheduled job.zUse `hf jobs scheduled trigger z)` to trigger a scheduled job immediately.)r$list_scheduled_jobsr3endswithrHrr+r rirvrrgrrVrrr*rr rrYrE)rr-rrrr:scheduled_jobsrfrrop filtered_jobsrqriimage_or_spacer command_strpropsrsjrrv status_dictrwr first_item_ids r6 scheduled_psrs, 5 !C,,y,AN*,G \r X !8aJC||C #2h NNCIIKU[[]; < KK:1#=UV W XM' , ''05w &//<<E$$,,2'*chhsm $''.SQX\fqrw/ ]+ ,#%E !"%88J'-2hhx(.B ??:.ll9%+&ll>:Ce]+.#((3-EYDLQYQ]Q]^bQc8D>#2.66sC@inZEP__UfEgK) *3B / 7 7S Amr Z((9-6Y T II _ WiiDGDD  6{mDE a  2=/Aijk 2=/Ajkl r8zhf jobs scheduled inspect scheduled_job_idsz>Scheduled Job IDs to inspect (or 'namespace/scheduled_job_id')cg}|D]"}t||\}}|j|$|}t|}|Dcgc]}|j||}}t j |Dcgc] }t |c}ycc}wcc}w)z:Display detailed information on one or more scheduled Jobsrtscheduled_job_idr-N)r7rHr$inspect_scheduled_jobr+rr*) rr-rrrr,r:rrrqs r6scheduled_inspectrSsJ#"8K &!"# 5 !C!2  !!3Cy!YNIInU]!-0UV Vr7deletezhf jobs scheduled delete rct||\}}t|}|j||tjd|y)zDelete a scheduled Job.rtrzScheduled Job deletedr:N)r7r$delete_scheduled_jobr+rIrr-rrr:s r6scheduled_deleterlF#??OQZ"[i 5 !C.>)TJJ&+;ct||\}}t|}|j||tjd|tj d|dy)z Suspend (pause) a scheduled Job.rtrzScheduled Job suspendedr:zUse `hf jobs scheduled resume z` to resume it.N)r7r$suspend_scheduled_jobr+rIrErs r6scheduled_suspendrys^#??OQZ"[i 5 !C/?9UJJ(-=>HH-.>-? OPr8resumezhf jobs scheduled resume ct||\}}t|}|j||tjd|y)z!Resume (unpause) a scheduled Job.rtrzScheduled Job resumedr:N)r7r$resume_scheduled_jobr+rIrs r6scheduled_resumerrr8triggerzhf jobs scheduled trigger c,t||\}}t|}|j||}tjd|j |j tjd|jjd|j dy) zJTrigger a scheduled Job to run immediately (does not change the schedule).rtrzScheduled Job triggeredryrr0z` to stream logs.N) r7r$trigger_scheduled_jobr+rIrVrzrErWrX)rr-rrr:rPs r6scheduled_triggerrs|#??OQZ"[i 5 !C # #5EQZ # [CJJ(SVVAHH$SYY^^$4AcffX=N OPr8)z1hf jobs scheduled labels --name daily-scriptz>hf jobs scheduled labels --label env=prod --label team=mlz%hf jobs scheduled labels --clearz)Remove all labels from the scheduled job.cz|s| |s td|s| |r tdt||\}}t|}|6|s4|j||jj xsi}i|d|i}nt ||xsi}|j|||} tjd | j y) zpUpdate labels on a scheduled Job. Passing --label replaces all existing labels; passing --name alone keeps them.NrMrNrtrrXru)rrwr-rOr:) rr7r$rrvrwrupdate_scheduled_job_labelsr+rIrV) rrXrjrKr-rrr:rQrwrqs r6scheduled_labelsrs" T\% s   !u z  #??OQZ"[i 5 !C   % %7GS\ % ] f f m m sqs 2N1FD1"5t4:33)&I4MJJM$4$45r8z)Schedule UV scripts on HF infrastructure.zBhf jobs scheduled uv run "0 0 * * *" --name daily-script script.pyzC*G6K 5 !C % %%2  T2+FyQ! & C$JJ&3662 | J399>>JZZ[\_\b\b[ccqrsHH.syy~~.>axG] ^_r8c|s|yi}|xsgD]$}d|vr|jddn|df\}}|||<&|d|vr td||d<|S)aParse label key-value pairs from CLI arguments. Args: labels: List of label strings in KEY=VALUE format. If KEY only, then VALUE is set to empty string. Returns: Dictionary mapping label keys to values, or None if no labels or name provided. NrrrTrXz4--name and --label name=... cannot both be provided.)r3r)rwrX labels_map label_varrrs r6rr s} dl!#J\r 03y0@Y__S!,yRTo U 3   Z QR R! 6 r8rrc tg||Dcgc]}td|D}}ttjjt |dz}t |t |z|kDr|jt|}||xxdzcc<t |t |z|kr#|t |z t |z ||z||<t |t |z|kDrdt |zj|}g}|j|j||j|j|Dcgc]}d|z c}|D]r} t| |D cgc];\} } t t| | kDrt| d| dz dzn t| =} } } |j|j| td j|Scc}wcc}wcc} } w) z Inspired by: - stackoverflow.com/a/8356620/593036 - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data c3DK|]}tt|ywr)rr)rxs r6rz_tabulate..*s/ac#a&k/s  rz{{:{}}} -Nz...r}) rmaxshutilget_terminal_sizecolumnsrsumindexformatrHrr) rrcol col_widthsterminal_widthcol_to_minimize row_formatlineswrr col_widthrow_format_argss r6rr#s 9$**3z?;?#)# w<#j/ )^ ;*83z?*JSQX\*Y\fgv\w*wJ ' g,Z (> 9 4*s7|+33Z@J E LL"""G,- LL"""j$AS1W$ABC:!$C 4 9033q6{Y/FCF?Y] #e +CPQF R    &Z&&89 : 99U #P%B sG* G/ 6AG4rqueuefunc.kwargsc@|di|D]}|j|y)Nr)put)rrrrIs r6_write_generator_to_queuerAs#.. &r8rrc #6Kt}|Dcgc]}|jt|||f}} |jdcc}w#t$r't d|Dr|j rYnY5wxYwn#t$rYnwxYw |Dcgc]}|jdncc}wc}y#tj$rYywxYw# |Dcgc]}|jdncc}wc}w#tj$rYwwxYwxYww)a; Takes a function that returns an iterable of items, and run it in parallel using threads to return the flattened iterable of items as they arrive. This is inspired by those three `map()` variants, and is the mix of all three: * `imap()`: like `map()` but returns an iterable instead of a list of results * `imap_unordered()`: like `imap()` but the output is sorted by time of arrival * `flatmap()`: like `map()` but given a function which returns a list, `flatmap()` returns the flattened list that is the concatenation of all the output lists g?)rnc3<K|]}|jywr)ready)r async_results r6rz%iflatmap_unordered..\sN |))+NsN) r apply_asyncrrrremptyKeyboardInterruptrrD)rrrrr async_resultsrs r6rrFsgEfqr\bT%%&?%vAVWrMr iii--s  N NNSXS^S^S`      @M N \  d  + N N++    @M N \  d  + N N++   sD A DBA B D+B;B?BBC B CBCB;B21 B;:D;CDCDDC=C43 C=<D=DDDDD)NNNNNNNNNFNFNN)FNNN)NNN)FNNrNNN)r.N)NN)NNFNN)NFNN)NNNNNNNNNNNFNFNNNN)NNNNNNNNNNNNNN)NNNNNNNNNNNNNNNNNN)__doc__r rmultiprocessing.poolrrcollections.abcrrrpathlibrrrrtypingr r r urllib.parser huggingface_hubr rrrrrhuggingface_hub._jobs_apirhuggingface_hub.errorsrhuggingface_hub.utilsr$huggingface_hub.utils._cache_managerrhuggingface_hub.utils._hf_urisrhuggingface_hub.utils._parsingrr _cli_utilsrrrrrr r!r"r#r$r%r&r' _frameworkr(r)_outputr*r+ get_logger__name__loggerrtupler7rrMrImageArgImageOpt FlavorOpt LabelsOptNameOpt TimeoutOptbool DetachOpt NamespaceOptint ExposeOpt SshEnabledOptWithOpt PythonOpt SuspendOptConcurrencyOpt ScheduleArg ScriptArg ScriptArgsArg CommandArgJobIdArg JobIdsArgScheduledJobIdArg JobVolumesOptjobs_clirargrrrrrrrrr2r6r<rJrRr[uv_app add_grouprf scheduled_apprrrrrrrrrscheduled_uv_apprrrrrrrrrr8r6rs<G .**!TT9+)=7J),   H %..t.cSVY]S]oH^.8) #Y )',)9X ]a '3'4' !)$sCx.!9JNs) eD#tDsO44568 '%=$>?"A' A'A' A' A'@A'H  $    " Sxa    j   xa  S D !(+y   xa"  S D  O   #xa2   V   3xa@AxaB CxaD  S D  E   ExaT Uxa xav *(:';<]=]6 )'A&BC#9  S =  99 9 9D9* (&?%@A# +  + +  + +B +  ( "4  S ?  4  d g  4 4  4  4 4 n  X]" , , , , T6)2STT U ,  ,  , ,,:  )-!" # #%## #  #  ##L d e6% .5 "& #'"'7, 7,7, 7,  7,  7,  7, 7, 7, 7,7,!7,7, 7, 7, 7, !7," #7,$ %7,& '7,( )7, 7,t#QR ={3 .5b c"& #'"#,t,t ,t,t ,t  ,t  ,t,t ,t ,t ,t,t!,t ,t,t ,t !,t" #,t$ %,t  ,t^'3I2JK " %Om    D   OmOm Om  S D  Q   Om& 'OmLOmdy,L+MN#W  S Q  WW W WOW0x+J*KL# =' = =  = =M =y,L+MN# Q' Q Q  Q QO Qx+J*KL# =' = =  = =M =y,L+MN# Q' Q Q  Q QO Q bg" 6'6 6 6 T6)2]^^ _ 6  6  6 66B!&QR (t4 .5LF"&"& #'")0`0` 0`0` 0`  0`  0` 0` 0`0` 0` 0` 0`0`!0`0` !0`"#0`$ %0`& '0`( )0`* +0`0`lGKd3i$.ttTWY\T\~`dOd,DcCi)DI#6 CLU1XXc8A;>N5OY]bf     ) ) 3 # $d  a[ r8