From d46c5fa6d2d8e7067c7178781f51597cb6f70aab Mon Sep 17 00:00:00 2001 From: Celina Hanouti Date: Mon, 15 Sep 2025 18:20:26 +0200 Subject: [PATCH 01/29] migrate CLI to typer --- setup.py | 1 + src/huggingface_hub/cli/__init__.py | 14 - src/huggingface_hub/cli/_cli_utils.py | 21 +- src/huggingface_hub/cli/auth.py | 258 +-- src/huggingface_hub/cli/cache.py | 207 +- src/huggingface_hub/cli/download.py | 289 +-- src/huggingface_hub/cli/hf.py | 100 +- src/huggingface_hub/cli/jobs.py | 1817 ++++++++--------- src/huggingface_hub/cli/lfs.py | 246 +-- src/huggingface_hub/cli/repo.py | 403 ++-- src/huggingface_hub/cli/repo_files.py | 137 +- src/huggingface_hub/cli/system.py | 31 +- src/huggingface_hub/cli/upload.py | 523 ++--- .../cli/upload_large_folder.py | 207 +- 14 files changed, 1988 insertions(+), 2266 deletions(-) diff --git a/setup.py b/setup.py index 97fc2efbe8..1fc7e8ad6e 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ def get_version() -> str: extras["cli"] = [ "InquirerPy==0.3.4", # Note: installs `prompt-toolkit` in the background + "typer", ] extras["inference"] = [ diff --git a/src/huggingface_hub/cli/__init__.py b/src/huggingface_hub/cli/__init__.py index 7a1a8d793b..8568c82be1 100644 --- a/src/huggingface_hub/cli/__init__.py +++ b/src/huggingface_hub/cli/__init__.py @@ -11,17 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from abc import ABC, abstractmethod -from argparse import _SubParsersAction - - -class BaseHuggingfaceCLICommand(ABC): - @staticmethod - @abstractmethod - def register_subcommand(parser: _SubParsersAction): - raise NotImplementedError() - - @abstractmethod - def run(self): - raise NotImplementedError() diff --git a/src/huggingface_hub/cli/_cli_utils.py b/src/huggingface_hub/cli/_cli_utils.py index d0f0b98d8a..83c65cac69 100644 --- a/src/huggingface_hub/cli/_cli_utils.py +++ b/src/huggingface_hub/cli/_cli_utils.py @@ -11,10 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Contains a utility for good-looking prints.""" +"""Contains CLI utilities (styling, helpers).""" import os -from typing import Union +from typing import Optional, Union + +import typer class ANSI: @@ -67,3 +69,18 @@ def tabulate(rows: list[list[Union[str, int]]], headers: list[str]) -> str: for row in rows: lines.append(row_format.format(*row)) return "\n".join(lines) + + +def validate_repo_type(value: Optional[str], param_name: str = "repo_type") -> Optional[str]: + """Validate repo type is one of model|dataset|space when provided. + + Returns the value if valid or None. Raises a Typer BadParameter otherwise. + """ + if value is None: + return None + if value in ("model", "dataset", "space"): + return value + raise typer.BadParameter( + "Invalid value for '--repo-type': must be one of 'model', 'dataset', 'space'.", + param_name=param_name, + ) diff --git a/src/huggingface_hub/cli/auth.py b/src/huggingface_hub/cli/auth.py index 8740a86423..e25c464352 100644 --- a/src/huggingface_hub/cli/auth.py +++ b/src/huggingface_hub/cli/auth.py @@ -30,10 +30,10 @@ hf auth whoami """ -from argparse import _SubParsersAction from typing import Optional -from huggingface_hub.commands import BaseHuggingfaceCLICommand +import typer + from huggingface_hub.constants import ENDPOINT from huggingface_hub.errors import HfHubHTTPError from huggingface_hub.hf_api import HfApi @@ -54,125 +54,53 @@ _inquirer_py_available = False -class AuthCommands(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - # Create the main 'auth' command - auth_parser = parser.add_parser("auth", help="Manage authentication (login, logout, etc.).") - auth_subparsers = auth_parser.add_subparsers(help="Authentication subcommands") - - # Show help if no subcommand is provided - auth_parser.set_defaults(func=lambda args: auth_parser.print_help()) - - # Add 'login' as a subcommand of 'auth' - login_parser = auth_subparsers.add_parser( - "login", help="Log in using a token from huggingface.co/settings/tokens" - ) - login_parser.add_argument( - "--token", - type=str, - help="Token generated from https://huggingface.co/settings/tokens", - ) - login_parser.add_argument( - "--add-to-git-credential", - action="store_true", - help="Optional: Save token to git credential helper.", - ) - login_parser.set_defaults(func=lambda args: AuthLogin(args)) - - # Add 'logout' as a subcommand of 'auth' - logout_parser = auth_subparsers.add_parser("logout", help="Log out") - logout_parser.add_argument( - "--token-name", - type=str, - help="Optional: Name of the access token to log out from.", - ) - logout_parser.set_defaults(func=lambda args: AuthLogout(args)) - - # Add 'whoami' as a subcommand of 'auth' - whoami_parser = auth_subparsers.add_parser( - "whoami", help="Find out which huggingface.co account you are logged in as." - ) - whoami_parser.set_defaults(func=lambda args: AuthWhoami(args)) - - # Existing subcommands - auth_switch_parser = auth_subparsers.add_parser("switch", help="Switch between access tokens") - auth_switch_parser.add_argument( - "--token-name", - type=str, - help="Optional: Name of the access token to switch to.", - ) - auth_switch_parser.add_argument( - "--add-to-git-credential", - action="store_true", - help="Optional: Save token to git credential helper.", - ) - auth_switch_parser.set_defaults(func=lambda args: AuthSwitch(args)) - - auth_list_parser = auth_subparsers.add_parser("list", help="List all stored access tokens") - auth_list_parser.set_defaults(func=lambda args: AuthList(args)) - - -class BaseAuthCommand: - def __init__(self, args): - self.args = args - self._api = HfApi() - - -class AuthLogin(BaseAuthCommand): - def run(self): - logging.set_verbosity_info() - login( - token=self.args.token, - add_to_git_credential=self.args.add_to_git_credential, - ) - - -class AuthLogout(BaseAuthCommand): - def run(self): - logging.set_verbosity_info() - logout(token_name=self.args.token_name) - - -class AuthSwitch(BaseAuthCommand): - def run(self): - logging.set_verbosity_info() - token_name = self.args.token_name - if token_name is None: - token_name = self._select_token_name() - - if token_name is None: - print("No token name provided. Aborting.") - exit() - auth_switch(token_name, add_to_git_credential=self.args.add_to_git_credential) - - def _select_token_name(self) -> Optional[str]: - token_names = list(get_stored_tokens().keys()) - - if not token_names: - logger.error("No stored tokens found. Please login first.") - return None +auth_app = typer.Typer(help="Manage authentication (login, logout, etc.)") + + +def _api() -> HfApi: + return HfApi() + + +@auth_app.command("login", help="Login using a token from huggingface.co/settings/tokens") +def auth_login( + token: Optional[str] = typer.Option( + None, + "--token", + help="Token from https://huggingface.co/settings/tokens", + ), + add_to_git_credential: bool = typer.Option( + False, + "--add-to-git-credential", + help="Save to git credential helper", + ), +) -> None: + logging.set_verbosity_info() + login(token=token, add_to_git_credential=add_to_git_credential) + + +@auth_app.command( + "logout", + help="Logout from a specific token", +) +def auth_logout( + token_name: Optional[str] = typer.Option( + None, + "--token-name", + help="Name of token to logout", + ), +) -> None: + logging.set_verbosity_info() + logout(token_name=token_name) - if _inquirer_py_available: - return self._select_token_name_tui(token_names) - # if inquirer is not available, use a simpler terminal UI - print("Available stored tokens:") - for i, token_name in enumerate(token_names, 1): - print(f"{i}. {token_name}") - while True: - try: - choice = input("Enter the number of the token to switch to (or 'q' to quit): ") - if choice.lower() == "q": - return None - index = int(choice) - 1 - if 0 <= index < len(token_names): - return token_names[index] - else: - print("Invalid selection. Please try again.") - except ValueError: - print("Invalid input. Please enter a number or 'q' to quit.") - - def _select_token_name_tui(self, token_names: list[str]) -> Optional[str]: + +def _select_token_name() -> Optional[str]: + token_names = list(get_stored_tokens().keys()) + + if not token_names: + logger.error("No stored tokens found. Please login first.") + return None + + if _inquirer_py_available: choices = [Choice(token_name, name=token_name) for token_name in token_names] try: return inquirer.select( @@ -183,30 +111,68 @@ def _select_token_name_tui(self, token_names: list[str]) -> Optional[str]: except KeyboardInterrupt: logger.info("Token selection cancelled.") return None - - -class AuthList(BaseAuthCommand): - def run(self): - logging.set_verbosity_info() - auth_list() - - -class AuthWhoami(BaseAuthCommand): - def run(self): - token = get_token() - if token is None: - print("Not logged in") - exit() + # if inquirer is not available, use a simpler terminal UI + print("Available stored tokens:") + for i, token_name in enumerate(token_names, 1): + print(f"{i}. {token_name}") + while True: try: - info = self._api.whoami(token) - print(ANSI.bold("user: "), info["name"]) - orgs = [org["name"] for org in info["orgs"]] - if orgs: - print(ANSI.bold("orgs: "), ",".join(orgs)) - - if ENDPOINT != "https://huggingface.co": - print(f"Authenticated through private endpoint: {ENDPOINT}") - except HfHubHTTPError as e: - print(e) - print(ANSI.red(e.response.text)) - exit(1) + choice = input("Enter the number of the token to switch to (or 'q' to quit): ") + if choice.lower() == "q": + return None + index = int(choice) - 1 + if 0 <= index < len(token_names): + return token_names[index] + else: + print("Invalid selection. Please try again.") + except ValueError: + print("Invalid input. Please enter a number or 'q' to quit.") + + +@auth_app.command("switch", help="Switch between accesstokens") +def auth_switch_cmd( + token_name: Optional[str] = typer.Option( + None, + "--token-name", + help="Name of the token to switch to", + ), + add_to_git_credential: bool = typer.Option( + False, + "--add-to-git-credential", + help="Save to git credential", + ), +) -> None: + logging.set_verbosity_info() + if token_name is None: + token_name = _select_token_name() + if token_name is None: + print("No token name provided. Aborting.") + raise typer.Exit() + auth_switch(token_name, add_to_git_credential=add_to_git_credential) + + +@auth_app.command("list", help="List all stored access tokens") +def auth_list_cmd() -> None: + logging.set_verbosity_info() + auth_list() + + +@auth_app.command("whoami", help="Find out which huggingface.co account you are logged in as.") +def auth_whoami() -> None: + token = get_token() + if token is None: + print("Not logged in") + raise typer.Exit() + try: + info = _api().whoami(token) + print(ANSI.bold("user: "), info["name"]) + orgs = [org["name"] for org in info["orgs"]] + if orgs: + print(ANSI.bold("orgs: "), ",".join(orgs)) + + if ENDPOINT != "https://huggingface.co": + print(f"Authenticated through private endpoint: {ENDPOINT}") + except HfHubHTTPError as e: + print(e) + print(ANSI.red(e.response.text)) + raise typer.Exit(code=1) diff --git a/src/huggingface_hub/cli/cache.py b/src/huggingface_hub/cli/cache.py index 7eb3e82509..f42a52156b 100644 --- a/src/huggingface_hub/cli/cache.py +++ b/src/huggingface_hub/cli/cache.py @@ -16,13 +16,13 @@ import os import time -from argparse import Namespace, _SubParsersAction from functools import wraps from tempfile import mkstemp from typing import Any, Callable, Iterable, Literal, Optional, Union +import typer + from ..utils import CachedRepoInfo, CachedRevisionInfo, CacheNotFound, HFCacheInfo, scan_cache_dir -from . import BaseHuggingfaceCLICommand from ._cli_utils import ANSI, tabulate @@ -40,6 +40,14 @@ _CANCEL_DELETION_STR = "CANCEL_DELETION" +def _validate_sort_option(sort: Optional[str]) -> Optional[str]: + if sort is None: + return None + if sort not in SortingOption_T: + raise typer.BadParameter(f"Invalid sort option: {sort}", param_name="sort") + return sort + + def require_inquirer_py(fn: Callable) -> Callable: @wraps(fn) def _inner(*args, **kwargs): @@ -54,122 +62,93 @@ def _inner(*args, **kwargs): return _inner -class CacheCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - cache_parser = parser.add_parser("cache", help="Manage local cache directory.") - cache_subparsers = cache_parser.add_subparsers(dest="cache_command", help="Cache subcommands") +cache_app = typer.Typer(help="Manage local cache directory.") - # Show help if no subcommand is provided - cache_parser.set_defaults(func=lambda args: cache_parser.print_help()) - # Scan subcommand - scan_parser = cache_subparsers.add_parser("scan", help="Scan cache directory.") - scan_parser.add_argument( - "--dir", - type=str, - default=None, - help="cache directory to scan (optional). Default to the default HuggingFace cache.", - ) - scan_parser.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="show a more verbose output", - ) - scan_parser.set_defaults(func=CacheCommand, cache_command="scan") - # Delete subcommand - delete_parser = cache_subparsers.add_parser("delete", help="Delete revisions from the cache directory.") - delete_parser.add_argument( - "--dir", - type=str, - default=None, - help="cache directory (optional). Default to the default HuggingFace cache.", - ) - delete_parser.add_argument( - "--disable-tui", - action="store_true", - help=( - "Disable Terminal User Interface (TUI) mode. Useful if your platform/terminal doesn't support the multiselect menu." - ), - ) - delete_parser.add_argument( - "--sort", - nargs="?", - choices=["alphabetical", "lastUpdated", "lastUsed", "size"], - help=( - "Sort repositories by the specified criteria. Options: " - "'alphabetical' (A-Z), " - "'lastUpdated' (newest first), " - "'lastUsed' (most recent first), " - "'size' (largest first)." - ), - ) - delete_parser.set_defaults(func=CacheCommand, cache_command="delete") - - def __init__(self, args: Namespace) -> None: - self.args = args - self.verbosity: int = getattr(args, "verbose", 0) - self.cache_dir: Optional[str] = getattr(args, "dir", None) - self.disable_tui: bool = getattr(args, "disable_tui", False) - self.sort_by: Optional[SortingOption_T] = getattr(args, "sort", None) - self.cache_command: Optional[str] = getattr(args, "cache_command", None) - - def run(self): - if self.cache_command == "scan": - self._run_scan() - elif self.cache_command == "delete": - self._run_delete() +@cache_app.command("scan", help="Scan the cache directory") +def cache_scan( + dir: Optional[str] = typer.Option( + None, + "--dir", + help="Cache directory to scan (defaults to Hugging Face cache).", + ), + verbose: int = typer.Option( + 0, + "-v", + "--verbose", + count=True, + help="Increase verbosity (-v, -vv, -vvv).", + ), +) -> None: + _run_scan(cache_dir=dir, verbosity=verbose) + + +def _run_scan(cache_dir: Optional[str], verbosity: int) -> None: + try: + t0 = time.time() + hf_cache_info = scan_cache_dir(cache_dir) + t1 = time.time() + except CacheNotFound as exc: + cache_dir = exc.cache_dir + print(f"Cache directory not found: {cache_dir}") + return + print(get_table(hf_cache_info, verbosity=verbosity)) + print( + f"\nDone in {round(t1 - t0, 1)}s. Scanned {len(hf_cache_info.repos)} repo(s)" + f" for a total of {ANSI.red(hf_cache_info.size_on_disk_str)}." + ) + if len(hf_cache_info.warnings) > 0: + message = f"Got {len(hf_cache_info.warnings)} warning(s) while scanning." + if verbosity >= 3: + print(ANSI.gray(message)) + for warning in hf_cache_info.warnings: + print(ANSI.gray(str(warning))) else: - print("Please specify a cache subcommand (scan or delete). Use -h for help.") - - def _run_scan(self): - try: - t0 = time.time() - hf_cache_info = scan_cache_dir(self.cache_dir) - t1 = time.time() - except CacheNotFound as exc: - cache_dir = exc.cache_dir - print(f"Cache directory not found: {cache_dir}") - return - print(get_table(hf_cache_info, verbosity=self.verbosity)) - print( - f"\nDone in {round(t1 - t0, 1)}s. Scanned {len(hf_cache_info.repos)} repo(s)" - f" for a total of {ANSI.red(hf_cache_info.size_on_disk_str)}." - ) - if len(hf_cache_info.warnings) > 0: - message = f"Got {len(hf_cache_info.warnings)} warning(s) while scanning." - if self.verbosity >= 3: - print(ANSI.gray(message)) - for warning in hf_cache_info.warnings: - print(ANSI.gray(str(warning))) - else: - print(ANSI.gray(message + " Use -vvv to print details.")) - - def _run_delete(self): - hf_cache_info = scan_cache_dir(self.cache_dir) - if self.disable_tui: - selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) + print(ANSI.gray(message + " Use -vvv to print details.")) + + +@cache_app.command("delete", help="Delete revisions from the cache directory") +def cache_delete( + dir: Optional[str] = typer.Option( + None, + "--dir", + help="Cache directory (defaults to Hugging Face cache).", + ), + disable_tui: bool = typer.Option( + False, + "--disable-tui", + help="Disable Terminal User Interface (TUI) mode. Useful if your platform/terminal doesn't support the multiselect menu.", + ), + sort: Optional[str] = typer.Option( + None, + "--sort", + help="Sort repositories by the specified criteria. Options: 'alphabetical' (A-Z), 'lastUpdated' (newest first), 'lastUsed' (most recent first), 'size' (largest first).", + case_sensitive=False, + ), +) -> None: + sort = _validate_sort_option(sort) + hf_cache_info = scan_cache_dir(dir) + if disable_tui: + selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[], sort_by=sort) + else: + selected_hashes = _manual_review_tui(hf_cache_info, preselected=[], sort_by=sort) + if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes: + confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?" + if disable_tui: + confirmed = _ask_for_confirmation_no_tui(confirm_message) else: - selected_hashes = _manual_review_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) - if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes: - confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?" - if self.disable_tui: - confirmed = _ask_for_confirmation_no_tui(confirm_message) - else: - confirmed = _ask_for_confirmation_tui(confirm_message) - if confirmed: - strategy = hf_cache_info.delete_revisions(*selected_hashes) - print("Start deletion.") - strategy.execute() - print( - f"Done. Deleted {len(strategy.repos)} repo(s) and" - f" {len(strategy.snapshots)} revision(s) for a total of" - f" {strategy.expected_freed_size_str}." - ) - return - print("Deletion is cancelled. Do nothing.") + confirmed = _ask_for_confirmation_tui(confirm_message) + if confirmed: + strategy = hf_cache_info.delete_revisions(*selected_hashes) + print("Start deletion.") + strategy.execute() + print( + f"Done. Deleted {len(strategy.repos)} repo(s) and" + f" {len(strategy.snapshots)} revision(s) for a total of" + f" {strategy.expected_freed_size_str}." + ) + return + print("Deletion is cancelled. Do nothing.") def get_table(hf_cache_info: HFCacheInfo, *, verbosity: int = 0) -> str: diff --git a/src/huggingface_hub/cli/download.py b/src/huggingface_hub/cli/download.py index ea6714d124..15c3b896d4 100644 --- a/src/huggingface_hub/cli/download.py +++ b/src/huggingface_hub/cli/download.py @@ -37,145 +37,178 @@ """ import warnings -from argparse import Namespace, _SubParsersAction from typing import Optional +import typer + from huggingface_hub import logging from huggingface_hub._snapshot_download import snapshot_download -from huggingface_hub.commands import BaseHuggingfaceCLICommand from huggingface_hub.file_download import hf_hub_download from huggingface_hub.utils import disable_progress_bars, enable_progress_bars +from ._cli_utils import validate_repo_type + logger = logging.get_logger(__name__) -class DownloadCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - download_parser = parser.add_parser("download", help="Download files from the Hub") - download_parser.add_argument( - "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)." - ) - download_parser.add_argument( - "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)." - ) - download_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Type of repo to download from (defaults to 'model').", - ) - download_parser.add_argument( - "--revision", - type=str, - help="An optional Git revision id which can be a branch name, a tag, or a commit hash.", - ) - download_parser.add_argument( - "--include", nargs="*", type=str, help="Glob patterns to match files to download." - ) - download_parser.add_argument( - "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download." - ) - download_parser.add_argument( - "--cache-dir", type=str, help="Path to the directory where to save the downloaded files." - ) - download_parser.add_argument( - "--local-dir", - type=str, - help=( - "If set, the downloaded file will be placed under this directory. Check out" - " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more" - " details." - ), - ) - download_parser.add_argument( - "--force-download", - action="store_true", - help="If True, the files will be downloaded even if they are already cached.", - ) - download_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - download_parser.add_argument( - "--quiet", - action="store_true", - help="If True, progress bars are disabled and only the path to the download files is printed.", - ) - download_parser.add_argument( - "--max-workers", - type=int, - default=8, - help="Maximum number of workers to use for downloading files. Default is 8.", - ) - download_parser.set_defaults(func=DownloadCommand) - - def __init__(self, args: Namespace) -> None: - self.token = args.token - self.repo_id: str = args.repo_id - self.filenames: list[str] = args.filenames - self.repo_type: str = args.repo_type - self.revision: Optional[str] = args.revision - self.include: Optional[list[str]] = args.include - self.exclude: Optional[list[str]] = args.exclude - self.cache_dir: Optional[str] = args.cache_dir - self.local_dir: Optional[str] = args.local_dir - self.force_download: bool = args.force_download - self.quiet: bool = args.quiet - self.max_workers: int = args.max_workers - - def run(self) -> None: - if self.quiet: - disable_progress_bars() - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - print(self._download()) # Print path to downloaded files - enable_progress_bars() - else: - logging.set_verbosity_info() - print(self._download()) # Print path to downloaded files - logging.set_verbosity_warning() - - def _download(self) -> str: - # Warn user if patterns are ignored - if len(self.filenames) > 0: - if self.include is not None and len(self.include) > 0: - warnings.warn("Ignoring `--include` since filenames have being explicitly set.") - if self.exclude is not None and len(self.exclude) > 0: - warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.") - - # Single file to download: use `hf_hub_download` - if len(self.filenames) == 1: - return hf_hub_download( - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - filename=self.filenames[0], - cache_dir=self.cache_dir, - force_download=self.force_download, - token=self.token, - local_dir=self.local_dir, - library_name="hf", +def download( + repo_id: str = typer.Argument( + ..., + help="ID of the repo to download from (e.g. `username/repo-name`).", + ), + filenames: Optional[list[str]] = typer.Argument( + None, + help="Files to download (e.g. `config.json`, `data/metadata.jsonl`).", + ), + repo_type: Optional[str] = typer.Option( + "model", + "--repo-type", + help="Type of repo to download from.", + ), + revision: Optional[str] = typer.Option( + None, + "--revision", + help="Git revision id which can be a branch name, a tag, or a commit hash.", + ), + include: Optional[list[str]] = typer.Option( + None, + "--include", + help="Glob patterns to include from files to download. eg: *.json", + ), + exclude: Optional[list[str]] = typer.Option( + None, + "--exclude", + help="Glob patterns to exclude from files to download.", + ), + cache_dir: Optional[str] = typer.Option( + None, + "--cache-dir", + help="Directory where to save files.", + ), + local_dir: Optional[str] = typer.Option( + None, + "--local-dir", + help="If set, the downloaded file will be placed under this directory. Check out https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more details.", + ), + force_download: Optional[bool] = typer.Option( + False, + "--force-download", + help="If True, the files will be downloaded even if they are already cached.", + ), + token: Optional[str] = typer.Option( + None, + "--token", + help="A User Access Token generated from https://huggingface.co/settings/tokens", + ), + quiet: Optional[bool] = typer.Option( + False, + "--quiet", + help="If True, progress bars are disabled and only the path to the download files is printed.", + ), + max_workers: Optional[int] = typer.Option( + 8, + "--max-workers", + help="Maximum number of workers to use for downloading files. Default is 8.", + ), +) -> None: + """Download files from the Hub.""" + # Validate repo_type if provided + repo_type = validate_repo_type(repo_type) + + if quiet: + disable_progress_bars() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + print( + _download_impl( + repo_id=repo_id, + filenames=filenames or [], + repo_type=repo_type, + revision=revision, + include=include, + exclude=exclude, + cache_dir=cache_dir, + local_dir=local_dir, + force_download=force_download, + token=token, + max_workers=max_workers, + ) ) - - # Otherwise: use `snapshot_download` to ensure all files comes from same revision - elif len(self.filenames) == 0: - allow_patterns = self.include - ignore_patterns = self.exclude - else: - allow_patterns = self.filenames - ignore_patterns = None - - return snapshot_download( - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - force_download=self.force_download, - cache_dir=self.cache_dir, - token=self.token, - local_dir=self.local_dir, + enable_progress_bars() + else: + logging.set_verbosity_info() + print( + _download_impl( + repo_id=repo_id, + filenames=filenames or [], + repo_type=repo_type, + revision=revision, + include=include, + exclude=exclude, + cache_dir=cache_dir, + local_dir=local_dir, + force_download=force_download, + token=token, + max_workers=max_workers, + ) + ) + logging.set_verbosity_warning() + + +def _download_impl( + *, + repo_id: str, + filenames: list[str], + repo_type: str, + revision: Optional[str], + include: Optional[list[str]], + exclude: Optional[list[str]], + cache_dir: Optional[str], + local_dir: Optional[str], + force_download: bool, + token: Optional[str], + max_workers: int, +) -> str: + # Warn user if patterns are ignored + if len(filenames) > 0: + if include is not None and len(include) > 0: + warnings.warn("Ignoring `--include` since filenames have being explicitly set.") + if exclude is not None and len(exclude) > 0: + warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.") + + # Single file to download: use `hf_hub_download` + if len(filenames) == 1: + return hf_hub_download( + repo_id=repo_id, + repo_type=repo_type, + revision=revision, + filename=filenames[0], + cache_dir=cache_dir, + force_download=force_download, + token=token, + local_dir=local_dir, library_name="hf", - max_workers=self.max_workers, ) + + # Otherwise: use `snapshot_download` to ensure all files comes from same revision + elif len(filenames) == 0: + allow_patterns = include + ignore_patterns = exclude + else: + allow_patterns = filenames + ignore_patterns = None + + return snapshot_download( + repo_id=repo_id, + repo_type=repo_type, + revision=revision, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + force_download=force_download, + cache_dir=cache_dir, + token=token, + local_dir=local_dir, + library_name="hf", + max_workers=max_workers, + ) diff --git a/src/huggingface_hub/cli/hf.py b/src/huggingface_hub/cli/hf.py index 2587918b29..ba6b714644 100644 --- a/src/huggingface_hub/cli/hf.py +++ b/src/huggingface_hub/cli/hf.py @@ -12,51 +12,69 @@ # See the License for the specific language governing permissions and # limitations under the License. -from argparse import ArgumentParser +import typer -from huggingface_hub.cli.auth import AuthCommands -from huggingface_hub.cli.cache import CacheCommand -from huggingface_hub.cli.download import DownloadCommand -from huggingface_hub.cli.jobs import JobsCommands -from huggingface_hub.cli.lfs import LfsCommands -from huggingface_hub.cli.repo import RepoCommands -from huggingface_hub.cli.repo_files import RepoFilesCommand -from huggingface_hub.cli.system import EnvironmentCommand, VersionCommand -from huggingface_hub.cli.upload import UploadCommand -from huggingface_hub.cli.upload_large_folder import UploadLargeFolderCommand +from huggingface_hub.cli.auth import auth_app +from huggingface_hub.cli.cache import cache_app +from huggingface_hub.cli.download import download +from huggingface_hub.cli.jobs import jobs_app + +# from huggingface_hub.cli.jobs import jobs_app +from huggingface_hub.cli.lfs import lfs_enable_largefiles, lfs_multipart_upload +from huggingface_hub.cli.repo import repo_app +from huggingface_hub.cli.repo_files import repo_files_app +from huggingface_hub.cli.system import env as env_command +from huggingface_hub.cli.system import version as version_command +from huggingface_hub.cli.upload import upload +from huggingface_hub.cli.upload_large_folder import upload_large_folder + + +app = typer.Typer(add_completion=False, no_args_is_help=True, help="Hugging Face Hub CLI") + + +# top level single commands (defined in their respective files) +app.command( + name="download", + help="Download files from the Hub", +)(download) +app.command( + name="upload", + help="Upload a file or a folder to the Hub", +)(upload) +app.command( + name="upload-large-folder", + help="Upload a large folder to the Hub. Recommended for resumable uploads.", +)(upload_large_folder) +app.command( + name="env", + help="Print information about the environment.", +)(env_command) +app.command( + name="version", + help="Print information about the hf version.", +)(version_command) +app.command( + name="lfs-enable-largefiles", + help="Configure your repository to enable upload of files > 5GB.", + hidden=True, +)(lfs_enable_largefiles) +app.command( + name="lfs-multipart-upload", + help="Upload large files to the Hub.", + hidden=True, +)(lfs_multipart_upload) + + +# command groups +app.add_typer(auth_app, name="auth") +app.add_typer(cache_app, name="cache") +app.add_typer(repo_app, name="repo") +app.add_typer(repo_files_app, name="repo-files") +app.add_typer(jobs_app, name="jobs") def main(): - parser = ArgumentParser("hf", usage="hf []") - commands_parser = parser.add_subparsers(help="hf command helpers") - - # Register commands - AuthCommands.register_subcommand(commands_parser) - CacheCommand.register_subcommand(commands_parser) - DownloadCommand.register_subcommand(commands_parser) - JobsCommands.register_subcommand(commands_parser) - RepoCommands.register_subcommand(commands_parser) - RepoFilesCommand.register_subcommand(commands_parser) - UploadCommand.register_subcommand(commands_parser) - UploadLargeFolderCommand.register_subcommand(commands_parser) - - # System commands - EnvironmentCommand.register_subcommand(commands_parser) - VersionCommand.register_subcommand(commands_parser) - - # LFS commands (hidden in --help) - LfsCommands.register_subcommand(commands_parser) - - # Let's go - args = parser.parse_args() - if not hasattr(args, "func"): - parser.print_help() - exit(1) - - # Run - service = args.func(args) - if service is not None: - service.run() + app() if __name__ == "__main__": diff --git a/src/huggingface_hub/cli/jobs.py b/src/huggingface_hub/cli/jobs.py index b4f8385e1d..81465deb06 100644 --- a/src/huggingface_hub/cli/jobs.py +++ b/src/huggingface_hub/cli/jobs.py @@ -28,161 +28,828 @@ # Cancel a running job hf jobs cancel + + # Run a UV script + hf jobs uv run