diff --git a/docs/source/en/guides/cli.md b/docs/source/en/guides/cli.md index cd3e6cacfd..c9e2c74364 100644 --- a/docs/source/en/guides/cli.md +++ b/docs/source/en/guides/cli.md @@ -24,24 +24,27 @@ Once installed, you can check that the CLI is correctly setup: ``` >>> hf --help -usage: hf [] - -positional arguments: - {auth,cache,download,repo,repo-files,upload,upload-large-folder,env,version,lfs-enable-largefiles,lfs-multipart-upload} - hf command helpers - auth Manage authentication (login, logout, etc.). - cache Manage local cache directory. - download Download files from the Hub - repo Manage repos on the Hub. - repo-files Manage files in a repo on the Hub. - upload Upload a file or a folder to the Hub. Recommended for single-commit uploads. - upload-large-folder - Upload a large folder to the Hub. Recommended for resumable uploads. - env Print information about the environment. - version Print information about the hf version. - -options: - -h, --help show this help message and exit +Usage: hf [OPTIONS] COMMAND [ARGS]... + + Hugging Face Hub CLI + +Options: + --install-completion Install completion for the current shell. + --show-completion Show completion for the current shell, to copy it or + customize the installation. + --help Show this message and exit. + +Commands: + download Download files from the Hub. + upload Upload a file or a folder to the Hub. + upload-large-folder Upload a large folder to the Hub. + env Print information about the environment. + version Print information about the hf version. + auth Manage authentication (login, logout, etc.). + cache Manage local cache directory. + repo Manage repos on the Hub. + repo-files Manage files in a repo on the Hub. + jobs Run and manage Jobs on the Hub. ``` If the CLI is correctly installed, you should see a list of all the options available in the CLI. If you get an error message such as `command not found: hf`, please refer to the [Installation](../installation) guide. diff --git a/setup.py b/setup.py index 97fc2efbe8..85b9192f4f 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ def get_version() -> str: "pyyaml>=5.1", "httpx>=0.23.0, <1", "tqdm>=4.42.1", + "typer-slim", "typing-extensions>=3.7.4.3", # to be able to import TypeAlias ] @@ -28,6 +29,7 @@ def get_version() -> str: extras["cli"] = [ "InquirerPy==0.3.4", # Note: installs `prompt-toolkit` in the background + "shellingham", ] extras["inference"] = [ diff --git a/src/huggingface_hub/cli/__init__.py b/src/huggingface_hub/cli/__init__.py index 7a1a8d793b..8568c82be1 100644 --- a/src/huggingface_hub/cli/__init__.py +++ b/src/huggingface_hub/cli/__init__.py @@ -11,17 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from abc import ABC, abstractmethod -from argparse import _SubParsersAction - - -class BaseHuggingfaceCLICommand(ABC): - @staticmethod - @abstractmethod - def register_subcommand(parser: _SubParsersAction): - raise NotImplementedError() - - @abstractmethod - def run(self): - raise NotImplementedError() diff --git a/src/huggingface_hub/cli/_cli_utils.py b/src/huggingface_hub/cli/_cli_utils.py index d0f0b98d8a..2cd08d3416 100644 --- a/src/huggingface_hub/cli/_cli_utils.py +++ b/src/huggingface_hub/cli/_cli_utils.py @@ -11,10 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Contains a utility for good-looking prints.""" +"""Contains CLI utilities (styling, helpers).""" import os -from typing import Union +from enum import Enum +from typing import Annotated, Optional, Union + +import click +import typer + +from huggingface_hub import __version__ +from huggingface_hub.hf_api import HfApi class ANSI: @@ -67,3 +74,73 @@ def tabulate(rows: list[list[Union[str, int]]], headers: list[str]) -> str: for row in rows: lines.append(row_format.format(*row)) return "\n".join(lines) + + +#### TYPER UTILS + + +class AlphabeticalMixedGroup(typer.core.TyperGroup): + """ + Typer Group that lists commands and sub-apps mixed and alphabetically. + """ + + def list_commands(self, ctx: click.Context) -> list[str]: # type: ignore[name-defined] + # click.Group stores both commands and sub-groups in `self.commands` + return sorted(self.commands.keys()) + + +def typer_factory(help: str) -> typer.Typer: + return typer.Typer( + help=help, + add_completion=True, + rich_markup_mode=None, + no_args_is_help=True, + cls=AlphabeticalMixedGroup, + ) + + +class RepoType(str, Enum): + model = "model" + dataset = "dataset" + space = "space" + + +RepoIdArg = Annotated[ + str, + typer.Argument( + help="The ID of the repo (e.g. `username/repo-name`).", + ), +] + + +RepoTypeOpt = Annotated[ + RepoType, + typer.Option( + help="The type of repository (model, dataset, or space).", + ), +] + +TokenOpt = Annotated[ + Optional[str], + typer.Option( + help="A User Access Token generated from https://huggingface.co/settings/tokens.", + ), +] + +PrivateOpt = Annotated[ + bool, + typer.Option( + help="Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists.", + ), +] + +RevisionOpt = Annotated[ + Optional[str], + typer.Option( + help="Git revision id which can be a branch name, a tag, or a commit hash.", + ), +] + + +def get_hf_api(token: Optional[str] = None) -> HfApi: + return HfApi(token=token, library_name="hf", library_version=__version__) diff --git a/src/huggingface_hub/cli/auth.py b/src/huggingface_hub/cli/auth.py index 8740a86423..ce075b31f6 100644 --- a/src/huggingface_hub/cli/auth.py +++ b/src/huggingface_hub/cli/auth.py @@ -30,17 +30,17 @@ hf auth whoami """ -from argparse import _SubParsersAction -from typing import Optional +from typing import Annotated, Optional + +import typer -from huggingface_hub.commands import BaseHuggingfaceCLICommand from huggingface_hub.constants import ENDPOINT from huggingface_hub.errors import HfHubHTTPError -from huggingface_hub.hf_api import HfApi +from huggingface_hub.hf_api import whoami from .._login import auth_list, auth_switch, login, logout from ..utils import get_stored_tokens, get_token, logging -from ._cli_utils import ANSI +from ._cli_utils import ANSI, TokenOpt, typer_factory logger = logging.get_logger(__name__) @@ -54,125 +54,42 @@ _inquirer_py_available = False -class AuthCommands(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - # Create the main 'auth' command - auth_parser = parser.add_parser("auth", help="Manage authentication (login, logout, etc.).") - auth_subparsers = auth_parser.add_subparsers(help="Authentication subcommands") - - # Show help if no subcommand is provided - auth_parser.set_defaults(func=lambda args: auth_parser.print_help()) - - # Add 'login' as a subcommand of 'auth' - login_parser = auth_subparsers.add_parser( - "login", help="Log in using a token from huggingface.co/settings/tokens" - ) - login_parser.add_argument( - "--token", - type=str, - help="Token generated from https://huggingface.co/settings/tokens", - ) - login_parser.add_argument( - "--add-to-git-credential", - action="store_true", - help="Optional: Save token to git credential helper.", - ) - login_parser.set_defaults(func=lambda args: AuthLogin(args)) - - # Add 'logout' as a subcommand of 'auth' - logout_parser = auth_subparsers.add_parser("logout", help="Log out") - logout_parser.add_argument( - "--token-name", - type=str, - help="Optional: Name of the access token to log out from.", - ) - logout_parser.set_defaults(func=lambda args: AuthLogout(args)) - - # Add 'whoami' as a subcommand of 'auth' - whoami_parser = auth_subparsers.add_parser( - "whoami", help="Find out which huggingface.co account you are logged in as." - ) - whoami_parser.set_defaults(func=lambda args: AuthWhoami(args)) - - # Existing subcommands - auth_switch_parser = auth_subparsers.add_parser("switch", help="Switch between access tokens") - auth_switch_parser.add_argument( - "--token-name", - type=str, - help="Optional: Name of the access token to switch to.", - ) - auth_switch_parser.add_argument( - "--add-to-git-credential", - action="store_true", - help="Optional: Save token to git credential helper.", - ) - auth_switch_parser.set_defaults(func=lambda args: AuthSwitch(args)) - - auth_list_parser = auth_subparsers.add_parser("list", help="List all stored access tokens") - auth_list_parser.set_defaults(func=lambda args: AuthList(args)) - - -class BaseAuthCommand: - def __init__(self, args): - self.args = args - self._api = HfApi() - - -class AuthLogin(BaseAuthCommand): - def run(self): - logging.set_verbosity_info() - login( - token=self.args.token, - add_to_git_credential=self.args.add_to_git_credential, - ) - - -class AuthLogout(BaseAuthCommand): - def run(self): - logging.set_verbosity_info() - logout(token_name=self.args.token_name) - - -class AuthSwitch(BaseAuthCommand): - def run(self): - logging.set_verbosity_info() - token_name = self.args.token_name - if token_name is None: - token_name = self._select_token_name() - - if token_name is None: - print("No token name provided. Aborting.") - exit() - auth_switch(token_name, add_to_git_credential=self.args.add_to_git_credential) - - def _select_token_name(self) -> Optional[str]: - token_names = list(get_stored_tokens().keys()) - - if not token_names: - logger.error("No stored tokens found. Please login first.") - return None +auth_cli = typer_factory(help="Manage authentication (login, logout, etc.).") + + +@auth_cli.command("login", help="Login using a token from huggingface.co/settings/tokens") +def auth_login( + token: TokenOpt = None, + add_to_git_credential: Annotated[ + bool, + typer.Option( + help="Save to git credential helper. Useful only if you plan to run git commands directly.", + ), + ] = False, +) -> None: + login(token=token, add_to_git_credential=add_to_git_credential) + + +@auth_cli.command("logout", help="Logout from a specific token") +def auth_logout( + token_name: Annotated[ + Optional[str], + typer.Option( + help="Name of token to logout", + ), + ] = None, +) -> None: + logout(token_name=token_name) + - if _inquirer_py_available: - return self._select_token_name_tui(token_names) - # if inquirer is not available, use a simpler terminal UI - print("Available stored tokens:") - for i, token_name in enumerate(token_names, 1): - print(f"{i}. {token_name}") - while True: - try: - choice = input("Enter the number of the token to switch to (or 'q' to quit): ") - if choice.lower() == "q": - return None - index = int(choice) - 1 - if 0 <= index < len(token_names): - return token_names[index] - else: - print("Invalid selection. Please try again.") - except ValueError: - print("Invalid input. Please enter a number or 'q' to quit.") - - def _select_token_name_tui(self, token_names: list[str]) -> Optional[str]: +def _select_token_name() -> Optional[str]: + token_names = list(get_stored_tokens().keys()) + + if not token_names: + logger.error("No stored tokens found. Please login first.") + return None + + if _inquirer_py_available: choices = [Choice(token_name, name=token_name) for token_name in token_names] try: return inquirer.select( @@ -183,30 +100,68 @@ def _select_token_name_tui(self, token_names: list[str]) -> Optional[str]: except KeyboardInterrupt: logger.info("Token selection cancelled.") return None - - -class AuthList(BaseAuthCommand): - def run(self): - logging.set_verbosity_info() - auth_list() - - -class AuthWhoami(BaseAuthCommand): - def run(self): - token = get_token() - if token is None: - print("Not logged in") - exit() + # if inquirer is not available, use a simpler terminal UI + print("Available stored tokens:") + for i, token_name in enumerate(token_names, 1): + print(f"{i}. {token_name}") + while True: try: - info = self._api.whoami(token) - print(ANSI.bold("user: "), info["name"]) - orgs = [org["name"] for org in info["orgs"]] - if orgs: - print(ANSI.bold("orgs: "), ",".join(orgs)) - - if ENDPOINT != "https://huggingface.co": - print(f"Authenticated through private endpoint: {ENDPOINT}") - except HfHubHTTPError as e: - print(e) - print(ANSI.red(e.response.text)) - exit(1) + choice = input("Enter the number of the token to switch to (or 'q' to quit): ") + if choice.lower() == "q": + return None + index = int(choice) - 1 + if 0 <= index < len(token_names): + return token_names[index] + else: + print("Invalid selection. Please try again.") + except ValueError: + print("Invalid input. Please enter a number or 'q' to quit.") + + +@auth_cli.command("switch", help="Switch between access tokens") +def auth_switch_cmd( + token_name: Annotated[ + Optional[str], + typer.Option( + help="Name of the token to switch to", + ), + ] = None, + add_to_git_credential: Annotated[ + bool, + typer.Option( + help="Save to git credential helper. Useful only if you plan to run git commands directly.", + ), + ] = False, +) -> None: + if token_name is None: + token_name = _select_token_name() + if token_name is None: + print("No token name provided. Aborting.") + raise typer.Exit() + auth_switch(token_name, add_to_git_credential=add_to_git_credential) + + +@auth_cli.command("list", help="List all stored access tokens") +def auth_list_cmd() -> None: + auth_list() + + +@auth_cli.command("whoami", help="Find out which huggingface.co account you are logged in as.") +def auth_whoami() -> None: + token = get_token() + if token is None: + print("Not logged in") + raise typer.Exit() + try: + info = whoami(token) + print(ANSI.bold("user: "), info["name"]) + orgs = [org["name"] for org in info["orgs"]] + if orgs: + print(ANSI.bold("orgs: "), ",".join(orgs)) + + if ENDPOINT != "https://huggingface.co": + print(f"Authenticated through private endpoint: {ENDPOINT}") + except HfHubHTTPError as e: + print(e) + print(ANSI.red(e.response.text)) + raise typer.Exit(code=1) diff --git a/src/huggingface_hub/cli/cache.py b/src/huggingface_hub/cli/cache.py index 7eb3e82509..c57f9f207f 100644 --- a/src/huggingface_hub/cli/cache.py +++ b/src/huggingface_hub/cli/cache.py @@ -16,14 +16,15 @@ import os import time -from argparse import Namespace, _SubParsersAction +from enum import Enum from functools import wraps from tempfile import mkstemp -from typing import Any, Callable, Iterable, Literal, Optional, Union +from typing import Annotated, Any, Callable, Iterable, Optional, Union + +import typer from ..utils import CachedRepoInfo, CachedRevisionInfo, CacheNotFound, HFCacheInfo, scan_cache_dir -from . import BaseHuggingfaceCLICommand -from ._cli_utils import ANSI, tabulate +from ._cli_utils import ANSI, tabulate, typer_factory # --- DELETE helpers (from delete_cache.py) --- @@ -36,10 +37,16 @@ except ImportError: _inquirer_py_available = False -SortingOption_T = Literal["alphabetical", "lastUpdated", "lastUsed", "size"] _CANCEL_DELETION_STR = "CANCEL_DELETION" +class SortingOption(str, Enum): + alphabetical = "alphabetical" + lastUpdated = "lastUpdated" + lastUsed = "lastUsed" + size = "size" + + def require_inquirer_py(fn: Callable) -> Callable: @wraps(fn) def _inner(*args, **kwargs): @@ -54,122 +61,93 @@ def _inner(*args, **kwargs): return _inner -class CacheCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - cache_parser = parser.add_parser("cache", help="Manage local cache directory.") - cache_subparsers = cache_parser.add_subparsers(dest="cache_command", help="Cache subcommands") +cache_cli = typer_factory(help="Manage local cache directory.") - # Show help if no subcommand is provided - cache_parser.set_defaults(func=lambda args: cache_parser.print_help()) - # Scan subcommand - scan_parser = cache_subparsers.add_parser("scan", help="Scan cache directory.") - scan_parser.add_argument( - "--dir", - type=str, - default=None, - help="cache directory to scan (optional). Default to the default HuggingFace cache.", - ) - scan_parser.add_argument( +@cache_cli.command("scan", help="Scan the cache directory") +def cache_scan( + dir: Annotated[ + Optional[str], + typer.Option( + help="Cache directory to scan (defaults to Hugging Face cache).", + ), + ] = None, + verbose: Annotated[ + int, + typer.Option( "-v", "--verbose", - action="count", - default=0, - help="show a more verbose output", - ) - scan_parser.set_defaults(func=CacheCommand, cache_command="scan") - # Delete subcommand - delete_parser = cache_subparsers.add_parser("delete", help="Delete revisions from the cache directory.") - delete_parser.add_argument( - "--dir", - type=str, - default=None, - help="cache directory (optional). Default to the default HuggingFace cache.", - ) - delete_parser.add_argument( - "--disable-tui", - action="store_true", - help=( - "Disable Terminal User Interface (TUI) mode. Useful if your platform/terminal doesn't support the multiselect menu." - ), - ) - delete_parser.add_argument( - "--sort", - nargs="?", - choices=["alphabetical", "lastUpdated", "lastUsed", "size"], - help=( - "Sort repositories by the specified criteria. Options: " - "'alphabetical' (A-Z), " - "'lastUpdated' (newest first), " - "'lastUsed' (most recent first), " - "'size' (largest first)." - ), - ) - delete_parser.set_defaults(func=CacheCommand, cache_command="delete") - - def __init__(self, args: Namespace) -> None: - self.args = args - self.verbosity: int = getattr(args, "verbose", 0) - self.cache_dir: Optional[str] = getattr(args, "dir", None) - self.disable_tui: bool = getattr(args, "disable_tui", False) - self.sort_by: Optional[SortingOption_T] = getattr(args, "sort", None) - self.cache_command: Optional[str] = getattr(args, "cache_command", None) - - def run(self): - if self.cache_command == "scan": - self._run_scan() - elif self.cache_command == "delete": - self._run_delete() + count=True, + help="Increase verbosity (-v, -vv, -vvv).", + ), + ] = 0, +) -> None: + try: + t0 = time.time() + hf_cache_info = scan_cache_dir(dir) + t1 = time.time() + except CacheNotFound as exc: + print(f"Cache directory not found: {str(exc.cache_dir)}") + return + print(get_table(hf_cache_info, verbosity=verbose)) + print( + f"\nDone in {round(t1 - t0, 1)}s. Scanned {len(hf_cache_info.repos)} repo(s)" + f" for a total of {ANSI.red(hf_cache_info.size_on_disk_str)}." + ) + if len(hf_cache_info.warnings) > 0: + message = f"Got {len(hf_cache_info.warnings)} warning(s) while scanning." + if verbose >= 3: + print(ANSI.gray(message)) + for warning in hf_cache_info.warnings: + print(ANSI.gray(str(warning))) else: - print("Please specify a cache subcommand (scan or delete). Use -h for help.") - - def _run_scan(self): - try: - t0 = time.time() - hf_cache_info = scan_cache_dir(self.cache_dir) - t1 = time.time() - except CacheNotFound as exc: - cache_dir = exc.cache_dir - print(f"Cache directory not found: {cache_dir}") - return - print(get_table(hf_cache_info, verbosity=self.verbosity)) - print( - f"\nDone in {round(t1 - t0, 1)}s. Scanned {len(hf_cache_info.repos)} repo(s)" - f" for a total of {ANSI.red(hf_cache_info.size_on_disk_str)}." - ) - if len(hf_cache_info.warnings) > 0: - message = f"Got {len(hf_cache_info.warnings)} warning(s) while scanning." - if self.verbosity >= 3: - print(ANSI.gray(message)) - for warning in hf_cache_info.warnings: - print(ANSI.gray(str(warning))) - else: - print(ANSI.gray(message + " Use -vvv to print details.")) - - def _run_delete(self): - hf_cache_info = scan_cache_dir(self.cache_dir) - if self.disable_tui: - selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) + print(ANSI.gray(message + " Use -vvv to print details.")) + + +@cache_cli.command("delete", help="Delete revisions from the cache directory") +def cache_delete( + dir: Annotated[ + Optional[str], + typer.Option( + help="Cache directory (defaults to Hugging Face cache).", + ), + ] = None, + disable_tui: Annotated[ + bool, + typer.Option( + help="Disable Terminal User Interface (TUI) mode. Useful if your platform/terminal doesn't support the multiselect menu.", + ), + ] = False, + sort: Annotated[ + Optional[SortingOption], + typer.Option( + help="Sort repositories by the specified criteria. Options: 'alphabetical' (A-Z), 'lastUpdated' (newest first), 'lastUsed' (most recent first), 'size' (largest first).", + ), + ] = None, +) -> None: + hf_cache_info = scan_cache_dir(dir) + sort_by = sort.value if sort is not None else None + if disable_tui: + selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[], sort_by=sort_by) + else: + selected_hashes = _manual_review_tui(hf_cache_info, preselected=[], sort_by=sort_by) + if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes: + confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?" + if disable_tui: + confirmed = _ask_for_confirmation_no_tui(confirm_message) else: - selected_hashes = _manual_review_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) - if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes: - confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?" - if self.disable_tui: - confirmed = _ask_for_confirmation_no_tui(confirm_message) - else: - confirmed = _ask_for_confirmation_tui(confirm_message) - if confirmed: - strategy = hf_cache_info.delete_revisions(*selected_hashes) - print("Start deletion.") - strategy.execute() - print( - f"Done. Deleted {len(strategy.repos)} repo(s) and" - f" {len(strategy.snapshots)} revision(s) for a total of" - f" {strategy.expected_freed_size_str}." - ) - return - print("Deletion is cancelled. Do nothing.") + confirmed = _ask_for_confirmation_tui(confirm_message) + if confirmed: + strategy = hf_cache_info.delete_revisions(*selected_hashes) + print("Start deletion.") + strategy.execute() + print( + f"Done. Deleted {len(strategy.repos)} repo(s) and" + f" {len(strategy.snapshots)} revision(s) for a total of" + f" {strategy.expected_freed_size_str}." + ) + return + print("Deletion is cancelled. Do nothing.") def get_table(hf_cache_info: HFCacheInfo, *, verbosity: int = 0) -> str: @@ -228,7 +206,7 @@ def get_table(hf_cache_info: HFCacheInfo, *, verbosity: int = 0) -> str: ) -def _get_repo_sorting_key(repo: CachedRepoInfo, sort_by: Optional[SortingOption_T] = None): +def _get_repo_sorting_key(repo: CachedRepoInfo, sort_by: Optional[str] = None): if sort_by == "alphabetical": return (repo.repo_type, repo.repo_id.lower()) elif sort_by == "lastUpdated": @@ -242,9 +220,7 @@ def _get_repo_sorting_key(repo: CachedRepoInfo, sort_by: Optional[SortingOption_ @require_inquirer_py -def _manual_review_tui( - hf_cache_info: HFCacheInfo, preselected: list[str], sort_by: Optional[SortingOption_T] = None -) -> list[str]: +def _manual_review_tui(hf_cache_info: HFCacheInfo, preselected: list[str], sort_by: Optional[str] = None) -> list[str]: choices = _get_tui_choices_from_scan(repos=hf_cache_info.repos, preselected=preselected, sort_by=sort_by) checkbox = inquirer.checkbox( message="Select revisions to delete:", @@ -277,7 +253,7 @@ def _ask_for_confirmation_tui(message: str, default: bool = True) -> bool: def _get_tui_choices_from_scan( - repos: Iterable[CachedRepoInfo], preselected: list[str], sort_by: Optional[SortingOption_T] = None + repos: Iterable[CachedRepoInfo], preselected: list[str], sort_by: Optional[str] = None ) -> list: choices: list[Union["Choice", "Separator"]] = [] choices.append( @@ -306,7 +282,7 @@ def _get_tui_choices_from_scan( def _manual_review_no_tui( - hf_cache_info: HFCacheInfo, preselected: list[str], sort_by: Optional[SortingOption_T] = None + hf_cache_info: HFCacheInfo, preselected: list[str], sort_by: Optional[str] = None ) -> list[str]: fd, tmp_path = mkstemp(suffix=".txt") os.close(fd) diff --git a/src/huggingface_hub/cli/download.py b/src/huggingface_hub/cli/download.py index ea6714d124..5a1ab42d74 100644 --- a/src/huggingface_hub/cli/download.py +++ b/src/huggingface_hub/cli/download.py @@ -37,145 +37,128 @@ """ import warnings -from argparse import Namespace, _SubParsersAction -from typing import Optional +from typing import Annotated, Optional + +import typer from huggingface_hub import logging from huggingface_hub._snapshot_download import snapshot_download -from huggingface_hub.commands import BaseHuggingfaceCLICommand from huggingface_hub.file_download import hf_hub_download from huggingface_hub.utils import disable_progress_bars, enable_progress_bars +from ._cli_utils import RepoIdArg, RepoTypeOpt, RevisionOpt, TokenOpt + logger = logging.get_logger(__name__) -class DownloadCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - download_parser = parser.add_parser("download", help="Download files from the Hub") - download_parser.add_argument( - "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)." - ) - download_parser.add_argument( - "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)." - ) - download_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Type of repo to download from (defaults to 'model').", - ) - download_parser.add_argument( - "--revision", - type=str, - help="An optional Git revision id which can be a branch name, a tag, or a commit hash.", - ) - download_parser.add_argument( - "--include", nargs="*", type=str, help="Glob patterns to match files to download." - ) - download_parser.add_argument( - "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download." - ) - download_parser.add_argument( - "--cache-dir", type=str, help="Path to the directory where to save the downloaded files." - ) - download_parser.add_argument( - "--local-dir", - type=str, - help=( - "If set, the downloaded file will be placed under this directory. Check out" - " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more" - " details." - ), - ) - download_parser.add_argument( - "--force-download", - action="store_true", +def download( + repo_id: RepoIdArg, + filenames: Annotated[ + Optional[list[str]], + typer.Argument( + help="Files to download (e.g. `config.json`, `data/metadata.jsonl`).", + ), + ] = None, + repo_type: RepoTypeOpt = RepoTypeOpt.model, + revision: RevisionOpt = None, + include: Annotated[ + Optional[list[str]], + typer.Option( + help="Glob patterns to include from files to download. eg: *.json", + ), + ] = None, + exclude: Annotated[ + Optional[list[str]], + typer.Option( + help="Glob patterns to exclude from files to download.", + ), + ] = None, + cache_dir: Annotated[ + Optional[str], + typer.Option( + help="Directory where to save files.", + ), + ] = None, + local_dir: Annotated[ + Optional[str], + typer.Option( + help="If set, the downloaded file will be placed under this directory. Check out https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more details.", + ), + ] = None, + force_download: Annotated[ + bool, + typer.Option( help="If True, the files will be downloaded even if they are already cached.", - ) - download_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - download_parser.add_argument( - "--quiet", - action="store_true", + ), + ] = False, + token: TokenOpt = None, + quiet: Annotated[ + bool, + typer.Option( help="If True, progress bars are disabled and only the path to the download files is printed.", - ) - download_parser.add_argument( - "--max-workers", - type=int, - default=8, + ), + ] = False, + max_workers: Annotated[ + int, + typer.Option( help="Maximum number of workers to use for downloading files. Default is 8.", - ) - download_parser.set_defaults(func=DownloadCommand) - - def __init__(self, args: Namespace) -> None: - self.token = args.token - self.repo_id: str = args.repo_id - self.filenames: list[str] = args.filenames - self.repo_type: str = args.repo_type - self.revision: Optional[str] = args.revision - self.include: Optional[list[str]] = args.include - self.exclude: Optional[list[str]] = args.exclude - self.cache_dir: Optional[str] = args.cache_dir - self.local_dir: Optional[str] = args.local_dir - self.force_download: bool = args.force_download - self.quiet: bool = args.quiet - self.max_workers: int = args.max_workers - - def run(self) -> None: - if self.quiet: - disable_progress_bars() - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - print(self._download()) # Print path to downloaded files - enable_progress_bars() - else: - logging.set_verbosity_info() - print(self._download()) # Print path to downloaded files - logging.set_verbosity_warning() + ), + ] = 8, +) -> None: + """Download files from the Hub.""" - def _download(self) -> str: + def run_download() -> str: + filenames_list = filenames if filenames is not None else [] # Warn user if patterns are ignored - if len(self.filenames) > 0: - if self.include is not None and len(self.include) > 0: + if len(filenames_list) > 0: + if include is not None and len(include) > 0: warnings.warn("Ignoring `--include` since filenames have being explicitly set.") - if self.exclude is not None and len(self.exclude) > 0: + if exclude is not None and len(exclude) > 0: warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.") # Single file to download: use `hf_hub_download` - if len(self.filenames) == 1: + if len(filenames_list) == 1: return hf_hub_download( - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - filename=self.filenames[0], - cache_dir=self.cache_dir, - force_download=self.force_download, - token=self.token, - local_dir=self.local_dir, + repo_id=repo_id, + repo_type=repo_type.value, + revision=revision, + filename=filenames_list[0], + cache_dir=cache_dir, + force_download=force_download, + token=token, + local_dir=local_dir, library_name="hf", ) # Otherwise: use `snapshot_download` to ensure all files comes from same revision - elif len(self.filenames) == 0: - allow_patterns = self.include - ignore_patterns = self.exclude + if len(filenames_list) == 0: + allow_patterns = include + ignore_patterns = exclude else: - allow_patterns = self.filenames + allow_patterns = filenames_list ignore_patterns = None return snapshot_download( - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, + repo_id=repo_id, + repo_type=repo_type.value, + revision=revision, allow_patterns=allow_patterns, ignore_patterns=ignore_patterns, - force_download=self.force_download, - cache_dir=self.cache_dir, - token=self.token, - local_dir=self.local_dir, + force_download=force_download, + cache_dir=cache_dir, + token=token, + local_dir=local_dir, library_name="hf", - max_workers=self.max_workers, + max_workers=max_workers, ) + + if quiet: + disable_progress_bars() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + print(run_download()) + enable_progress_bars() + else: + print(run_download()) + logging.set_verbosity_warning() diff --git a/src/huggingface_hub/cli/hf.py b/src/huggingface_hub/cli/hf.py index 2587918b29..bb8b3b80d0 100644 --- a/src/huggingface_hub/cli/hf.py +++ b/src/huggingface_hub/cli/hf.py @@ -12,51 +12,47 @@ # See the License for the specific language governing permissions and # limitations under the License. -from argparse import ArgumentParser -from huggingface_hub.cli.auth import AuthCommands -from huggingface_hub.cli.cache import CacheCommand -from huggingface_hub.cli.download import DownloadCommand -from huggingface_hub.cli.jobs import JobsCommands -from huggingface_hub.cli.lfs import LfsCommands -from huggingface_hub.cli.repo import RepoCommands -from huggingface_hub.cli.repo_files import RepoFilesCommand -from huggingface_hub.cli.system import EnvironmentCommand, VersionCommand -from huggingface_hub.cli.upload import UploadCommand -from huggingface_hub.cli.upload_large_folder import UploadLargeFolderCommand +from huggingface_hub.cli._cli_utils import typer_factory +from huggingface_hub.cli.auth import auth_cli +from huggingface_hub.cli.cache import cache_cli +from huggingface_hub.cli.download import download +from huggingface_hub.cli.jobs import jobs_cli +from huggingface_hub.cli.lfs import lfs_enable_largefiles, lfs_multipart_upload +from huggingface_hub.cli.repo import repo_cli +from huggingface_hub.cli.repo_files import repo_files_cli +from huggingface_hub.cli.system import env, version + +# from huggingface_hub.cli.jobs import jobs_app +from huggingface_hub.cli.upload import upload +from huggingface_hub.cli.upload_large_folder import upload_large_folder +from huggingface_hub.utils import logging + + +app = typer_factory(help="Hugging Face Hub CLI") + + +# top level single commands (defined in their respective files) +app.command(help="Download files from the Hub.")(download) +app.command(help="Upload a file or a folder to the Hub.")(upload) +app.command(help="Upload a large folder to the Hub. Recommended for resumable uploads.")(upload_large_folder) +app.command(name="env", help="Print information about the environment.")(env) +app.command(help="Print information about the hf version.")(version) +app.command(help="Configure your repository to enable upload of files > 5GB.", hidden=True)(lfs_enable_largefiles) +app.command(help="Upload large files to the Hub.", hidden=True)(lfs_multipart_upload) + + +# command groups +app.add_typer(auth_cli, name="auth") +app.add_typer(cache_cli, name="cache") +app.add_typer(repo_cli, name="repo") +app.add_typer(repo_files_cli, name="repo-files") +app.add_typer(jobs_cli, name="jobs") def main(): - parser = ArgumentParser("hf", usage="hf []") - commands_parser = parser.add_subparsers(help="hf command helpers") - - # Register commands - AuthCommands.register_subcommand(commands_parser) - CacheCommand.register_subcommand(commands_parser) - DownloadCommand.register_subcommand(commands_parser) - JobsCommands.register_subcommand(commands_parser) - RepoCommands.register_subcommand(commands_parser) - RepoFilesCommand.register_subcommand(commands_parser) - UploadCommand.register_subcommand(commands_parser) - UploadLargeFolderCommand.register_subcommand(commands_parser) - - # System commands - EnvironmentCommand.register_subcommand(commands_parser) - VersionCommand.register_subcommand(commands_parser) - - # LFS commands (hidden in --help) - LfsCommands.register_subcommand(commands_parser) - - # Let's go - args = parser.parse_args() - if not hasattr(args, "func"): - parser.print_help() - exit(1) - - # Run - service = args.func(args) - if service is not None: - service.run() + logging.set_verbosity_info() + app() if __name__ == "__main__": diff --git a/src/huggingface_hub/cli/jobs.py b/src/huggingface_hub/cli/jobs.py index b4f8385e1d..5b61cd8731 100644 --- a/src/huggingface_hub/cli/jobs.py +++ b/src/huggingface_hub/cli/jobs.py @@ -28,1072 +28,745 @@ # Cancel a running job hf jobs cancel + + # Run a UV script + hf jobs uv run