diff --git a/requirements.in b/requirements.in index 1b6a7fe..9c2d8c7 100644 --- a/requirements.in +++ b/requirements.in @@ -1,2 +1,4 @@ click colorama +pyyaml +requests diff --git a/requirements.txt b/requirements.txt index 3e12358..c2d6d42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,13 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --upgrade requirements.txt +# pip-compile # +certifi==2019.3.9 # via requests +chardet==3.0.4 # via requests click==7.0 colorama==0.4.1 +idna==2.8 # via requests +pyyaml==5.1 +requests==2.21.0 +urllib3==1.24.3 # via requests diff --git a/src/checks.py b/src/checks.py index 743995b..466421b 100644 --- a/src/checks.py +++ b/src/checks.py @@ -8,23 +8,14 @@ from typing import Callable, Dict, List, Optional, Tuple, Union import apache_2_license +from config import Config from helpers import sh, step from report import Report, Result, ResultKind, color_result @dataclass -class State: - project: str - module: Optional[str] - version: str +class State(Config): work_dir: str - incubating: bool - zipname_template: str - sourcedir_template: str - github_reponame_template: str - gpg_key: str - git_hash: str - build_and_test_command: Optional[str] def _generate_optional_placeholders( self, key: str, value: str, condition: bool @@ -73,7 +64,7 @@ def _pattern_placeholders(self) -> Dict[str, str]: @classmethod def list_placeholder_keys(cls) -> List[str]: # There's probably a better way to do this, but it'll do for now - instance = cls("", "", "", "", False, "", "", "", "", "", None) + instance = cls(*([None] * 12)) # type: ignore return list(instance._pattern_placeholders.keys()) def _format_template(self, template: str) -> str: @@ -170,6 +161,7 @@ def make_check(fun: CheckFun) -> Check: def run_checks(state: State, checks: List[Check]) -> Report: results = [] + for check in checks: step(f"Running check: {check.name}") try: diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..71c5a57 --- /dev/null +++ b/src/config.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class Config: + repo: str + project: str + module: Optional[str] + version: str + incubating: bool + zipname_template: str + sourcedir_template: str + github_reponame_template: str + gpg_key: str + git_hash: str + build_and_test_command: Optional[str] diff --git a/src/main.py b/src/main.py index 740079f..a298cf6 100644 --- a/src/main.py +++ b/src/main.py @@ -2,13 +2,16 @@ import os import sys import tempfile -from typing import Optional +from typing import Any, Dict, Optional, Union import click import colorama +import requests +import yaml from colorama import Fore, Style from checks import State, checks, run_checks +from config import Config from helpers import header, sh, step from report import print_report @@ -20,15 +23,127 @@ USER_AGENT = "gh:openzipkin-contrib/apache-release-verification" -@click.command() -@click.option("--project", default="zipkin") -@click.option("--module") +def _load_yaml(x: Any) -> Dict: + return {key.replace("-", "_"): value for key, value in yaml.safe_load(x).items()} + + +def local_config_callback( + ctx: click.Context, + _param: Union[click.Option, click.Parameter], + value: Optional[str], +) -> Optional[str]: + if value is None: + logging.debug("local_config_callback: value is None, not loading anything") + return None + with open(value) as f: + data = _load_yaml(f) + logging.debug(f"local_config_callback: loaded data from {value}: {data}") + original = ctx.default_map or {} + ctx.default_map = {**original, **data} + return value + + +def remote_config_provider(is_default: bool, url: str) -> Dict: + if not url.startswith("http://") and not url.startswith("https://"): + url = ( + "https://openzipkin-contrib.github.io/apache-release-verification/" + f"presets/{url}.yaml" + ) + logging.debug(f"remote_config_provider: Loading remote config from {url}") + resp = requests.get(url, headers={"User-Agent": USER_AGENT}) + try: + resp.raise_for_status() + data = _load_yaml(resp.content) + logging.debug(f"remote_config_provider: Loaded data: {data}") + return data + except requests.exceptions.HTTPError: + if is_default: + return {} + else: + raise + + +def remote_config_callback( + ctx: click.Context, + _param: Union[click.Option, click.Parameter], + value: Optional[str], +) -> Optional[str]: + is_default = False + if value is None: + is_default = True + project = ctx.params["project"] + module = ctx.params["module"] + if project is not None and module is not None: + value = f"{project}/{module}" + logging.debug(f"remote_config_callback: inferred URL {value}") + else: + logging.debug( + "remote_config_callback: no value specified, and project or " + "module is None, not fetching remote config" + ) + if value is not None: + original = ctx.default_map or {} + ctx.default_map = {**original, **remote_config_provider(is_default, value)} + return value + + +def configure_logging(verbose: bool): + if verbose: + level = logging.DEBUG + else: + level = logging.INFO + logging.basicConfig(level=level, format="%(message)s") + + +def configure_logging_callback( + _ctx: click.Context, _param: Union[click.Option, click.Parameter], verbose: bool +) -> bool: + configure_logging(verbose) + return verbose + + +@click.command(context_settings=dict(max_content_width=120)) +@click.option( + "-v", + "--verbose", + is_flag=True, + expose_value=False, + # We don't actually use this; it's evaluated in the __main__ block. + # See comment there for details. +) +@click.option("--project", default="zipkin", is_eager=True) +@click.option("--module", is_eager=True) +@click.option( + "--config", + default=None, + callback=local_config_callback, + expose_value=False, + is_eager=True, + help="Path to a local .yml file to load options from.", +) +@click.option( + "--remote-config", + default=None, + callback=remote_config_callback, + expose_value=False, + is_eager=True, + help="Remote file to load options from. Can be a full HTTP(S) URL, or a " + "simple string PROJECT/MODULE, which will be expanded to load from " + "the central repository at https://openzipkin-contrib.github.io/" + "apache-release-verification/presets/PROJECT/MODULE.yaml. Defaults " + "to $PROJECT/$MODULE", +) @click.option("--version", required=True) @click.option("--gpg-key", required=True, help="ID of GPG key used to sign the release") @click.option( "--git-hash", required=True, help="Git hash of the commit the release is built from" ) -@click.option("--repo", default="dev", help="dev, release, or test") +@click.option( + "--repo", + type=click.Choice(["dev", "release", "test"]), + default="dev", + help="dev, release, or test", +) @click.option( "--incubating/--not-incubating", is_flag=True, @@ -62,60 +177,27 @@ "test the release. Executed with the exctracted source release archive " "as the working directory.", ) -@click.option("-v", "--verbose", is_flag=True) -def main( - project: str, - module: Optional[str], - version: str, - git_hash: str, - gpg_key: str, - repo: str, - incubating: bool, - zipname_template: str, - sourcedir_template: str, - github_reponame_template: str, - build_and_test_command: Optional[str], - verbose: bool, -) -> None: - configure_logging(verbose) - logging.debug( - f"Arguments: project={project} module={module} version={version} " - f"incubating={incubating} verbose={verbose} " - f"zipname_template={zipname_template} sourcedir_template={sourcedir_template} " - f"github_reponame_template={github_reponame_template} " - f"build_and_test_command={build_and_test_command} " - f"gpg_key={gpg_key} git_hash={git_hash}" - ) +def main(**kwargs) -> None: + config = Config(**kwargs) + logging.debug(f"Resolved config: {config}") - header_msg = f"Verifying release candidate for {project}" - if module: - header_msg += f"/{module}" - header_msg += f" {version}" + header_msg = f"Verifying release candidate for {config.project}" + if config.module: + header_msg += f"/{config.module}" + header_msg += f" {config.version}" header(header_msg) logging.info(f"{Fore.YELLOW}{DISCLAIMER}{Style.RESET_ALL}") workdir = make_and_enter_workdir() logging.info(f"Working directory: {workdir}") - base_url = generate_base_url(repo, project, incubating) + base_url = generate_base_url(config.repo, config.project, config.incubating) logging.debug(f"Base URL: {base_url}") - fetch_project(base_url, module, version, incubating) + fetch_project(base_url, config.module, config.version, config.incubating) fetch_keys(base_url) - state = State( - project=project, - module=module, - version=version, - work_dir=workdir, - incubating=incubating, - zipname_template=zipname_template, - sourcedir_template=sourcedir_template, - github_reponame_template=github_reponame_template, - gpg_key=gpg_key, - git_hash=git_hash, - build_and_test_command=build_and_test_command, - ) + state = State(work_dir=workdir, **config.__dict__) # TODO this is the place to filter checks here with optional arguments report = run_checks(state, checks=checks) @@ -130,14 +212,6 @@ def main( sys.exit(1) -def configure_logging(verbose: bool) -> None: - if verbose: - level = logging.DEBUG - else: - level = logging.INFO - logging.basicConfig(level=level, format="%(message)s") - - def make_and_enter_workdir() -> str: workdir = tempfile.mkdtemp() os.chdir(workdir) @@ -181,4 +255,15 @@ def fetch_keys(base_url: str) -> None: if __name__ == "__main__": colorama.init() + + # There is only a single level of eagerness in Click, and we use that to + # load config options from local or remote files. But we need to handle + # --verbose before that happens, so that we can log from the related + # functions. So... you know, this is it. + if "-v" in sys.argv or "--verbose" in sys.argv: + configure_logging(True) + else: + configure_logging(False) + + # Now we can execute the actual program main()