From 179e4e5777b4b9b70ca17c67524e7840af1d6bcb Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Thu, 2 May 2019 21:29:44 +0100 Subject: [PATCH 1/3] Add support for YAML config files --- requirements.in | 2 ++ requirements.txt | 6 ++++- src/checks.py | 16 +++--------- src/config.py | 18 +++++++++++++ src/main.py | 67 +++++++++++++++++------------------------------- 5 files changed, 53 insertions(+), 56 deletions(-) create mode 100644 src/config.py diff --git a/requirements.in b/requirements.in index 1b6a7fe..49ea77c 100644 --- a/requirements.in +++ b/requirements.in @@ -1,2 +1,4 @@ click +click-config-file colorama +pyyaml diff --git a/requirements.txt b/requirements.txt index 3e12358..2847f17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,11 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --upgrade requirements.txt +# pip-compile # +click-config-file==0.5.0 click==7.0 colorama==0.4.1 +configobj==5.0.6 # via click-config-file +pyyaml==5.1 +six==1.12.0 # via configobj diff --git a/src/checks.py b/src/checks.py index 743995b..6b7378c 100644 --- a/src/checks.py +++ b/src/checks.py @@ -8,23 +8,14 @@ from typing import Callable, Dict, List, Optional, Tuple, Union import apache_2_license +from config import Config from helpers import sh, step from report import Report, Result, ResultKind, color_result @dataclass -class State: - project: str - module: Optional[str] - version: str +class State(Config): work_dir: str - incubating: bool - zipname_template: str - sourcedir_template: str - github_reponame_template: str - gpg_key: str - git_hash: str - build_and_test_command: Optional[str] def _generate_optional_placeholders( self, key: str, value: str, condition: bool @@ -73,7 +64,7 @@ def _pattern_placeholders(self) -> Dict[str, str]: @classmethod def list_placeholder_keys(cls) -> List[str]: # There's probably a better way to do this, but it'll do for now - instance = cls("", "", "", "", False, "", "", "", "", "", None) + instance = cls(*([None] * 13)) # type: ignore return list(instance._pattern_placeholders.keys()) def _format_template(self, template: str) -> str: @@ -170,6 +161,7 @@ def make_check(fun: CheckFun) -> Check: def run_checks(state: State, checks: List[Check]) -> Report: results = [] + for check in checks: step(f"Running check: {check.name}") try: diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..1ef349c --- /dev/null +++ b/src/config.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class Config: + repo: str + project: str + module: Optional[str] + version: str + incubating: bool + zipname_template: str + sourcedir_template: str + github_reponame_template: str + gpg_key: str + git_hash: str + build_and_test_command: Optional[str] + verbose: bool diff --git a/src/main.py b/src/main.py index 740079f..c741dde 100644 --- a/src/main.py +++ b/src/main.py @@ -2,13 +2,16 @@ import os import sys import tempfile -from typing import Optional +from typing import Dict, Optional import click +import click_config_file # type: ignore import colorama +import yaml from colorama import Fore, Style from checks import State, checks, run_checks +from config import Config from helpers import header, sh, step from report import print_report @@ -20,6 +23,13 @@ USER_AGENT = "gh:openzipkin-contrib/apache-release-verification" +def yaml_config_provider(path: str, cmd_name: str) -> Dict: + with open(path) as f: + return { + key.replace("-", "_"): value for key, value in yaml.safe_load(f).items() + } + + @click.command() @click.option("--project", default="zipkin") @click.option("--module") @@ -63,59 +73,30 @@ "as the working directory.", ) @click.option("-v", "--verbose", is_flag=True) -def main( - project: str, - module: Optional[str], - version: str, - git_hash: str, - gpg_key: str, - repo: str, - incubating: bool, - zipname_template: str, - sourcedir_template: str, - github_reponame_template: str, - build_and_test_command: Optional[str], - verbose: bool, -) -> None: - configure_logging(verbose) - logging.debug( - f"Arguments: project={project} module={module} version={version} " - f"incubating={incubating} verbose={verbose} " - f"zipname_template={zipname_template} sourcedir_template={sourcedir_template} " - f"github_reponame_template={github_reponame_template} " - f"build_and_test_command={build_and_test_command} " - f"gpg_key={gpg_key} git_hash={git_hash}" - ) +@click_config_file.configuration_option(implicit=False, provider=yaml_config_provider) +def main(**kwargs) -> None: + config = Config(**kwargs) + configure_logging(config.verbose) - header_msg = f"Verifying release candidate for {project}" - if module: - header_msg += f"/{module}" - header_msg += f" {version}" + logging.debug(config) + + header_msg = f"Verifying release candidate for {config.project}" + if config.module: + header_msg += f"/{config.module}" + header_msg += f" {config.version}" header(header_msg) logging.info(f"{Fore.YELLOW}{DISCLAIMER}{Style.RESET_ALL}") workdir = make_and_enter_workdir() logging.info(f"Working directory: {workdir}") - base_url = generate_base_url(repo, project, incubating) + base_url = generate_base_url(config.repo, config.project, config.incubating) logging.debug(f"Base URL: {base_url}") - fetch_project(base_url, module, version, incubating) + fetch_project(base_url, config.module, config.version, config.incubating) fetch_keys(base_url) - state = State( - project=project, - module=module, - version=version, - work_dir=workdir, - incubating=incubating, - zipname_template=zipname_template, - sourcedir_template=sourcedir_template, - github_reponame_template=github_reponame_template, - gpg_key=gpg_key, - git_hash=git_hash, - build_and_test_command=build_and_test_command, - ) + state = State(work_dir=workdir, **config.__dict__) # TODO this is the place to filter checks here with optional arguments report = run_checks(state, checks=checks) From ce4719e8b49a19eca35d4fae9d55c26bef5c8154 Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Thu, 2 May 2019 22:10:58 +0100 Subject: [PATCH 2/3] Add support for fetching remote configuration from the gh-pages branch of this repo, for example https://openzipkin-contrib.github.io/apache-release-verification/presets/zipkin/zipkin.yaml --- requirements.in | 1 + requirements.txt | 5 +++++ src/main.py | 29 ++++++++++++++++++++++++----- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/requirements.in b/requirements.in index 49ea77c..12af40d 100644 --- a/requirements.in +++ b/requirements.in @@ -2,3 +2,4 @@ click click-config-file colorama pyyaml +requests diff --git a/requirements.txt b/requirements.txt index 2847f17..2eeb214 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,14 @@ # # pip-compile # +certifi==2019.3.9 # via requests +chardet==3.0.4 # via requests click-config-file==0.5.0 click==7.0 colorama==0.4.1 configobj==5.0.6 # via click-config-file +idna==2.8 # via requests pyyaml==5.1 +requests==2.21.0 six==1.12.0 # via configobj +urllib3==1.24.3 # via requests diff --git a/src/main.py b/src/main.py index c741dde..ff3bfdc 100644 --- a/src/main.py +++ b/src/main.py @@ -2,11 +2,12 @@ import os import sys import tempfile -from typing import Dict, Optional +from typing import Any, Dict, Optional import click import click_config_file # type: ignore import colorama +import requests import yaml from colorama import Fore, Style @@ -23,11 +24,24 @@ USER_AGENT = "gh:openzipkin-contrib/apache-release-verification" +def _load_yaml(x: Any) -> Dict: + return {key.replace("-", "_"): value for key, value in yaml.safe_load(x).items()} + + def yaml_config_provider(path: str, cmd_name: str) -> Dict: with open(path) as f: - return { - key.replace("-", "_"): value for key, value in yaml.safe_load(f).items() - } + return _load_yaml(f) + + +def remote_config_provider(url: str, cmd_name: str) -> Dict: + if not url.startswith("http://") and not url.startswith("https://"): + url = ( + "https://openzipkin-contrib.github.io/apache-release-verification/" + f"presets/{url}.yaml" + ) + resp = requests.get(url, headers={"User-Agent": USER_AGENT}) + resp.raise_for_status() + return _load_yaml(resp.content) @click.command() @@ -73,7 +87,12 @@ def yaml_config_provider(path: str, cmd_name: str) -> Dict: "as the working directory.", ) @click.option("-v", "--verbose", is_flag=True) -@click_config_file.configuration_option(implicit=False, provider=yaml_config_provider) +@click_config_file.configuration_option( + "--config", implicit=False, provider=yaml_config_provider +) +@click_config_file.configuration_option( + "--remote-config", implicit=False, provider=remote_config_provider +) def main(**kwargs) -> None: config = Config(**kwargs) configure_logging(config.verbose) From dc71c909f3b3c37d3ded30810776138069096a0e Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Wed, 15 May 2019 18:35:43 +0100 Subject: [PATCH 3/3] Infer --remote-config from --project and --module --- requirements.in | 1 - requirements.txt | 3 - src/checks.py | 2 +- src/config.py | 1 - src/main.py | 145 +++++++++++++++++++++++++++++++++++++---------- 5 files changed, 116 insertions(+), 36 deletions(-) diff --git a/requirements.in b/requirements.in index 12af40d..9c2d8c7 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,4 @@ click -click-config-file colorama pyyaml requests diff --git a/requirements.txt b/requirements.txt index 2eeb214..c2d6d42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,12 +6,9 @@ # certifi==2019.3.9 # via requests chardet==3.0.4 # via requests -click-config-file==0.5.0 click==7.0 colorama==0.4.1 -configobj==5.0.6 # via click-config-file idna==2.8 # via requests pyyaml==5.1 requests==2.21.0 -six==1.12.0 # via configobj urllib3==1.24.3 # via requests diff --git a/src/checks.py b/src/checks.py index 6b7378c..466421b 100644 --- a/src/checks.py +++ b/src/checks.py @@ -64,7 +64,7 @@ def _pattern_placeholders(self) -> Dict[str, str]: @classmethod def list_placeholder_keys(cls) -> List[str]: # There's probably a better way to do this, but it'll do for now - instance = cls(*([None] * 13)) # type: ignore + instance = cls(*([None] * 12)) # type: ignore return list(instance._pattern_placeholders.keys()) def _format_template(self, template: str) -> str: diff --git a/src/config.py b/src/config.py index 1ef349c..71c5a57 100644 --- a/src/config.py +++ b/src/config.py @@ -15,4 +15,3 @@ class Config: gpg_key: str git_hash: str build_and_test_command: Optional[str] - verbose: bool diff --git a/src/main.py b/src/main.py index ff3bfdc..a298cf6 100644 --- a/src/main.py +++ b/src/main.py @@ -2,10 +2,9 @@ import os import sys import tempfile -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Union import click -import click_config_file # type: ignore import colorama import requests import yaml @@ -28,31 +27,123 @@ def _load_yaml(x: Any) -> Dict: return {key.replace("-", "_"): value for key, value in yaml.safe_load(x).items()} -def yaml_config_provider(path: str, cmd_name: str) -> Dict: - with open(path) as f: - return _load_yaml(f) +def local_config_callback( + ctx: click.Context, + _param: Union[click.Option, click.Parameter], + value: Optional[str], +) -> Optional[str]: + if value is None: + logging.debug("local_config_callback: value is None, not loading anything") + return None + with open(value) as f: + data = _load_yaml(f) + logging.debug(f"local_config_callback: loaded data from {value}: {data}") + original = ctx.default_map or {} + ctx.default_map = {**original, **data} + return value -def remote_config_provider(url: str, cmd_name: str) -> Dict: +def remote_config_provider(is_default: bool, url: str) -> Dict: if not url.startswith("http://") and not url.startswith("https://"): url = ( "https://openzipkin-contrib.github.io/apache-release-verification/" f"presets/{url}.yaml" ) + logging.debug(f"remote_config_provider: Loading remote config from {url}") resp = requests.get(url, headers={"User-Agent": USER_AGENT}) - resp.raise_for_status() - return _load_yaml(resp.content) + try: + resp.raise_for_status() + data = _load_yaml(resp.content) + logging.debug(f"remote_config_provider: Loaded data: {data}") + return data + except requests.exceptions.HTTPError: + if is_default: + return {} + else: + raise + + +def remote_config_callback( + ctx: click.Context, + _param: Union[click.Option, click.Parameter], + value: Optional[str], +) -> Optional[str]: + is_default = False + if value is None: + is_default = True + project = ctx.params["project"] + module = ctx.params["module"] + if project is not None and module is not None: + value = f"{project}/{module}" + logging.debug(f"remote_config_callback: inferred URL {value}") + else: + logging.debug( + "remote_config_callback: no value specified, and project or " + "module is None, not fetching remote config" + ) + if value is not None: + original = ctx.default_map or {} + ctx.default_map = {**original, **remote_config_provider(is_default, value)} + return value + + +def configure_logging(verbose: bool): + if verbose: + level = logging.DEBUG + else: + level = logging.INFO + logging.basicConfig(level=level, format="%(message)s") + +def configure_logging_callback( + _ctx: click.Context, _param: Union[click.Option, click.Parameter], verbose: bool +) -> bool: + configure_logging(verbose) + return verbose -@click.command() -@click.option("--project", default="zipkin") -@click.option("--module") + +@click.command(context_settings=dict(max_content_width=120)) +@click.option( + "-v", + "--verbose", + is_flag=True, + expose_value=False, + # We don't actually use this; it's evaluated in the __main__ block. + # See comment there for details. +) +@click.option("--project", default="zipkin", is_eager=True) +@click.option("--module", is_eager=True) +@click.option( + "--config", + default=None, + callback=local_config_callback, + expose_value=False, + is_eager=True, + help="Path to a local .yml file to load options from.", +) +@click.option( + "--remote-config", + default=None, + callback=remote_config_callback, + expose_value=False, + is_eager=True, + help="Remote file to load options from. Can be a full HTTP(S) URL, or a " + "simple string PROJECT/MODULE, which will be expanded to load from " + "the central repository at https://openzipkin-contrib.github.io/" + "apache-release-verification/presets/PROJECT/MODULE.yaml. Defaults " + "to $PROJECT/$MODULE", +) @click.option("--version", required=True) @click.option("--gpg-key", required=True, help="ID of GPG key used to sign the release") @click.option( "--git-hash", required=True, help="Git hash of the commit the release is built from" ) -@click.option("--repo", default="dev", help="dev, release, or test") +@click.option( + "--repo", + type=click.Choice(["dev", "release", "test"]), + default="dev", + help="dev, release, or test", +) @click.option( "--incubating/--not-incubating", is_flag=True, @@ -86,18 +177,9 @@ def remote_config_provider(url: str, cmd_name: str) -> Dict: "test the release. Executed with the exctracted source release archive " "as the working directory.", ) -@click.option("-v", "--verbose", is_flag=True) -@click_config_file.configuration_option( - "--config", implicit=False, provider=yaml_config_provider -) -@click_config_file.configuration_option( - "--remote-config", implicit=False, provider=remote_config_provider -) def main(**kwargs) -> None: config = Config(**kwargs) - configure_logging(config.verbose) - - logging.debug(config) + logging.debug(f"Resolved config: {config}") header_msg = f"Verifying release candidate for {config.project}" if config.module: @@ -130,14 +212,6 @@ def main(**kwargs) -> None: sys.exit(1) -def configure_logging(verbose: bool) -> None: - if verbose: - level = logging.DEBUG - else: - level = logging.INFO - logging.basicConfig(level=level, format="%(message)s") - - def make_and_enter_workdir() -> str: workdir = tempfile.mkdtemp() os.chdir(workdir) @@ -181,4 +255,15 @@ def fetch_keys(base_url: str) -> None: if __name__ == "__main__": colorama.init() + + # There is only a single level of eagerness in Click, and we use that to + # load config options from local or remote files. But we need to handle + # --verbose before that happens, so that we can log from the related + # functions. So... you know, this is it. + if "-v" in sys.argv or "--verbose" in sys.argv: + configure_logging(True) + else: + configure_logging(False) + + # Now we can execute the actual program main()