From 6795cbd02c62a376b72c755b7751140a572c4283 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Fri, 26 Jul 2024 08:41:16 +0100
Subject: [PATCH 1/3] Allow Cylc Config to output a metadata JSON.
Response to review
- Ensure --json help message is useful.
- Make the null value reporting only happen to actual null values.
Prevent --json being combined with print.. options
Tidy up
---
cylc/flow/parsec/OrderedDict.py | 13 +++
cylc/flow/parsec/config.py | 43 +++++++-
cylc/flow/scripts/config.py | 38 ++++++-
tests/functional/cylc-config/11-json-dump.t | 113 ++++++++++++++++++++
4 files changed, 202 insertions(+), 5 deletions(-)
create mode 100644 tests/functional/cylc-config/11-json-dump.t
diff --git a/cylc/flow/parsec/OrderedDict.py b/cylc/flow/parsec/OrderedDict.py
index ed5b85c20f4..8e727719b40 100644
--- a/cylc/flow/parsec/OrderedDict.py
+++ b/cylc/flow/parsec/OrderedDict.py
@@ -100,6 +100,19 @@ def prepend(self, key, value):
self[key] = value
self.move_to_end(key, last=False)
+ @staticmethod
+ def repl_val(target, replace, replacement):
+ """Replace dictionary values with a string.
+
+ Designed to be used recursively.
+ """
+ for key, val in target.items():
+ if isinstance(val, dict):
+ OrderedDictWithDefaults.repl_val(
+ val, replace, replacement)
+ elif val == replace:
+ target[key] = replacement
+
class DictTree:
"""An object providing a single point of access to a tree of dicts.
diff --git a/cylc/flow/parsec/config.py b/cylc/flow/parsec/config.py
index a8ae29f0808..b51a69bdea7 100644
--- a/cylc/flow/parsec/config.py
+++ b/cylc/flow/parsec/config.py
@@ -15,9 +15,11 @@
# along with this program. If not, see .
from copy import deepcopy
+import json
import re
+import sys
from textwrap import dedent
-from typing import TYPE_CHECKING, Callable, Iterable, List, Optional
+from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, TextIO
from cylc.flow.context_node import ContextNode
from cylc.flow.parsec.exceptions import (
@@ -33,6 +35,7 @@
if TYPE_CHECKING:
from optparse import Values
+ from typing_extensions import Literal
class DefaultList(list):
@@ -41,6 +44,7 @@ class DefaultList(list):
class ParsecConfig:
"""Object wrapper for parsec functions."""
+ META: "Literal['meta']" = 'meta'
def __init__(
self,
@@ -166,7 +170,7 @@ def get(self, keys: Optional[Iterable[str]] = None, sparse: bool = False):
return cfg
def idump(self, items=None, sparse=False, prefix='',
- oneline=False, none_str='', handle=None):
+ oneline=False, none_str='', handle=None, json=False):
"""
items is a list of --item style inputs:
'[runtime][foo]script'.
@@ -182,7 +186,40 @@ def idump(self, items=None, sparse=False, prefix='',
mkeys.append(j)
if null:
mkeys = [[]]
- self.mdump(mkeys, sparse, prefix, oneline, none_str, handle=handle)
+ if json:
+ self.jdump(mkeys, sparse, oneline, none_str, handle=handle)
+ else:
+ self.mdump(mkeys, sparse, prefix, oneline, none_str, handle=handle)
+
+ def jdump(
+ self,
+ mkeys: Optional[Iterable] = None,
+ sparse: bool = False,
+ oneline: bool = False,
+ none_str: Optional[str] = None,
+ handle: Optional[TextIO] = None
+ ) -> None:
+ """Dump a config to JSON format.
+
+ Args:
+ mkeys: Items to display.
+ sparse: Only display user set items, not defaults.
+ oneline: Output on a single line.
+ none_str: Value to give instead of null.
+ handle: Where to write the output.
+ """
+ # Use json indent to control online output:
+ indent = None if oneline else 4
+
+ for keys in mkeys or []:
+ if not keys:
+ keys = []
+ cfg = self.get(keys, sparse)
+ if none_str:
+ cfg.repl_val(cfg, None, none_str)
+ data = json.dumps(cfg, indent=indent)
+
+ print(data, file=handle or sys.stdout)
def mdump(self, mkeys=None, sparse=False, prefix='',
oneline=False, none_str='', handle=None):
diff --git a/cylc/flow/scripts/config.py b/cylc/flow/scripts/config.py
index 830eae46e9a..64444dc649a 100755
--- a/cylc/flow/scripts/config.py
+++ b/cylc/flow/scripts/config.py
@@ -111,6 +111,15 @@ def get_option_parser() -> COP:
"overrides any settings it shares with those higher up."),
action="store_true", default=False, dest="print_hierarchy")
+ parser.add_option(
+ '--json',
+ help=(
+ 'Returns config as JSON rather than Cylc Config format.'),
+ default=False,
+ action='store_true',
+ dest='json'
+ )
+
parser.add_option(icp_option)
platform_listing_options_group = parser.add_option_group(
@@ -140,6 +149,28 @@ def get_option_parser() -> COP:
return parser
+def json_opt_check(parser, options):
+ """Return an error if --json and incompatible options used.
+ """
+ not_with_json = {
+ '--print-hierarchy': 'print_hierarchy',
+ '--platform-names': 'print_platform_names',
+ '--platforms': 'print_platforms'
+ }
+
+ if not options.json:
+ return
+
+ not_with_json = [
+ name for name, dest
+ in not_with_json.items()
+ if options.__dict__[dest]]
+
+ if not_with_json:
+ parser.error(
+ f'--json incompatible with {" or ".join(not_with_json)}')
+
+
def get_config_file_hierarchy(workflow_id: Optional[str] = None) -> List[str]:
filepaths = [os.path.join(path, glbl_cfg().CONF_BASENAME)
for _, path in glbl_cfg().conf_dir_hierarchy]
@@ -164,6 +195,7 @@ async def _main(
options: 'Values',
*ids,
) -> None:
+ json_opt_check(parser, options)
if options.print_platform_names and options.print_platforms:
options.print_platform_names = False
@@ -189,7 +221,8 @@ async def _main(
options.item,
not options.defaults,
oneline=options.oneline,
- none_str=options.none_str
+ none_str=options.none_str,
+ json=options.json,
)
return
@@ -219,5 +252,6 @@ async def _main(
options.item,
not options.defaults,
oneline=options.oneline,
- none_str=options.none_str
+ none_str=options.none_str,
+ json=options.json
)
diff --git a/tests/functional/cylc-config/11-json-dump.t b/tests/functional/cylc-config/11-json-dump.t
new file mode 100644
index 00000000000..5e2af26aa33
--- /dev/null
+++ b/tests/functional/cylc-config/11-json-dump.t
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#-------------------------------------------------------------------------------
+# Test cylc config can dump json files.
+# n.b. not heavily tested because most of this functionality
+# is from Standard library json.
+. "$(dirname "$0")/test_header"
+#-------------------------------------------------------------------------------
+set_test_number 9
+#-------------------------------------------------------------------------------
+
+# Test that option parser errors if incompat options given:
+cylc config --json --platforms 2> err
+named_grep_ok "${TEST_NAME_BASE}.CLI-one-incompat-item" \
+ "--json incompatible with --platforms" \
+ err
+
+cylc config --json --platforms --platform-names 2> err
+named_grep_ok "${TEST_NAME_BASE}.CLI-two-incompat-items" \
+ "--json incompatible with --platform-names or --platforms" \
+ err
+
+cylc config --json --platforms --platform-names --print-hierarchy 2> err
+named_grep_ok "${TEST_NAME_BASE}.CLI-three-incompat-items" \
+ "--json incompatible with --print-hierarchy or --platform-names or --platforms" \
+ err
+
+
+# Test the global.cylc
+TEST_NAME="${TEST_NAME_BASE}-global"
+
+cat > "global.cylc" <<__HEREDOC__
+[platforms]
+ [[golders_green]]
+ [[[meta]]]
+ can = "Test lots of things"
+ because = metadata, is, not, fussy
+ number = 99
+__HEREDOC__
+
+export CYLC_CONF_PATH="${PWD}"
+run_ok "${TEST_NAME}" cylc config --json --one-line
+cmp_ok "${TEST_NAME}.stdout" <<__HERE__
+{"platforms": {"golders_green": {"meta": {"can": "Test lots of things", "because": "metadata, is, not, fussy", "number": "99"}}}}
+__HERE__
+
+# Test a flow.cylc
+TEST_NAME="${TEST_NAME_BASE}-workflow"
+
+cat > "flow.cylc" <<__HERE__
+[scheduling]
+ [[graph]]
+ P1D = foo
+
+[runtime]
+ [[foo]]
+__HERE__
+
+run_ok "${TEST_NAME}" cylc config . --json --icp 1000
+cmp_ok "${TEST_NAME}.stdout" <<__HERE__
+{
+ "scheduling": {
+ "graph": {
+ "P1D": "foo"
+ },
+ "initial cycle point": "1000"
+ },
+ "runtime": {
+ "root": {},
+ "foo": {
+ "completion": "succeeded"
+ }
+ }
+}
+__HERE__
+
+# Test an empty global.cylc to check:
+# * item selection
+# * null value setting
+# * showing defaults
+TEST_NAME="${TEST_NAME_BASE}-defaults-item-null-value"
+echo "" > global.cylc
+export CYLC_CONF_PATH="${PWD}"
+
+run_ok "${TEST_NAME}" cylc config \
+ -i '[scheduler][mail]' \
+ --json \
+ --defaults \
+ --null-value='zilch'
+
+cmp_ok "${TEST_NAME}.stdout" <<__HERE__
+{
+ "from": "zilch",
+ "smtp": "zilch",
+ "to": "zilch",
+ "footer": "zilch",
+ "task event batch interval": 300.0
+}
+__HERE__
From c70efd24338681af641d4cec73918fc5d5c5a298 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Thu, 29 Aug 2024 16:12:36 +0100
Subject: [PATCH 2/3] make repl_string work on any dict like object.
---
cylc/flow/parsec/config.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cylc/flow/parsec/config.py b/cylc/flow/parsec/config.py
index b51a69bdea7..48d81cbb748 100644
--- a/cylc/flow/parsec/config.py
+++ b/cylc/flow/parsec/config.py
@@ -216,7 +216,7 @@ def jdump(
keys = []
cfg = self.get(keys, sparse)
if none_str:
- cfg.repl_val(cfg, None, none_str)
+ OrderedDictWithDefaults.repl_val(cfg, None, none_str)
data = json.dumps(cfg, indent=indent)
print(data, file=handle or sys.stdout)
From 754fab4f639aa65a1d5caf730e5a02c7a29ec1c4 Mon Sep 17 00:00:00 2001
From: Tim Pillinger
Date: Mon, 24 Mar 2025 13:14:24 +0000
Subject: [PATCH 3/3] replace tests with much faster integration tests
---
tests/functional/cylc-config/11-json-dump.t | 113 ---------------
tests/integration/scripts/test_config.py | 145 ++++++++++++++++++++
2 files changed, 145 insertions(+), 113 deletions(-)
delete mode 100644 tests/functional/cylc-config/11-json-dump.t
create mode 100644 tests/integration/scripts/test_config.py
diff --git a/tests/functional/cylc-config/11-json-dump.t b/tests/functional/cylc-config/11-json-dump.t
deleted file mode 100644
index 5e2af26aa33..00000000000
--- a/tests/functional/cylc-config/11-json-dump.t
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#-------------------------------------------------------------------------------
-# Test cylc config can dump json files.
-# n.b. not heavily tested because most of this functionality
-# is from Standard library json.
-. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
-set_test_number 9
-#-------------------------------------------------------------------------------
-
-# Test that option parser errors if incompat options given:
-cylc config --json --platforms 2> err
-named_grep_ok "${TEST_NAME_BASE}.CLI-one-incompat-item" \
- "--json incompatible with --platforms" \
- err
-
-cylc config --json --platforms --platform-names 2> err
-named_grep_ok "${TEST_NAME_BASE}.CLI-two-incompat-items" \
- "--json incompatible with --platform-names or --platforms" \
- err
-
-cylc config --json --platforms --platform-names --print-hierarchy 2> err
-named_grep_ok "${TEST_NAME_BASE}.CLI-three-incompat-items" \
- "--json incompatible with --print-hierarchy or --platform-names or --platforms" \
- err
-
-
-# Test the global.cylc
-TEST_NAME="${TEST_NAME_BASE}-global"
-
-cat > "global.cylc" <<__HEREDOC__
-[platforms]
- [[golders_green]]
- [[[meta]]]
- can = "Test lots of things"
- because = metadata, is, not, fussy
- number = 99
-__HEREDOC__
-
-export CYLC_CONF_PATH="${PWD}"
-run_ok "${TEST_NAME}" cylc config --json --one-line
-cmp_ok "${TEST_NAME}.stdout" <<__HERE__
-{"platforms": {"golders_green": {"meta": {"can": "Test lots of things", "because": "metadata, is, not, fussy", "number": "99"}}}}
-__HERE__
-
-# Test a flow.cylc
-TEST_NAME="${TEST_NAME_BASE}-workflow"
-
-cat > "flow.cylc" <<__HERE__
-[scheduling]
- [[graph]]
- P1D = foo
-
-[runtime]
- [[foo]]
-__HERE__
-
-run_ok "${TEST_NAME}" cylc config . --json --icp 1000
-cmp_ok "${TEST_NAME}.stdout" <<__HERE__
-{
- "scheduling": {
- "graph": {
- "P1D": "foo"
- },
- "initial cycle point": "1000"
- },
- "runtime": {
- "root": {},
- "foo": {
- "completion": "succeeded"
- }
- }
-}
-__HERE__
-
-# Test an empty global.cylc to check:
-# * item selection
-# * null value setting
-# * showing defaults
-TEST_NAME="${TEST_NAME_BASE}-defaults-item-null-value"
-echo "" > global.cylc
-export CYLC_CONF_PATH="${PWD}"
-
-run_ok "${TEST_NAME}" cylc config \
- -i '[scheduler][mail]' \
- --json \
- --defaults \
- --null-value='zilch'
-
-cmp_ok "${TEST_NAME}.stdout" <<__HERE__
-{
- "from": "zilch",
- "smtp": "zilch",
- "to": "zilch",
- "footer": "zilch",
- "task event batch interval": 300.0
-}
-__HERE__
diff --git a/tests/integration/scripts/test_config.py b/tests/integration/scripts/test_config.py
new file mode 100644
index 00000000000..746524b1898
--- /dev/null
+++ b/tests/integration/scripts/test_config.py
@@ -0,0 +1,145 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import json
+import pytest
+
+from cylc.flow.option_parsers import Options
+from cylc.flow.scripts.config import _main, get_option_parser
+
+
+@pytest.fixture(scope='module')
+def setup(mod_one_conf, mod_flow):
+ parser = get_option_parser()
+ ConfigOptions = Options(parser)
+ opts = ConfigOptions()
+ opts.json = True
+ wid = mod_flow(mod_one_conf)
+ yield parser, opts, wid
+
+
+async def test_json_basic(setup, capsys):
+ """Test that the output is in JSON format."""
+ await _main(*setup)
+
+ result = capsys.readouterr()
+ assert result.err == ''
+ assert json.loads(result.out)['scheduling']['graph'] == {
+ 'R1': 'one'
+ }
+
+
+async def test_json_workflow_cfg(flow, capsys):
+ """It fills in values from CLI."""
+ wid = flow(
+ {
+ 'scheduling': {'graph': {'P1D': 'foo'}},
+ 'runtime': {'foo': {}},
+ }
+ )
+ parser = get_option_parser()
+ ConfigOptions = Options(parser)
+ opts = ConfigOptions()
+ opts.json = True
+ opts.icp = ' '
+
+ await _main(parser, opts, wid)
+
+ returned_config = json.loads(capsys.readouterr().out)
+ assert returned_config['scheduling']['initial cycle point'] == '1000'
+ assert returned_config['runtime']['foo'] == {
+ 'completion': 'succeeded',
+ 'simulation': {'default run length': 0.0}
+ }
+
+
+@pytest.mark.parametrize(
+ 'not_with',
+ [
+ (['print_platforms']),
+ (['print_platforms', 'print_platform_names']),
+ (['print_platforms', 'print_platform_names', 'print_hierarchy']),
+ ],
+)
+async def test_json_and_not_other_option(
+ setup, capsys, not_with
+):
+ """It fails if incompatible options provided."""
+ parser, opts, wid = setup
+ for key in not_with:
+ setattr(opts, key, True)
+
+ with pytest.raises(SystemExit):
+ await _main(parser, opts, wid)
+
+ result = capsys.readouterr()
+ assert result.out == ''
+ assert '--json incompatible with' in result.err
+ for key in not_with:
+ if 'platform' in key:
+ key = key.strip('print_')
+ assert key.replace('_', '-') in result.err
+
+ # Clean up, since setup object is shared:
+ for key in not_with:
+ setattr(opts, key, False)
+
+
+async def test_json_global_cfg(setup, mock_glbl_cfg, capsys):
+ """It returns the global configuration in JSON format."""
+ mock_glbl_cfg(
+ 'cylc.flow.scripts.config.glbl_cfg',
+ '''
+ [platforms]
+ [[golders_green]]
+ [[[meta]]]
+ can = "Test lots of things"
+ because = metadata, is, not, fussy
+ number = 99
+ ''',
+ )
+ parser, opts, _ = setup
+
+ await _main(parser, opts)
+
+ returned_config = json.loads(capsys.readouterr().out)
+ assert returned_config == {
+ 'platforms': {
+ 'golders_green': {
+ 'meta': {
+ 'can': 'Test lots of things',
+ 'because': 'metadata, is, not, fussy',
+ 'number': '99',
+ }
+ }
+ }
+ }
+
+
+async def test_json_global_cfg_empty(setup, mock_glbl_cfg, capsys):
+ """It returns an empty global configuration in JSON format."""
+ parser, opts, _ = setup
+ mock_glbl_cfg('cylc.flow.scripts.config.glbl_cfg', '')
+ opts.item = ['scheduler][mail]']
+ opts.json = True
+ opts.defaults = True
+ opts.none_str = 'zilch'
+
+ await _main(parser, opts)
+
+ returned_config = json.loads(capsys.readouterr().out)
+ for key in ['footer', 'from', 'smtp', 'to']:
+ assert returned_config[key] == 'zilch'