From ac36722108bc169c3ade6bfe31ea62f9df8ef7c2 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 20 May 2024 19:33:56 +0530 Subject: [PATCH 01/78] initial commit --- ndcube/asdf/__init__.py | 0 ndcube/asdf/converters/ndcube_converter.py | 30 ++++++++++++++ ndcube/asdf/entry_points.py | 40 +++++++++++++++++++ .../resources/manifests/ndcube-1.0.0.yaml | 10 +++++ .../asdf/resources/schemas/NDCube-1.0.0.yaml | 21 ++++++++++ pyproject.toml | 6 +++ 6 files changed, 107 insertions(+) create mode 100644 ndcube/asdf/__init__.py create mode 100644 ndcube/asdf/converters/ndcube_converter.py create mode 100644 ndcube/asdf/entry_points.py create mode 100644 ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml create mode 100644 ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml diff --git a/ndcube/asdf/__init__.py b/ndcube/asdf/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py new file mode 100644 index 000000000..a6ff4be70 --- /dev/null +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -0,0 +1,30 @@ +from asdf.extension import Converter +import numpy as np + + +class NDCubeConverter(Converter): + tags = ["tag:sunpy.org:ndcube/ndcube/NDCube-*"] + + @property + def types(self): + from ndcube.ndcube import NDCube + return [NDCube] + + def select_tag(self, obj, tags, ctx): + # Sort the tags in reverse alphabetical order and pick the first (i.e. + # the one with the highest version). This assumes that all the tags for + # this converter are named the same other than the version number. + tags = list(sorted(tags, reverse=True)) + return tags[0] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube import ndcube + data = np.asanyarray(node["data"]) + wcs = node["wcs"] + return(ndcube.NDCube(data, wcs)) + + def to_yaml_tree(self, ndcube, tag, ctx): + node = {} + node["data"] = np.asarray(ndcube.data) + node["wcs"] = ndcube.wcs + return node diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py new file mode 100644 index 000000000..2dc63a334 --- /dev/null +++ b/ndcube/asdf/entry_points.py @@ -0,0 +1,40 @@ +""" +This file contains the entry points for asdf. +""" +import importlib.resources as importlib_resources +from asdf.extension import ManifestExtension +from asdf.resource import DirectoryResourceMapping + + +def get_resource_mappings(): + """ + Get the resource mapping instances for myschemas + and manifests. This method is registered with the + asdf.resource_mappings entry point. + + Returns + ------- + list of collections.abc.Mapping + """ + from ndcube.asdf import resources + resources_root = importlib_resources.files(resources) + return [ + DirectoryResourceMapping( + resources_root / "schemas", "asdf://sunpy.org/ndcube/schemas/"), + DirectoryResourceMapping( + resources_root / "manifests", "asdf://sunpy.org/ndcube/manifests/"), + ] + + +def get_extensions(): + """ + Get the list of extensions. + """ + from ndcube.asdf.converters.ndcube_converter import NDCubeConverter + + ndcube_converters = [NDCubeConverter()] + _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0" + + return [ + ManifestExtension.from_uri(_manifest_uri, converters=ndcube_converters) + ] diff --git a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml new file mode 100644 index 000000000..aab04090e --- /dev/null +++ b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml @@ -0,0 +1,10 @@ +%YAML 1.1 +--- +id: asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0 +extension_uri: asdf://sunpy.org/extensions/ndcube-1.0.0 +title: NDCube ASDF Manifest +description: ASDF schemas and tags for NDCube classes. + +tags: + - tag_uri: "tag:sunpy.org:ndcube/ndcube/NDCube-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/NDCube-1.0.0" \ No newline at end of file diff --git a/ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml b/ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml new file mode 100644 index 000000000..2d4862745 --- /dev/null +++ b/ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml @@ -0,0 +1,21 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/NDCube-1.0.0" + +title: | + Represents the ndcube.NDCube class object + +description: + Test ndcube class + +type: object +properties: + data: + tag: "tag:stsci.edu:asdf/core/ndarray-1.0.0" + wcs: + $ref: "http://stsci.edu/schemas/gwcs/wcs-1.1.0" + +required: [data , wcs] +allowAdditionalProperties: False +... \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index bec645c10..8756cc1fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,12 @@ exclude = ["ndcube._dev*"] [tool.setuptools_scm] write_to = "ndcube/_version.py" +[project.entry-points."asdf.resource_mappings"] +ndcube = "ndcube.asdf.entry_points:get_resource_mappings" + +[project.entry-points."asdf.extensions"] +ndcube = "ndcube.asdf.entry_points:get_extensions" + [tool.towncrier] package = "ndcube" filename = "CHANGELOG.rst" From 6be61e73e770be4c45ffc550596793e6b4fbd4bc Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 20 May 2024 21:04:24 +0530 Subject: [PATCH 02/78] version alignment --- ndcube/asdf/entry_points.py | 2 +- ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml | 10 ++++++++++ ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml | 10 ---------- .../schemas/{NDCube-1.0.0.yaml => NDCube-0.1.0.yaml} | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) create mode 100644 ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml delete mode 100644 ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml rename ndcube/asdf/resources/schemas/{NDCube-1.0.0.yaml => NDCube-0.1.0.yaml} (70%) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 2dc63a334..36126d25c 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -33,7 +33,7 @@ def get_extensions(): from ndcube.asdf.converters.ndcube_converter import NDCubeConverter ndcube_converters = [NDCubeConverter()] - _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0" + _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" return [ ManifestExtension.from_uri(_manifest_uri, converters=ndcube_converters) diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml new file mode 100644 index 000000000..1d45b4af7 --- /dev/null +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -0,0 +1,10 @@ +%YAML 1.1 +--- +id: asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0 +extension_uri: asdf://sunpy.org/extensions/ndcube-0.1.0 +title: NDCube ASDF Manifest +description: ASDF schemas and tags for NDCube classes. + +tags: + - tag_uri: "tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" \ No newline at end of file diff --git a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml deleted file mode 100644 index aab04090e..000000000 --- a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml +++ /dev/null @@ -1,10 +0,0 @@ -%YAML 1.1 ---- -id: asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0 -extension_uri: asdf://sunpy.org/extensions/ndcube-1.0.0 -title: NDCube ASDF Manifest -description: ASDF schemas and tags for NDCube classes. - -tags: - - tag_uri: "tag:sunpy.org:ndcube/ndcube/NDCube-1.0.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/NDCube-1.0.0" \ No newline at end of file diff --git a/ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml similarity index 70% rename from ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml rename to ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml index 2d4862745..5154938e1 100644 --- a/ndcube/asdf/resources/schemas/NDCube-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml @@ -1,13 +1,13 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/NDCube-1.0.0" +id: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" title: | - Represents the ndcube.NDCube class object + Represents the ndcube NDCube object description: - Test ndcube class + Represents the ndcube NDCube object type: object properties: From 6e619fc5c004952a942b007ec4dfa1641c68ac8a Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 23 May 2024 04:16:00 +0530 Subject: [PATCH 03/78] Remove method to remove any delay and --- ndcube/asdf/converters/ndcube_converter.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index a6ff4be70..f84f17603 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -4,18 +4,7 @@ class NDCubeConverter(Converter): tags = ["tag:sunpy.org:ndcube/ndcube/NDCube-*"] - - @property - def types(self): - from ndcube.ndcube import NDCube - return [NDCube] - - def select_tag(self, obj, tags, ctx): - # Sort the tags in reverse alphabetical order and pick the first (i.e. - # the one with the highest version). This assumes that all the tags for - # this converter are named the same other than the version number. - tags = list(sorted(tags, reverse=True)) - return tags[0] + types = ["ndcube.ndcube.NDCube"] def from_yaml_tree(self, node, tag, ctx): from ndcube import ndcube @@ -25,6 +14,6 @@ def from_yaml_tree(self, node, tag, ctx): def to_yaml_tree(self, ndcube, tag, ctx): node = {} - node["data"] = np.asarray(ndcube.data) + node["data"] =np.array(ndcube.data) node["wcs"] = ndcube.wcs return node From f505703b3886beee44996cc6d585e8f45ad495ad Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 23 May 2024 04:17:05 +0530 Subject: [PATCH 04/78] Update tag and --- ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml index 5154938e1..207890b0f 100644 --- a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml @@ -3,7 +3,7 @@ $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" id: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" -title: | +title: Represents the ndcube NDCube object description: @@ -12,9 +12,11 @@ description: type: object properties: data: - tag: "tag:stsci.edu:asdf/core/ndarray-1.0.0" + tag: "tag:stsci.edu:asdf/core/ndarray-1.*" wcs: - $ref: "http://stsci.edu/schemas/gwcs/wcs-1.1.0" + anyOf: + - $ref: "http://stsci.edu/schemas/gwcs/wcs-1.1.0" + - $ref: "http://stsci.edu/schemas/gwcs/wcs-1.0.0" required: [data , wcs] allowAdditionalProperties: False From af2f9c78ecf37b0fc2a1207434ac7f4d02bf2ab9 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 23 May 2024 04:18:24 +0530 Subject: [PATCH 05/78] enable schema test --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8756cc1fd..4e3750763 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,10 @@ exclude = ["ndcube._dev*"] [tool.setuptools_scm] write_to = "ndcube/_version.py" +[tool.pytest.ini_options] +asdf_schema_root = 'ndcube/asdf/resources/schemas' +asdf_schema_tests_enabled = true + [project.entry-points."asdf.resource_mappings"] ndcube = "ndcube.asdf.entry_points:get_resource_mappings" From adfc6b3f68b79f459cfb213790967061cf5d5374 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 23 May 2024 05:05:45 +0530 Subject: [PATCH 06/78] revert small change --- ndcube/asdf/converters/ndcube_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index f84f17603..86739dbae 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -14,6 +14,6 @@ def from_yaml_tree(self, node, tag, ctx): def to_yaml_tree(self, ndcube, tag, ctx): node = {} - node["data"] =np.array(ndcube.data) + node["data"] =np.asarray(ndcube.data) node["wcs"] = ndcube.wcs return node From 78eeb5325b1ae0183f785985ca8dbf8a1efc92bf Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 30 May 2024 21:54:55 +0530 Subject: [PATCH 07/78] Add the converters for ExtraCoords and TimeTableCoordinate class --- .../asdf/converters/extracoords_converter.py | 29 +++++++++++++++++++ ndcube/asdf/converters/ndcube_converter.py | 12 ++++++-- .../asdf/converters/tablecoord_converter.py | 29 +++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 ndcube/asdf/converters/extracoords_converter.py create mode 100644 ndcube/asdf/converters/tablecoord_converter.py diff --git a/ndcube/asdf/converters/extracoords_converter.py b/ndcube/asdf/converters/extracoords_converter.py new file mode 100644 index 000000000..1532efe2b --- /dev/null +++ b/ndcube/asdf/converters/extracoords_converter.py @@ -0,0 +1,29 @@ +from asdf.extension import Converter + + +class ExtraCoordsConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/extra_coords/ExtraCoords-*"] + types = ["ndcube.extra_coords.extra_coords.ExtraCoords"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.extra_coords import ExtraCoords + extra_coords = ExtraCoords() + extra_coords._wcs = node.get("wcs") + extra_coords._mapping = node.get("mapping") + if "lookup_tables" in node: + extra_coords._lookup_tables = node.get("lookup_tables") + extra_coords._dropped_tables = node.get("dropped_tables") + extra_coords._ndcube = node.get("ndcube") + return extra_coords + + def to_yaml_tree(self, extracoords, tag, ctx): + node = {} + if extracoords._wcs is not None: + node["wcs"] = extracoords._wcs + if extracoords._mapping is not None: + node["mapping"] = extracoords._mapping + if extracoords._lookup_tables: + node["lookup_tables"] = extracoords._lookup_tables + node["dropped_tables"] = extracoords._dropped_tables + node["ndcube"] = extracoords._ndcube + return node diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 86739dbae..1b2d7e8e4 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -7,13 +7,19 @@ class NDCubeConverter(Converter): types = ["ndcube.ndcube.NDCube"] def from_yaml_tree(self, node, tag, ctx): - from ndcube import ndcube + from ndcube.ndcube import NDCube + data = np.asanyarray(node["data"]) wcs = node["wcs"] - return(ndcube.NDCube(data, wcs)) + ndcube = NDCube(data, wcs) + ndcube._extra_coords = node["extra_coords"] + + return ndcube def to_yaml_tree(self, ndcube, tag, ctx): node = {} - node["data"] =np.asarray(ndcube.data) + node["data"] = np.asarray(ndcube.data) node["wcs"] = ndcube.wcs + node["extra_coords"] = ndcube.extra_coords + return node diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py new file mode 100644 index 000000000..728703dd6 --- /dev/null +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -0,0 +1,29 @@ +from asdf.extension import Converter + + +class TableCoordConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/TimeTableCoordinate-*"] + types = ["ndcube.extra_coords.table_coord.TimeTableCoordinate"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.table_coord import TimeTableCoordinate + + table = node.get("table") + names = node.get("names") + physical_types = node.get("physical_types") + reference_time = node.get("reference_time") + timetablecoordinate = TimeTableCoordinate( + table, names=names, physical_types=physical_types, reference_time=reference_time) + + return timetablecoordinate + + def to_yaml_tree(self, timetablecoordinate, tag, ctx): + node = {} + node["table"] = timetablecoordinate.table + node["names"] = timetablecoordinate.names + node["mesh"] = timetablecoordinate.mesh + if timetablecoordinate.physical_types is not None: + node["physical_types"] = timetablecoordinate.physical_types + node["reference_time"] = timetablecoordinate.reference_time + + return node From 0804f1d13609f216b0076342e9b57e67b44746e5 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 30 May 2024 21:59:22 +0530 Subject: [PATCH 08/78] Add ExtraCoords and TimeTableCoord Schema --- .../resources/schemas/ExtraCoords-0.1.0.yaml | 34 +++++++++++++++++++ .../asdf/resources/schemas/NDCube-0.1.0.yaml | 6 ++-- .../schemas/TimeTableCoordinate-0.1.0.yaml | 27 +++++++++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml create mode 100644 ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml diff --git a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml new file mode 100644 index 000000000..d71be90e8 --- /dev/null +++ b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml @@ -0,0 +1,34 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" + +title: + Represents the ndcube ExtraCoords object + +description: + Represents the ndcube ExtraCoords object + +type: object +properties: + wcs: + tag: "tag:stsci.edu:gwcs/wcs-1.*" + mapping: + type: array + lookup_tables: + type: array + items: + type: array + properties: + index: + type: array + table: + $ref: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" + dropped_tables: + type: array + ndcube: + $ref: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" + +required: [ndcube] +allowAdditionalProperties: False +... \ No newline at end of file diff --git a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml index 207890b0f..c4edd7210 100644 --- a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml @@ -14,9 +14,9 @@ properties: data: tag: "tag:stsci.edu:asdf/core/ndarray-1.*" wcs: - anyOf: - - $ref: "http://stsci.edu/schemas/gwcs/wcs-1.1.0" - - $ref: "http://stsci.edu/schemas/gwcs/wcs-1.0.0" + tag: "tag:stsci.edu:gwcs/wcs-1.*" + extra_coords: + $ref: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" required: [data , wcs] allowAdditionalProperties: False diff --git a/ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml new file mode 100644 index 000000000..b52b01783 --- /dev/null +++ b/ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml @@ -0,0 +1,27 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" + +title: + Represents the TimeTableCoords class + +description: + Represents the TimeTableCoords class + +type: object +properties: + table: + tag: "tag:stsci.edu:asdf/time/time-1*" + names: + type: array + mesh: + type: boolean + physical_types: + type: array + reference_time: + tag: "tag:stsci.edu:asdf/time/time-1*" + +required: ["table"] +additionalProperties: False +... From 0f4a86f5c5d08a0aafbbd00df3057e9f619dd44e Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 30 May 2024 22:01:58 +0530 Subject: [PATCH 09/78] Update manifest and entry_point.py --- ndcube/asdf/entry_points.py | 4 +++- ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 36126d25c..0398ba91e 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -31,8 +31,10 @@ def get_extensions(): Get the list of extensions. """ from ndcube.asdf.converters.ndcube_converter import NDCubeConverter + from ndcube.asdf.converters.extracoords_converter import ExtraCoordsConverter + from ndcube.asdf.converters.tablecoord_converter import TableCoordConverter - ndcube_converters = [NDCubeConverter()] + ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TableCoordConverter()] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" return [ diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index 1d45b4af7..1fa8638dd 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -7,4 +7,10 @@ description: ASDF schemas and tags for NDCube classes. tags: - tag_uri: "tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" \ No newline at end of file + schema_uri: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/extra_coords/ExtraCoords-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/TimeTableCoordinate-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" \ No newline at end of file From 86bfac931434c5b3df20eeaf08c1516d99cdf95a Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 1 Jun 2024 01:46:42 +0530 Subject: [PATCH 10/78] Add the QuantityTableCoordinate and SkyCoordTableCoordinate converters --- .../asdf/converters/tablecoord_converter.py | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index 728703dd6..2d292ab7f 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -1,7 +1,7 @@ from asdf.extension import Converter -class TableCoordConverter(Converter): +class TimeTableCoordConverter(Converter): tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/TimeTableCoordinate-*"] types = ["ndcube.extra_coords.table_coord.TimeTableCoordinate"] @@ -27,3 +27,59 @@ def to_yaml_tree(self, timetablecoordinate, tag, ctx): node["reference_time"] = timetablecoordinate.reference_time return node + + +class QuantityTableCoordinateConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/QuantityTableCoordinate-*"] + types = ["ndcube.extra_coords.table_coord.QuantityTableCoordinate"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.table_coord import QuantityTableCoordinate + + unit = node.get("unit") + table = node.get("table") + names = node.get("names") + mesh = node.get("mesh") + physical_types = node.get("physical_types") + quantitytablecoordinate = QuantityTableCoordinate(*table, + names=names, physical_types=physical_types,) + quantitytablecoordinate.unit = unit + quantitytablecoordinate.mesh = mesh + return quantitytablecoordinate + + def to_yaml_tree(self, quantitytablecoordinate, tag, ctx): + node = {} + node["unit"] = quantitytablecoordinate.unit + node["table"] = quantitytablecoordinate.table + node["names"] = quantitytablecoordinate.names + node["mesh"] = quantitytablecoordinate.mesh + if quantitytablecoordinate.physical_types is not None: + node["physical_types"] = quantitytablecoordinate.physical_types + + return node + + +class SkyCoordTableCoordinateConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/SkyCoordTableCoordinate-*"] + types = ["ndcube.extra_coords.table_coord.SkyCoordTableCoordinate"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.table_coord import SkyCoordTableCoordinate + + table = node.get("table") + names = node.get("names") + mesh = node.get("mesh") + physical_types = node.get("physical_types") + skycoordinatetablecoordinate = SkyCoordTableCoordinate(table, mesh=mesh, + names=names, physical_types=physical_types) + return skycoordinatetablecoordinate + + def to_yaml_tree(self, skycoordinatetablecoordinate, tag, ctx): + node = {} + node["table"] = skycoordinatetablecoordinate.table + node["names"] = skycoordinatetablecoordinate.names + node["mesh"] = skycoordinatetablecoordinate.mesh + if skycoordinatetablecoordinate.physical_types is not None: + node["physical_types"] = skycoordinatetablecoordinate.physical_types + + return node From da1c14eef746f04952aba95ce3c1a10686623ce7 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 1 Jun 2024 01:49:02 +0530 Subject: [PATCH 11/78] Add the schema for QuantityTableCoordinate and SkyCoordTableCoordinate --- .../QuantityTableCoordinate-0.1.0.yaml | 29 +++++++++++++++++++ .../SkyCoordTableCoordinate-0.1.0.yaml | 25 ++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml create mode 100644 ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml diff --git a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml new file mode 100644 index 000000000..28072f043 --- /dev/null +++ b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml @@ -0,0 +1,29 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" + +title: + Represents the QuantityTableCoords class + +description: + Represents the QuantityTableCoords class + +type: object +properties: + unit: + tag: "tag:stsci.edu:asdf/unit/unit-*" + table: + type: array + properties: + tag: "tag:stsci.edu:asdf/unit/quantity-*" + names: + type: array + mesh: + type: boolean + physical_types: + type: array + +required: ["table"] +additionalProperties: False +... diff --git a/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml new file mode 100644 index 000000000..5d9051918 --- /dev/null +++ b/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml @@ -0,0 +1,25 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" + +title: + Represents the SkyCoordTableCoordinate class + +description: + Represents the SkyCoordTableCoordinate class + +type: object +properties: + table: + tag: "tag:astropy.org:astropy/coordinates/skycoord-*" + names: + type: array + mesh: + type: boolean + physical_types: + type: array + +required: ["table"] +additionalProperties: False +... From dc24d40932551c79d7a9e13e511fe85cf7c1611c Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 1 Jun 2024 01:51:42 +0530 Subject: [PATCH 12/78] Update manifest and entry_point.py --- ndcube/asdf/entry_points.py | 6 ++++-- ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml | 8 +++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 0398ba91e..8b001db7b 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -32,9 +32,11 @@ def get_extensions(): """ from ndcube.asdf.converters.ndcube_converter import NDCubeConverter from ndcube.asdf.converters.extracoords_converter import ExtraCoordsConverter - from ndcube.asdf.converters.tablecoord_converter import TableCoordConverter + from ndcube.asdf.converters.tablecoord_converter import TimeTableCoordConverter + from ndcube.asdf.converters.tablecoord_converter import QuantityTableCoordinateConverter + from ndcube.asdf.converters.tablecoord_converter import SkyCoordTableCoordinateConverter - ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TableCoordConverter()] + ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TimeTableCoordConverter(),QuantityTableCoordinateConverter(),SkyCoordTableCoordinateConverter()] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" return [ diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index 1fa8638dd..66d893d74 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -13,4 +13,10 @@ tags: schema_uri: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/TimeTableCoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" \ No newline at end of file + schema_uri: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/QuantityTableCoordinate-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/SkyCoordTableCoordinate-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" \ No newline at end of file From ef208edbde4df10003bcb1def2a51dea158f7335 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 5 Jun 2024 03:28:52 +0530 Subject: [PATCH 13/78] Add validation for schema and manifests --- pytest.ini | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pytest.ini b/pytest.ini index 4fe506eef..763e86394 100644 --- a/pytest.ini +++ b/pytest.ini @@ -25,6 +25,11 @@ addopts = -p no:threadexception -m "not mpl_image_compare" --doctest-ignore-import-errors +asdf_schema_tests_enabled = true +asdf_schema_root = ndcube/asdf/resources +remote_data_strict = True +doctest_subpackage_requires = + docs/explaining_ndcube/* = numpy>=2.0.0 filterwarnings = # Turn all warnings into errors so they do not pass silently. error @@ -48,6 +53,3 @@ filterwarnings = ignore:Animating a NDCube does not support transposing the array. The world axes may not display as expected because the array will not be transposed:UserWarning # This is raised by the Windows and mac os build for visualization.rst ignore:FigureCanvasAgg is non-interactive, and thus cannot be shown:UserWarning -remote_data_strict = True -doctest_subpackage_requires = - docs/explaining_ndcube/* = numpy>=2.0.0 From c3d2606219f018d99513da8c94804b328e7f18e3 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 5 Jun 2024 12:02:41 +0100 Subject: [PATCH 14/78] pre-commit --- ndcube/asdf/converters/ndcube_converter.py | 3 ++- ndcube/asdf/entry_points.py | 11 +++++++---- ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml | 2 +- ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml | 2 +- ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml | 2 +- .../schemas/QuantityTableCoordinate-0.1.0.yaml | 2 +- .../schemas/SkyCoordTableCoordinate-0.1.0.yaml | 2 +- pyproject.toml | 2 +- 8 files changed, 15 insertions(+), 11 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 1b2d7e8e4..cc96557b9 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -1,6 +1,7 @@ -from asdf.extension import Converter import numpy as np +from asdf.extension import Converter + class NDCubeConverter(Converter): tags = ["tag:sunpy.org:ndcube/ndcube/NDCube-*"] diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 8b001db7b..0b51aae81 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -2,6 +2,7 @@ This file contains the entry points for asdf. """ import importlib.resources as importlib_resources + from asdf.extension import ManifestExtension from asdf.resource import DirectoryResourceMapping @@ -30,11 +31,13 @@ def get_extensions(): """ Get the list of extensions. """ - from ndcube.asdf.converters.ndcube_converter import NDCubeConverter from ndcube.asdf.converters.extracoords_converter import ExtraCoordsConverter - from ndcube.asdf.converters.tablecoord_converter import TimeTableCoordConverter - from ndcube.asdf.converters.tablecoord_converter import QuantityTableCoordinateConverter - from ndcube.asdf.converters.tablecoord_converter import SkyCoordTableCoordinateConverter + from ndcube.asdf.converters.ndcube_converter import NDCubeConverter + from ndcube.asdf.converters.tablecoord_converter import ( + QuantityTableCoordinateConverter, + SkyCoordTableCoordinateConverter, + TimeTableCoordConverter, + ) ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TimeTableCoordConverter(),QuantityTableCoordinateConverter(),SkyCoordTableCoordinateConverter()] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index 66d893d74..7bd5e9e53 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -19,4 +19,4 @@ tags: schema_uri: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/SkyCoordTableCoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" \ No newline at end of file + schema_uri: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" diff --git a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml index d71be90e8..e8f318504 100644 --- a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml @@ -31,4 +31,4 @@ properties: required: [ndcube] allowAdditionalProperties: False -... \ No newline at end of file +... diff --git a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml index c4edd7210..dcf4dac44 100644 --- a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml @@ -20,4 +20,4 @@ properties: required: [data , wcs] allowAdditionalProperties: False -... \ No newline at end of file +... diff --git a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml index 28072f043..b04a19246 100644 --- a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml @@ -11,7 +11,7 @@ description: type: object properties: - unit: + unit: tag: "tag:stsci.edu:asdf/unit/unit-*" table: type: array diff --git a/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml index 5d9051918..178eedb37 100644 --- a/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml @@ -17,7 +17,7 @@ properties: type: array mesh: type: boolean - physical_types: + physical_types: type: array required: ["table"] diff --git a/pyproject.toml b/pyproject.toml index 4e3750763..fa375b637 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ write_to = "ndcube/_version.py" [tool.pytest.ini_options] asdf_schema_root = 'ndcube/asdf/resources/schemas' asdf_schema_tests_enabled = true - + [project.entry-points."asdf.resource_mappings"] ndcube = "ndcube.asdf.entry_points:get_resource_mappings" From 86c2b6cf9ee46fccea41b7ec7a07804de35b6fc1 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 5 Jun 2024 21:37:06 +0530 Subject: [PATCH 15/78] Update the tox.ini and CI workflow --- .github/workflows/ci.yml | 1 + pyproject.toml | 4 ---- tox.ini | 11 +++++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49aa6d5b1..af9fba9ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,7 @@ jobs: - windows: py311 - macos: py310 - linux: py310-oldestdeps + - linux: schema secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/pyproject.toml b/pyproject.toml index fa375b637..8756cc1fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,10 +74,6 @@ exclude = ["ndcube._dev*"] [tool.setuptools_scm] write_to = "ndcube/_version.py" -[tool.pytest.ini_options] -asdf_schema_root = 'ndcube/asdf/resources/schemas' -asdf_schema_tests_enabled = true - [project.entry-points."asdf.resource_mappings"] ndcube = "ndcube.asdf.entry_points:get_resource_mappings" diff --git a/tox.ini b/tox.ini index 75af641a6..2268f5813 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = py310-oldestdeps codestyle build_docs + schema [testenv] # We use bash in some of our environments so we have to whitelist it. @@ -90,6 +91,16 @@ commands = {toxinidir}/docs \ {posargs} +[testenv:schema] +description = Run schema tests +deps = + pytest + asdf +set_env = + asdf_schema_root = {toxinidir}/ndcube/asdf/resources +commands= + pytest {env:asdf_schema_root} + [testenv:build_docs] change_dir = docs description = Invoke sphinx-build to build the HTML docs From 3a875ab2acb6ca368cc135cade6ebc877961d270 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 5 Jun 2024 21:51:36 +0530 Subject: [PATCH 16/78] Update schemas --- ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml | 2 +- .../asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml index e8f318504..d3ff106eb 100644 --- a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml @@ -27,7 +27,7 @@ properties: dropped_tables: type: array ndcube: - $ref: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" + tag: tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0 required: [ndcube] allowAdditionalProperties: False diff --git a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml index b04a19246..9a3e88a81 100644 --- a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml @@ -15,7 +15,7 @@ properties: tag: "tag:stsci.edu:asdf/unit/unit-*" table: type: array - properties: + items: tag: "tag:stsci.edu:asdf/unit/quantity-*" names: type: array From ce5a14bafcaace3845e0b68ed7b9a92ebfc8315b Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 17 Jun 2024 17:34:14 +0530 Subject: [PATCH 17/78] Add the converter and schema for GlobalCoords --- .../asdf/converters/globalcoords_converter.py | 28 +++++++++++++++++ .../resources/schemas/GlobalCoords-0.1.0.yaml | 30 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 ndcube/asdf/converters/globalcoords_converter.py create mode 100644 ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml diff --git a/ndcube/asdf/converters/globalcoords_converter.py b/ndcube/asdf/converters/globalcoords_converter.py new file mode 100644 index 000000000..221543c76 --- /dev/null +++ b/ndcube/asdf/converters/globalcoords_converter.py @@ -0,0 +1,28 @@ +from typing import OrderedDict + +from asdf.extension import Converter + + +class GlobalCoordsConverter(Converter): + tags = ["tag:sunpy.org:ndcube/global_coords/GlobalCoords-*"] + types = ["ndcube.global_coords.GlobalCoords"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.global_coords import GlobalCoords + + globalcoords = GlobalCoords() + if node.get("internal_coords") is not None: + globalcoords._internal_coords = OrderedDict(node.get("internal_coords")) + globalcoords._ndcube = node["ndcube"] + + return globalcoords + + def to_yaml_tree(self, globalcoords, tag, ctx): + node = {} + node["ndcube"] = globalcoords._ndcube + if globalcoords._internal_coords: + node["internal_coords"] = dict(globalcoords._internal_coords) + # Todo: Include `_all_coords` as a node key to preserve the dropped dimensions + # after ndcube support serialization of sliced NDCube object to asdf. + + return node diff --git a/ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml new file mode 100644 index 000000000..370be3bda --- /dev/null +++ b/ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml @@ -0,0 +1,30 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/GlobalCoords-0.1.0" + +title: + Represents the ndcube GlobalCoords object + +description: + Represents the ndcube GlobalCoords object + +type: object +properties: + internal_coords: + type: object + patternProperties: + .*: + type: array + items: + - type: string + - type: object + oneOf: + - tag: "tag:stsci.edu:asdf/unit/quantity-*" + - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" + ndcube: + tag: tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0 + +required: [ndcube] +allowAdditionalProperties: False +... From 9247b46f0df2eaa1e2d5538e1f5d9fd9175cc341 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 17 Jun 2024 17:37:27 +0530 Subject: [PATCH 18/78] Update converters --- ndcube/asdf/converters/extracoords_converter.py | 6 +++--- ndcube/asdf/converters/ndcube_converter.py | 2 ++ ndcube/asdf/converters/tablecoord_converter.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ndcube/asdf/converters/extracoords_converter.py b/ndcube/asdf/converters/extracoords_converter.py index 1532efe2b..79c99bba2 100644 --- a/ndcube/asdf/converters/extracoords_converter.py +++ b/ndcube/asdf/converters/extracoords_converter.py @@ -10,8 +10,7 @@ def from_yaml_tree(self, node, tag, ctx): extra_coords = ExtraCoords() extra_coords._wcs = node.get("wcs") extra_coords._mapping = node.get("mapping") - if "lookup_tables" in node: - extra_coords._lookup_tables = node.get("lookup_tables") + extra_coords._lookup_tables = node.get("lookup_tables") extra_coords._dropped_tables = node.get("dropped_tables") extra_coords._ndcube = node.get("ndcube") return extra_coords @@ -24,6 +23,7 @@ def to_yaml_tree(self, extracoords, tag, ctx): node["mapping"] = extracoords._mapping if extracoords._lookup_tables: node["lookup_tables"] = extracoords._lookup_tables - node["dropped_tables"] = extracoords._dropped_tables + if extracoords._dropped_tables: + node["dropped_tables"] = extracoords._dropped_tables node["ndcube"] = extracoords._ndcube return node diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index cc96557b9..2c116b0e5 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -14,6 +14,7 @@ def from_yaml_tree(self, node, tag, ctx): wcs = node["wcs"] ndcube = NDCube(data, wcs) ndcube._extra_coords = node["extra_coords"] + ndcube._global_coords = node["global_coords"] return ndcube @@ -22,5 +23,6 @@ def to_yaml_tree(self, ndcube, tag, ctx): node["data"] = np.asarray(ndcube.data) node["wcs"] = ndcube.wcs node["extra_coords"] = ndcube.extra_coords + node["global_coords"] = ndcube.global_coords return node diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index 2d292ab7f..3b3b4886a 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -42,7 +42,7 @@ def from_yaml_tree(self, node, tag, ctx): mesh = node.get("mesh") physical_types = node.get("physical_types") quantitytablecoordinate = QuantityTableCoordinate(*table, - names=names, physical_types=physical_types,) + names=names, physical_types=physical_types) quantitytablecoordinate.unit = unit quantitytablecoordinate.mesh = mesh return quantitytablecoordinate From c8a1bc4ed8e891749d9b578e3a2f3cfe9f946506 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 17 Jun 2024 17:45:13 +0530 Subject: [PATCH 19/78] Update entry_points,schemas and manifest --- ndcube/asdf/entry_points.py | 3 ++- ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml | 3 +++ .../asdf/resources/schemas/ExtraCoords-0.1.0.yaml | 15 +++++++++------ ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml | 2 ++ 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 0b51aae81..72b0ecf17 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -32,6 +32,7 @@ def get_extensions(): Get the list of extensions. """ from ndcube.asdf.converters.extracoords_converter import ExtraCoordsConverter + from ndcube.asdf.converters.globalcoords_converter import GlobalCoordsConverter from ndcube.asdf.converters.ndcube_converter import NDCubeConverter from ndcube.asdf.converters.tablecoord_converter import ( QuantityTableCoordinateConverter, @@ -39,7 +40,7 @@ def get_extensions(): TimeTableCoordConverter, ) - ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TimeTableCoordConverter(),QuantityTableCoordinateConverter(),SkyCoordTableCoordinateConverter()] + ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TimeTableCoordConverter(),QuantityTableCoordinateConverter(),SkyCoordTableCoordinateConverter(),GlobalCoordsConverter()] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" return [ diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index 7bd5e9e53..3d5ef1d19 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -20,3 +20,6 @@ tags: - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/SkyCoordTableCoordinate-0.1.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/global_coords/GlobalCoords-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/GlobalCoords-0.1.0" diff --git a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml index d3ff106eb..6acfa48aa 100644 --- a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml @@ -18,12 +18,15 @@ properties: lookup_tables: type: array items: - type: array - properties: - index: - type: array - table: - $ref: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" + type: array + items: + - oneOf: + - type: number + - type: array + - oneOf: + - $ref: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" + - $ref: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" + - $ref: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" dropped_tables: type: array ndcube: diff --git a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml index dcf4dac44..7a08d2395 100644 --- a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml @@ -17,6 +17,8 @@ properties: tag: "tag:stsci.edu:gwcs/wcs-1.*" extra_coords: $ref: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" + globa_coords: + $ref: "asdf://sunpy.org/ndcube/schemas/GlobalCoords-0.1.0" required: [data , wcs] allowAdditionalProperties: False From c681cb2c14dbbbcbdf2eec166773ea44ffaff277 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 19 Jun 2024 18:51:00 +0530 Subject: [PATCH 20/78] minor change --- ndcube/asdf/converters/globalcoords_converter.py | 2 -- ndcube/asdf/entry_points.py | 9 ++++++++- .../resources/schemas/QuantityTableCoordinate-0.1.0.yaml | 2 +- tox.ini | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ndcube/asdf/converters/globalcoords_converter.py b/ndcube/asdf/converters/globalcoords_converter.py index 221543c76..4f6a76d64 100644 --- a/ndcube/asdf/converters/globalcoords_converter.py +++ b/ndcube/asdf/converters/globalcoords_converter.py @@ -22,7 +22,5 @@ def to_yaml_tree(self, globalcoords, tag, ctx): node["ndcube"] = globalcoords._ndcube if globalcoords._internal_coords: node["internal_coords"] = dict(globalcoords._internal_coords) - # Todo: Include `_all_coords` as a node key to preserve the dropped dimensions - # after ndcube support serialization of sliced NDCube object to asdf. return node diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 72b0ecf17..55462842e 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -40,7 +40,14 @@ def get_extensions(): TimeTableCoordConverter, ) - ndcube_converters = [NDCubeConverter(),ExtraCoordsConverter(),TimeTableCoordConverter(),QuantityTableCoordinateConverter(),SkyCoordTableCoordinateConverter(),GlobalCoordsConverter()] + ndcube_converters = [ + NDCubeConverter(), + ExtraCoordsConverter(), + TimeTableCoordConverter(), + QuantityTableCoordinateConverter(), + SkyCoordTableCoordinateConverter(), + GlobalCoordsConverter(), + ] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" return [ diff --git a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml index 9a3e88a81..e24c7e08f 100644 --- a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml @@ -24,6 +24,6 @@ properties: physical_types: type: array -required: ["table"] +required: ["table","unit"] additionalProperties: False ... diff --git a/tox.ini b/tox.ini index 2268f5813..1c4743e71 100644 --- a/tox.ini +++ b/tox.ini @@ -98,7 +98,7 @@ deps = asdf set_env = asdf_schema_root = {toxinidir}/ndcube/asdf/resources -commands= +commands = pytest {env:asdf_schema_root} [testenv:build_docs] From 42c590e500abf76e537ec663d73309f1bda74e96 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 22 Jun 2024 00:27:29 +0530 Subject: [PATCH 21/78] lowercase schema URIs and use wildcards --- .../asdf/converters/extracoords_converter.py | 2 +- .../asdf/converters/globalcoords_converter.py | 2 +- ndcube/asdf/converters/ndcube_converter.py | 8 +++---- .../asdf/converters/tablecoord_converter.py | 6 ++--- .../resources/manifests/ndcube-0.1.0.yaml | 24 +++++++++---------- ...rds-0.1.0.yaml => extra_coords-0.1.0.yaml} | 12 +++++----- ...ds-0.1.0.yaml => global_coords-0.1.0.yaml} | 6 ++--- .../{NDCube-0.1.0.yaml => ndcube-0.1.0.yaml} | 10 ++++---- ...aml => quantitytablecoordinate-0.1.0.yaml} | 4 ++-- ...aml => skycoordtablecoordinate-0.1.0.yaml} | 2 +- ....0.yaml => timetablecoordinate-0.1.0.yaml} | 2 +- 11 files changed, 38 insertions(+), 40 deletions(-) rename ndcube/asdf/resources/schemas/{ExtraCoords-0.1.0.yaml => extra_coords-0.1.0.yaml} (55%) rename ndcube/asdf/resources/schemas/{GlobalCoords-0.1.0.yaml => global_coords-0.1.0.yaml} (79%) rename ndcube/asdf/resources/schemas/{NDCube-0.1.0.yaml => ndcube-0.1.0.yaml} (54%) rename ndcube/asdf/resources/schemas/{QuantityTableCoordinate-0.1.0.yaml => quantitytablecoordinate-0.1.0.yaml} (83%) rename ndcube/asdf/resources/schemas/{SkyCoordTableCoordinate-0.1.0.yaml => skycoordtablecoordinate-0.1.0.yaml} (86%) rename ndcube/asdf/resources/schemas/{TimeTableCoordinate-0.1.0.yaml => timetablecoordinate-0.1.0.yaml} (87%) diff --git a/ndcube/asdf/converters/extracoords_converter.py b/ndcube/asdf/converters/extracoords_converter.py index 79c99bba2..95d4d1cda 100644 --- a/ndcube/asdf/converters/extracoords_converter.py +++ b/ndcube/asdf/converters/extracoords_converter.py @@ -2,7 +2,7 @@ class ExtraCoordsConverter(Converter): - tags = ["tag:sunpy.org:ndcube/extra_coords/extra_coords/ExtraCoords-*"] + tags = ["tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-*"] types = ["ndcube.extra_coords.extra_coords.ExtraCoords"] def from_yaml_tree(self, node, tag, ctx): diff --git a/ndcube/asdf/converters/globalcoords_converter.py b/ndcube/asdf/converters/globalcoords_converter.py index 4f6a76d64..85f7bf340 100644 --- a/ndcube/asdf/converters/globalcoords_converter.py +++ b/ndcube/asdf/converters/globalcoords_converter.py @@ -4,7 +4,7 @@ class GlobalCoordsConverter(Converter): - tags = ["tag:sunpy.org:ndcube/global_coords/GlobalCoords-*"] + tags = ["tag:sunpy.org:ndcube/global_coords/globalcoords-*"] types = ["ndcube.global_coords.GlobalCoords"] def from_yaml_tree(self, node, tag, ctx): diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 2c116b0e5..c1bcc43b5 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -1,16 +1,14 @@ -import numpy as np - from asdf.extension import Converter class NDCubeConverter(Converter): - tags = ["tag:sunpy.org:ndcube/ndcube/NDCube-*"] + tags = ["tag:sunpy.org:ndcube/ndcube/ndcube-*"] types = ["ndcube.ndcube.NDCube"] def from_yaml_tree(self, node, tag, ctx): from ndcube.ndcube import NDCube - data = np.asanyarray(node["data"]) + data = node["data"] wcs = node["wcs"] ndcube = NDCube(data, wcs) ndcube._extra_coords = node["extra_coords"] @@ -20,7 +18,7 @@ def from_yaml_tree(self, node, tag, ctx): def to_yaml_tree(self, ndcube, tag, ctx): node = {} - node["data"] = np.asarray(ndcube.data) + node["data"] = ndcube.data node["wcs"] = ndcube.wcs node["extra_coords"] = ndcube.extra_coords node["global_coords"] = ndcube.global_coords diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index 3b3b4886a..70daccd4b 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -2,7 +2,7 @@ class TimeTableCoordConverter(Converter): - tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/TimeTableCoordinate-*"] + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-*"] types = ["ndcube.extra_coords.table_coord.TimeTableCoordinate"] def from_yaml_tree(self, node, tag, ctx): @@ -30,7 +30,7 @@ def to_yaml_tree(self, timetablecoordinate, tag, ctx): class QuantityTableCoordinateConverter(Converter): - tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/QuantityTableCoordinate-*"] + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-*"] types = ["ndcube.extra_coords.table_coord.QuantityTableCoordinate"] def from_yaml_tree(self, node, tag, ctx): @@ -60,7 +60,7 @@ def to_yaml_tree(self, quantitytablecoordinate, tag, ctx): class SkyCoordTableCoordinateConverter(Converter): - tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/SkyCoordTableCoordinate-*"] + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-*"] types = ["ndcube.extra_coords.table_coord.SkyCoordTableCoordinate"] def from_yaml_tree(self, node, tag, ctx): diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index 3d5ef1d19..4559a5499 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -6,20 +6,20 @@ title: NDCube ASDF Manifest description: ASDF schemas and tags for NDCube classes. tags: - - tag_uri: "tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcube-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcube-0.1.0" - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/extra_coords/ExtraCoords-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/extra_coords-0.1.0" - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/TimeTableCoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/timetablecoordinate-0.1.0" - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/QuantityTableCoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/quantitytablecoordinate-0.1.0" - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/SkyCoordTableCoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/skycoordtablecoordinate-0.1.0" - - tag_uri: "tag:sunpy.org:ndcube/global_coords/GlobalCoords-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/GlobalCoords-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/global_coords-0.1.0" diff --git a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/extra_coords-0.1.0.yaml similarity index 55% rename from ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml rename to ndcube/asdf/resources/schemas/extra_coords-0.1.0.yaml index 6acfa48aa..6b16e3c9c 100644 --- a/ndcube/asdf/resources/schemas/ExtraCoords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/extra_coords-0.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/extra_coords-0.1.0" title: Represents the ndcube ExtraCoords object @@ -24,14 +24,14 @@ properties: - type: number - type: array - oneOf: - - $ref: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" - - $ref: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" - - $ref: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-0.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-0.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-0.*" dropped_tables: type: array ndcube: - tag: tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0 + tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" required: [ndcube] -allowAdditionalProperties: False +additionalProperties: false ... diff --git a/ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml b/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml similarity index 79% rename from ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml rename to ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml index 370be3bda..3f1068096 100644 --- a/ndcube/asdf/resources/schemas/GlobalCoords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/GlobalCoords-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/global_coords-0.1.0" title: Represents the ndcube GlobalCoords object @@ -23,8 +23,8 @@ properties: - tag: "tag:stsci.edu:asdf/unit/quantity-*" - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" ndcube: - tag: tag:sunpy.org:ndcube/ndcube/NDCube-0.1.0 + tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" required: [ndcube] -allowAdditionalProperties: False +additionalProperties: false ... diff --git a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml similarity index 54% rename from ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml rename to ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml index 7a08d2395..14bb43ddf 100644 --- a/ndcube/asdf/resources/schemas/NDCube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml @@ -12,14 +12,14 @@ description: type: object properties: data: - tag: "tag:stsci.edu:asdf/core/ndarray-1.*" + description: "Must be compatible with ASDF serialization/deserialization and supported by NDCube." wcs: tag: "tag:stsci.edu:gwcs/wcs-1.*" extra_coords: - $ref: "asdf://sunpy.org/ndcube/schemas/ExtraCoords-0.1.0" - globa_coords: - $ref: "asdf://sunpy.org/ndcube/schemas/GlobalCoords-0.1.0" + tag: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-0.*" + global_coords: + tag: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.*" required: [data , wcs] -allowAdditionalProperties: False +additionalProperties: true ... diff --git a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml similarity index 83% rename from ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml rename to ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml index e24c7e08f..1d31476fb 100644 --- a/ndcube/asdf/resources/schemas/QuantityTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/QuantityTableCoordinate-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/quantitytablecoordinate-0.1.0" title: Represents the QuantityTableCoords class @@ -24,6 +24,6 @@ properties: physical_types: type: array -required: ["table","unit"] +required: ["table", "unit"] additionalProperties: False ... diff --git a/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/skycoordtablecoordinate-0.1.0.yaml similarity index 86% rename from ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml rename to ndcube/asdf/resources/schemas/skycoordtablecoordinate-0.1.0.yaml index 178eedb37..97f22843a 100644 --- a/ndcube/asdf/resources/schemas/SkyCoordTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/skycoordtablecoordinate-0.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/SkyCoordTableCoordinate-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/skycoordtablecoordinate-0.1.0" title: Represents the SkyCoordTableCoordinate class diff --git a/ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml similarity index 87% rename from ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml rename to ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml index b52b01783..d57cd6a00 100644 --- a/ndcube/asdf/resources/schemas/TimeTableCoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/TimeTableCoordinate-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/timetablecoordinate-0.1.0" title: Represents the TimeTableCoords class From 7f3a81b789dfe6290a3930ba2245c61259d33cb4 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Tue, 2 Jul 2024 02:22:04 +0530 Subject: [PATCH 22/78] Add tests and GWCS objects --- .../converters/tests/test_ndcube_converter.py | 43 +++++ ndcube/conftest.py | 175 ++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 ndcube/asdf/converters/tests/test_ndcube_converter.py diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py new file mode 100644 index 000000000..a3a5df2ed --- /dev/null +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -0,0 +1,43 @@ +import numpy as np +import pytest + +import asdf + +from ndcube.tests.helpers import assert_cubes_equal + + +@pytest.mark.parametrize("ndc",[("ndcube_gwcs_2d_ln_lt"), + ("ndcube_gwcs_3d_ln_lt_l"), + ("ndcube_gwcs_3d_ln_lt_l_ec_gc"), + ("ndcube_gwcs_3d_rotated"), + ("ndcube_gwcs_4d_ln_lt_l_t"), + ], indirect=("ndc",)) +def test_serialization(ndc, tmp_path): + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = ndc + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_cubes_equal(af["ndcube_gwcs"], ndc) + +@pytest.mark.xfail(reason="Serialization of sliced ndcube not supported") +def test_serialization_sliced_ndcube(ndcube_gwcs_3d_ln_lt_l, tmp_path): + sndc = ndcube_gwcs_3d_ln_lt_l[np.s_[0, :, :]] + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = sndc + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_cubes_equal(af["ndcube_gwcs"], sndc) + +@pytest.mark.xfail(reason="Serialization of ndcube with .wcs attribute as astropy.wcs.wcs.WCS not supported") +def test_serialization_ndcube_wcs(ndcube_3d_ln_lt_l, tmp_path): + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube"] = ndcube_3d_ln_lt_l + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_cubes_equal(af["ndcube"], ndcube_3d_ln_lt_l) diff --git a/ndcube/conftest.py b/ndcube/conftest.py index 130941e6a..38c98a227 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -7,10 +7,14 @@ import dask.array import numpy as np import pytest +from gwcs import coordinate_frames as cf +from gwcs import wcs import astropy.nddata import astropy.units as u +from astropy import coordinates as coord from astropy.coordinates import SkyCoord +from astropy.modeling import models from astropy.nddata import StdDevUncertainty from astropy.time import Time, TimeDelta from astropy.wcs import WCS @@ -81,6 +85,136 @@ def gen_ndcube_3d_l_ln_lt_ectime(wcs_3d_lt_ln_l, time_axis, time_base, global_co # WCS Fixtures ################################################################################ +@pytest.fixture +def gwcs_4d_t_l_lt_ln(): + """ + Creates a 4D GWCS object with time, wavelength, and celestial coordinates. + + - Time: Axis 0 + - Wavelength: Axis 1 + - Sky: Axes 2 and 3 + + Returns: + wcs.WCS: 4D GWCS object. + """ + + time_model = models.Identity(1) + time_frame = cf.TemporalFrame(axes_order=(0, ), unit=u.s, + reference_frame=Time("2000-01-01T00:00:00")) + + wave_frame = cf.SpectralFrame(axes_order=(1, ), unit=u.m, axes_names=('wavelength',)) + wave_model = models.Scale(0.2) + + shift = models.Shift(-5) & models.Shift(0) + scale = models.Scale(5) & models.Scale(20) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial(0, 0, 180) + cel_model = shift | scale | tan | celestial_rotation + sky_frame = cf.CelestialFrame(axes_order=(2, 3), name='icrs', + reference_frame=coord.ICRS(), + axes_names=("longitude", "latitude")) + + transform = time_model & wave_model & cel_model + + frame = cf.CompositeFrame([time_frame, wave_frame, sky_frame]) + detector_frame = cf.CoordinateFrame(name="detector", naxes=4, + axes_order=(0, 1, 2, 3), + axes_type=("pixel", "pixel", "pixel", "pixel"), + unit=(u.pix, u.pix, u.pix, u.pix)) + + return (wcs.WCS(forward_transform=transform, output_frame=frame, input_frame=detector_frame)) + +@pytest.fixture +def gwcs_3d_lt_ln_l(): + """ + Creates a 3D GWCS object with celestial coordinates and wavelength. + + - Sky: Axes 0 and 1 + - Wavelength: Axis 2 + + Returns: + wcs.WCS: 3D GWCS object. + """ + + shift = models.Shift(-5) & models.Identity(1) + scale = models.Scale(5) & models.Scale(10) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial(0, 0, 180) + cel_model = shift | scale | tan | celestial_rotation + sky_frame = cf.CelestialFrame(axes_order=(0, 1), name='icrs', + reference_frame=coord.ICRS(), + axes_names=("longitude", "latitude")) + + wave_model = models.Identity(1) | models.Scale(0.2) | models.Shift(10) + wave_frame = cf.SpectralFrame(axes_order=(2, ), unit=u.nm, axes_names=("wavelength",)) + + transform = cel_model & wave_model + + frame = cf.CompositeFrame([sky_frame, wave_frame]) + detector_frame = cf.CoordinateFrame(name="detector", naxes=3, + axes_order=(0, 1, 2), + axes_type=("pixel", "pixel", "pixel"), + axes_names=("x", "y", "z"), unit=(u.pix, u.pix, u.pix)) + + return (wcs.WCS(forward_transform=transform, output_frame=frame, input_frame=detector_frame)) + +@pytest.fixture +def gwcs_3d_ln_lt_t_rotated(): + """ + Creates a 3D GWCS object with celestial coordinates and wavelength, including rotation. + + - Sky: Axes 0 and 1 + - Wavelength: Axis 2 + + Returns: + wcs.WCS: 3D GWCS object with rotation. + """ + shift = models.Shift(-5) & models.Identity(1) + scale = models.Scale(5) & models.Scale(10) + matrix = np.array([[1.290551569736E-05, 5.9525007864732E-06], + [5.0226382102765E-06 , -1.2644844123757E-05]]) + rotation = models.AffineTransformation2D(matrix) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial(0, 0, 180) + cel_model = shift | scale| rotation | tan | celestial_rotation + sky_frame = cf.CelestialFrame(axes_order=(0, 1), name='icrs', + reference_frame=coord.ICRS(), + axes_names=("longitude", "latitude")) + + wave_model = models.Identity(1) | models.Scale(0.2) | models.Shift(10) + wave_frame = cf.SpectralFrame(axes_order=(2, ), unit=u.nm, axes_names=("wavelength",)) + + transform = cel_model & wave_model + + frame = cf.CompositeFrame([sky_frame, wave_frame]) + detector_frame = cf.CoordinateFrame(name="detector", naxes=3, + axes_order=(0, 1, 2), + axes_type=("pixel", "pixel", "pixel"), + axes_names=("x", "y", "z"), unit=(u.pix, u.pix, u.pix)) + + return (wcs.WCS(forward_transform=transform, output_frame=frame, input_frame=detector_frame)) + +@pytest.fixture +def gwcs_2d_lt_ln(): + """ + Creates a 2D GWCS object with celestial coordinates. + + - Sky: Axes 0 and 1 + + Returns: + wcs.WCS: 2D GWCS object. + """ + shift = models.Shift(-5) & models.Shift(-5) + scale = models.Scale(2) & models.Scale(4) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial(0, 0, 180) + cel_model = shift | scale | tan | celestial_rotation + input_frame = cf.Frame2D(name="detector", axes_names=("x", "y")) + sky_frame = cf.CelestialFrame(axes_order=(0, 1), name='icrs', + reference_frame=coord.ICRS(), + axes_names=("longitude", "latitude")) + + return (wcs.WCS(forward_transform=cel_model, output_frame=sky_frame, input_frame=input_frame)) @pytest.fixture def wcs_4d_t_l_lt_ln(): @@ -318,6 +452,47 @@ def extra_coords_sharing_axis(): # NDCube Fixtures ################################################################################ +@pytest.fixture +def ndcube_gwcs_4d_ln_lt_l_t(gwcs_4d_t_l_lt_ln): + shape = (5, 8, 10, 12) + gwcs_4d_t_l_lt_ln.array_shape = shape + data_cube = data_nd(shape) + return NDCube(data_cube, wcs=gwcs_4d_t_l_lt_ln) + +@pytest.fixture +def ndcube_gwcs_3d_ln_lt_l(gwcs_3d_lt_ln_l): + shape = (2, 3, 4) + gwcs_3d_lt_ln_l.array_shape = shape + data_cube = data_nd(shape) + return NDCube(data_cube, wcs=gwcs_3d_lt_ln_l) + +@pytest.fixture +def ndcube_gwcs_3d_rotated(gwcs_3d_lt_ln_l, simple_extra_coords_3d): + data_rotated = np.array([[[1, 2, 3, 4, 6], [2, 4, 5, 3, 1], [0, -1, 2, 4, 2], [3, 5, 1, 2, 0]], + [[2, 4, 5, 1, 3], [1, 5, 2, 2, 4], [2, 3, 4, 0, 5], [0, 1, 2, 3, 4]]]) + cube = NDCube( + data_rotated, + wcs=gwcs_3d_lt_ln_l) + cube._extra_coords = simple_extra_coords_3d + return cube + +@pytest.fixture +def ndcube_gwcs_3d_ln_lt_l_ec_gc(gwcs_3d_lt_ln_l, simple_extra_coords_3d): + shape = (2, 3, 4) + gwcs_3d_lt_ln_l.array_shape = shape + data_cube = data_nd(shape) + cube = NDCube(data_cube, wcs=gwcs_3d_lt_ln_l) + coord1 = 1 * u.m + cube.global_coords.add('name1', 'custom:physical_type1', coord1) + cube._extra_coords = simple_extra_coords_3d + return cube + +@pytest.fixture +def ndcube_gwcs_2d_ln_lt(gwcs_2d_lt_ln): + shape = (10, 12) + data_cube = data_nd(shape) + return NDCube(data_cube, wcs=gwcs_2d_lt_ln) + @pytest.fixture def ndcube_4d_ln_l_t_lt(wcs_4d_lt_t_l_ln): shape = (5, 10, 12, 8) From 76229509755731baf3116d31105bb221c266bb58 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 6 Jul 2024 23:53:34 +0530 Subject: [PATCH 23/78] apply suggestions from code review --- .../asdf/converters/extracoords_converter.py | 5 ++--- .../asdf/converters/globalcoords_converter.py | 8 +++----- ndcube/asdf/converters/ndcube_converter.py | 4 +--- ndcube/asdf/converters/tablecoord_converter.py | 12 ++++-------- .../converters/tests/test_ndcube_converter.py | 5 +++-- .../resources/schemas/global_coords-0.1.0.yaml | 17 ++++++++--------- .../asdf/resources/schemas/ndcube-0.1.0.yaml | 2 +- ndcube/conftest.py | 18 ++++++++++++++++-- 8 files changed, 38 insertions(+), 33 deletions(-) diff --git a/ndcube/asdf/converters/extracoords_converter.py b/ndcube/asdf/converters/extracoords_converter.py index 95d4d1cda..c94795943 100644 --- a/ndcube/asdf/converters/extracoords_converter.py +++ b/ndcube/asdf/converters/extracoords_converter.py @@ -10,7 +10,7 @@ def from_yaml_tree(self, node, tag, ctx): extra_coords = ExtraCoords() extra_coords._wcs = node.get("wcs") extra_coords._mapping = node.get("mapping") - extra_coords._lookup_tables = node.get("lookup_tables") + extra_coords._lookup_tables = node.get("lookup_tables", []) extra_coords._dropped_tables = node.get("dropped_tables") extra_coords._ndcube = node.get("ndcube") return extra_coords @@ -23,7 +23,6 @@ def to_yaml_tree(self, extracoords, tag, ctx): node["mapping"] = extracoords._mapping if extracoords._lookup_tables: node["lookup_tables"] = extracoords._lookup_tables - if extracoords._dropped_tables: - node["dropped_tables"] = extracoords._dropped_tables + node["dropped_tables"] = extracoords._dropped_tables node["ndcube"] = extracoords._ndcube return node diff --git a/ndcube/asdf/converters/globalcoords_converter.py b/ndcube/asdf/converters/globalcoords_converter.py index 85f7bf340..9700a5890 100644 --- a/ndcube/asdf/converters/globalcoords_converter.py +++ b/ndcube/asdf/converters/globalcoords_converter.py @@ -1,5 +1,3 @@ -from typing import OrderedDict - from asdf.extension import Converter @@ -11,8 +9,8 @@ def from_yaml_tree(self, node, tag, ctx): from ndcube.global_coords import GlobalCoords globalcoords = GlobalCoords() - if node.get("internal_coords") is not None: - globalcoords._internal_coords = OrderedDict(node.get("internal_coords")) + if "internal_coords" in node: + globalcoords._internal_coords = node["internal_coords"] globalcoords._ndcube = node["ndcube"] return globalcoords @@ -21,6 +19,6 @@ def to_yaml_tree(self, globalcoords, tag, ctx): node = {} node["ndcube"] = globalcoords._ndcube if globalcoords._internal_coords: - node["internal_coords"] = dict(globalcoords._internal_coords) + node["internal_coords"] = globalcoords._internal_coords return node diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index c1bcc43b5..18003d928 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -8,9 +8,7 @@ class NDCubeConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.ndcube import NDCube - data = node["data"] - wcs = node["wcs"] - ndcube = NDCube(data, wcs) + ndcube = NDCube(node["data"], node["wcs"]) ndcube._extra_coords = node["extra_coords"] ndcube._global_coords = node["global_coords"] diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index 70daccd4b..2c4305001 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -8,12 +8,11 @@ class TimeTableCoordConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.extra_coords.table_coord import TimeTableCoordinate - table = node.get("table") names = node.get("names") physical_types = node.get("physical_types") reference_time = node.get("reference_time") timetablecoordinate = TimeTableCoordinate( - table, names=names, physical_types=physical_types, reference_time=reference_time) + node["table"], names=names, physical_types=physical_types, reference_time=reference_time) return timetablecoordinate @@ -36,14 +35,12 @@ class QuantityTableCoordinateConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.extra_coords.table_coord import QuantityTableCoordinate - unit = node.get("unit") - table = node.get("table") names = node.get("names") mesh = node.get("mesh") physical_types = node.get("physical_types") - quantitytablecoordinate = QuantityTableCoordinate(*table, + quantitytablecoordinate = QuantityTableCoordinate(*node["table"], names=names, physical_types=physical_types) - quantitytablecoordinate.unit = unit + quantitytablecoordinate.unit = node["unit"] quantitytablecoordinate.mesh = mesh return quantitytablecoordinate @@ -66,11 +63,10 @@ class SkyCoordTableCoordinateConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.extra_coords.table_coord import SkyCoordTableCoordinate - table = node.get("table") names = node.get("names") mesh = node.get("mesh") physical_types = node.get("physical_types") - skycoordinatetablecoordinate = SkyCoordTableCoordinate(table, mesh=mesh, + skycoordinatetablecoordinate = SkyCoordTableCoordinate(node["table"], mesh=mesh, names=names, physical_types=physical_types) return skycoordinatetablecoordinate diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index a3a5df2ed..b327561b7 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -8,9 +8,10 @@ @pytest.mark.parametrize("ndc",[("ndcube_gwcs_2d_ln_lt"), ("ndcube_gwcs_3d_ln_lt_l"), - ("ndcube_gwcs_3d_ln_lt_l_ec_gc"), + ("ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim"), + ("ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc"), ("ndcube_gwcs_3d_rotated"), - ("ndcube_gwcs_4d_ln_lt_l_t"), + ("ndcube_gwcs_4d_ln_lt_l_t") ], indirect=("ndc",)) def test_serialization(ndc, tmp_path): file_path = tmp_path / "test.asdf" diff --git a/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml b/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml index 3f1068096..09657fede 100644 --- a/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml @@ -13,15 +13,14 @@ type: object properties: internal_coords: type: object - patternProperties: - .*: - type: array - items: - - type: string - - type: object - oneOf: - - tag: "tag:stsci.edu:asdf/unit/quantity-*" - - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" + additionalProperties: + type: array + items: + - type: string + - type: object + oneOf: + - tag: "tag:stsci.edu:asdf/unit/quantity-*" + - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" ndcube: tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" diff --git a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml index 14bb43ddf..8b4270be6 100644 --- a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml @@ -20,6 +20,6 @@ properties: global_coords: tag: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.*" -required: [data , wcs] +required: [data, wcs] additionalProperties: true ... diff --git a/ndcube/conftest.py b/ndcube/conftest.py index 38c98a227..56b8a0165 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -39,6 +39,10 @@ # Helper Functions ################################################################################ +def time_lut(shape): + base_time = Time('2000-01-01', format='fits', scale='utc') + timestamps = Time([base_time + TimeDelta(60 * i, format='sec') for i in range(shape[0])]) + return timestamps def skycoord_2d_lut(shape): total_len = np.prod(shape) @@ -477,14 +481,24 @@ def ndcube_gwcs_3d_rotated(gwcs_3d_lt_ln_l, simple_extra_coords_3d): return cube @pytest.fixture -def ndcube_gwcs_3d_ln_lt_l_ec_gc(gwcs_3d_lt_ln_l, simple_extra_coords_3d): +def ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim(gwcs_3d_lt_ln_l, time_and_simple_extra_coords_2d): shape = (2, 3, 4) gwcs_3d_lt_ln_l.array_shape = shape data_cube = data_nd(shape) cube = NDCube(data_cube, wcs=gwcs_3d_lt_ln_l) + cube._extra_coords = time_and_simple_extra_coords_2d[0] + return cube + +@pytest.fixture +def ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc(gwcs_3d_lt_ln_l): + shape = (3, 3, 4) + gwcs_3d_lt_ln_l.array_shape = shape + data_cube = data_nd(shape) + cube = NDCube(data_cube, wcs=gwcs_3d_lt_ln_l) coord1 = 1 * u.m cube.global_coords.add('name1', 'custom:physical_type1', coord1) - cube._extra_coords = simple_extra_coords_3d + cube.extra_coords.add("time", 0, time_lut(shape)) + cube.extra_coords.add("exposure_lut", 1, range(shape[1]) * u.s) return cube @pytest.fixture From f89e1953a308be5e3d73a4a27a5462fbc9531392 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 11 Jul 2024 13:08:57 +0530 Subject: [PATCH 24/78] Remove mesh as a property validator --- ndcube/asdf/converters/tablecoord_converter.py | 1 - ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml | 2 -- 2 files changed, 3 deletions(-) diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index 2c4305001..b07e630e6 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -20,7 +20,6 @@ def to_yaml_tree(self, timetablecoordinate, tag, ctx): node = {} node["table"] = timetablecoordinate.table node["names"] = timetablecoordinate.names - node["mesh"] = timetablecoordinate.mesh if timetablecoordinate.physical_types is not None: node["physical_types"] = timetablecoordinate.physical_types node["reference_time"] = timetablecoordinate.reference_time diff --git a/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml index d57cd6a00..3c5a828ce 100644 --- a/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml @@ -15,8 +15,6 @@ properties: tag: "tag:stsci.edu:asdf/time/time-1*" names: type: array - mesh: - type: boolean physical_types: type: array reference_time: From 8b519ae051c42d3f6a07420e090b586e46a02657 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Fri, 12 Jul 2024 00:18:59 +0530 Subject: [PATCH 25/78] Update the dependencies version --- pyproject.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8756cc1fd..9fcd7d50f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,9 +16,10 @@ authors = [ { name = "The SunPy Community", email = "sunpy@googlegroups.com" }, ] dependencies = [ - "astropy>=5.0.6,!=5.1.0", - "gwcs>=0.18", - "numpy>=1.23.0" + "astropy>=5.3", + "gwcs>=0.20", + "numpy>=1.23.0", + "asdf>=2.14.4" ] dynamic = ["version"] From 37bf5633bf4d3cf0e7f4baa22cf18eebd723a3b8 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 20 Jul 2024 00:51:06 +0530 Subject: [PATCH 26/78] Style changes and add warnings to NDCube converter --- ndcube/asdf/converters/ndcube_converter.py | 26 +++++++++++++++++++ .../converters/tests/test_ndcube_converter.py | 2 ++ .../quantitytablecoordinate-0.1.0.yaml | 4 ++- tox.ini | 4 +-- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 18003d928..355e54ce5 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -1,3 +1,5 @@ +import warnings + from asdf.extension import Converter @@ -15,10 +17,34 @@ def from_yaml_tree(self, node, tag, ctx): return ndcube def to_yaml_tree(self, ndcube, tag, ctx): + """ + Notes + ----- + This methods serializes the primary components of the NDCube object, + including the `data`, `wcs`, `extra_coords`, and `global_coords` attributes. + Issues a warning if unsupported attributes (uncertainty, mask, meta, unit) are present, + which are not currently serialized to ASDF. + + Warnings + -------- + UserWarning + Warns if the NDCube object has attributes 'uncertainty', 'mask', 'meta', + or 'unit' that are present but not being saved in the ASDF serialization. + This ensures that users are aware of potentially important information + that is not included in the serialized output. + """ node = {} node["data"] = ndcube.data node["wcs"] = ndcube.wcs node["extra_coords"] = ndcube.extra_coords node["global_coords"] = ndcube.global_coords + attributes = ['uncertainty', 'mask', 'unit'] + for attr in attributes: + if getattr(ndcube, attr) is not None: + warnings.warn(f"Attribute '{attr}' is present but not being saved in ASDF serialization.", UserWarning) + + if len(ndcube.meta) > 0: + warnings.warn("Attribute 'meta' is present but not being saved in ASDF serialization.", UserWarning) + return node diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index b327561b7..5104193d3 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -22,6 +22,7 @@ def test_serialization(ndc, tmp_path): with asdf.open(file_path) as af: assert_cubes_equal(af["ndcube_gwcs"], ndc) + @pytest.mark.xfail(reason="Serialization of sliced ndcube not supported") def test_serialization_sliced_ndcube(ndcube_gwcs_3d_ln_lt_l, tmp_path): sndc = ndcube_gwcs_3d_ln_lt_l[np.s_[0, :, :]] @@ -33,6 +34,7 @@ def test_serialization_sliced_ndcube(ndcube_gwcs_3d_ln_lt_l, tmp_path): with asdf.open(file_path) as af: assert_cubes_equal(af["ndcube_gwcs"], sndc) + @pytest.mark.xfail(reason="Serialization of ndcube with .wcs attribute as astropy.wcs.wcs.WCS not supported") def test_serialization_ndcube_wcs(ndcube_3d_ln_lt_l, tmp_path): file_path = tmp_path / "test.asdf" diff --git a/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml index 1d31476fb..728d46353 100644 --- a/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml @@ -12,7 +12,9 @@ description: type: object properties: unit: - tag: "tag:stsci.edu:asdf/unit/unit-*" + anyOf: + - tag: "tag:stsci.edu:asdf/unit/unit-*" + - tag: "tag:astropy.org:astropy/units/unit-1.*" table: type: array items: diff --git a/tox.ini b/tox.ini index 1c4743e71..2655a0b5d 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ envlist = py310-oldestdeps codestyle build_docs - schema + asdf_schemas [testenv] # We use bash in some of our environments so we have to whitelist it. @@ -91,7 +91,7 @@ commands = {toxinidir}/docs \ {posargs} -[testenv:schema] +[testenv:asdf_schemas] description = Run schema tests deps = pytest From 693e63ef186c4834ec65c3042e61f6559cc0ad95 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sat, 20 Jul 2024 02:17:39 +0530 Subject: [PATCH 27/78] env name update --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af9fba9ba..874a08936 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - windows: py311 - macos: py310 - linux: py310-oldestdeps - - linux: schema + - linux: asdf_schemas secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} From de6944e721853ef47f7ef41ecef90845654c6ef7 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 5 Aug 2024 01:55:41 +0530 Subject: [PATCH 28/78] Add asdf as an optional dep --- pyproject.toml | 16 ++++++++++------ tox.ini | 1 + 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9fcd7d50f..754bd98ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,10 +16,9 @@ authors = [ { name = "The SunPy Community", email = "sunpy@googlegroups.com" }, ] dependencies = [ - "astropy>=5.3", - "gwcs>=0.20", - "numpy>=1.23.0", - "asdf>=2.14.4" + "astropy>=5.0.6,!=5.1.0", + "gwcs>=0.18", + "numpy>=1.23.0" ] dynamic = ["version"] @@ -55,11 +54,16 @@ plotting = [ reproject = [ "reproject>=0.7.1", ] +asdf = [ + "asdf>=2.14.4", + "astropy>=5.3.0", + "gwcs>=0.20" +] all = [ - "ndcube[plotting,reproject]", + "ndcube[plotting,reproject,asdf]", ] dev = [ - "ndcube[tests,docs,plotting,reproject]", + "ndcube[tests,docs,plotting,reproject,asdf]", ] [project.urls] repository = "https://docs.sunpy.org/projects/ndcube" diff --git a/tox.ini b/tox.ini index 2655a0b5d..24e66e651 100644 --- a/tox.ini +++ b/tox.ini @@ -63,6 +63,7 @@ deps = figure-!devdeps: scipy # The following indicates which extras_require will be installed extras = + asdf plotting reproject tests From 22a839eb30c0cca36c0da7e4f96e9d13856656e6 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 5 Aug 2024 12:19:08 +0530 Subject: [PATCH 29/78] Remove asdf as an optional dependency --- ndcube/asdf/converters/tests/test_ndcube_converter.py | 3 +++ pyproject.toml | 9 ++------- tox.ini | 1 - 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index 5104193d3..19feefa1b 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -1,5 +1,7 @@ import numpy as np import pytest +from gwcs import __version__ as gwcs_version +from packaging.version import Version import asdf @@ -13,6 +15,7 @@ ("ndcube_gwcs_3d_rotated"), ("ndcube_gwcs_4d_ln_lt_l_t") ], indirect=("ndc",)) +@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") def test_serialization(ndc, tmp_path): file_path = tmp_path / "test.asdf" with asdf.AsdfFile() as af: diff --git a/pyproject.toml b/pyproject.toml index 754bd98ea..8756cc1fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,16 +54,11 @@ plotting = [ reproject = [ "reproject>=0.7.1", ] -asdf = [ - "asdf>=2.14.4", - "astropy>=5.3.0", - "gwcs>=0.20" -] all = [ - "ndcube[plotting,reproject,asdf]", + "ndcube[plotting,reproject]", ] dev = [ - "ndcube[tests,docs,plotting,reproject,asdf]", + "ndcube[tests,docs,plotting,reproject]", ] [project.urls] repository = "https://docs.sunpy.org/projects/ndcube" diff --git a/tox.ini b/tox.ini index 24e66e651..2655a0b5d 100644 --- a/tox.ini +++ b/tox.ini @@ -63,7 +63,6 @@ deps = figure-!devdeps: scipy # The following indicates which extras_require will be installed extras = - asdf plotting reproject tests From ec863ad931d1b870b1d737128d8e001397e1ada2 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 7 Aug 2024 02:21:09 +0530 Subject: [PATCH 30/78] Add support for meta in converter and schema --- ndcube/asdf/converters/ndcube_converter.py | 14 +++++++------- ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml | 2 ++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 355e54ce5..34d372019 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -10,9 +10,11 @@ class NDCubeConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.ndcube import NDCube - ndcube = NDCube(node["data"], node["wcs"]) - ndcube._extra_coords = node["extra_coords"] - ndcube._global_coords = node["global_coords"] + ndcube = NDCube(node["data"], node["wcs"], meta=node.get("meta")) + if "extra_coords" in node: + ndcube._extra_coords = node["extra_coords"] + if "global_coords" in node: + ndcube._global_coords = node["global_coords"] return ndcube @@ -28,7 +30,7 @@ def to_yaml_tree(self, ndcube, tag, ctx): Warnings -------- UserWarning - Warns if the NDCube object has attributes 'uncertainty', 'mask', 'meta', + Warns if the NDCube object has attributes 'uncertainty', 'mask', or 'unit' that are present but not being saved in the ASDF serialization. This ensures that users are aware of potentially important information that is not included in the serialized output. @@ -38,13 +40,11 @@ def to_yaml_tree(self, ndcube, tag, ctx): node["wcs"] = ndcube.wcs node["extra_coords"] = ndcube.extra_coords node["global_coords"] = ndcube.global_coords + node["meta"] = ndcube.meta attributes = ['uncertainty', 'mask', 'unit'] for attr in attributes: if getattr(ndcube, attr) is not None: warnings.warn(f"Attribute '{attr}' is present but not being saved in ASDF serialization.", UserWarning) - if len(ndcube.meta) > 0: - warnings.warn("Attribute 'meta' is present but not being saved in ASDF serialization.", UserWarning) - return node diff --git a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml index 8b4270be6..db10a488e 100644 --- a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml @@ -19,6 +19,8 @@ properties: tag: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-0.*" global_coords: tag: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.*" + meta: + type: object required: [data, wcs] additionalProperties: true From b2fd5e1480cdbb71e207aa1ab08f0275929d3dc4 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 21 Aug 2024 20:41:55 +0530 Subject: [PATCH 31/78] Add mask as a validator property --- ndcube/asdf/converters/ndcube_converter.py | 9 +++++++-- ndcube/asdf/converters/tests/test_ndcube_converter.py | 2 +- ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml | 2 ++ ndcube/conftest.py | 9 +++++++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 34d372019..c88b8bdff 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -10,7 +10,10 @@ class NDCubeConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.ndcube import NDCube - ndcube = NDCube(node["data"], node["wcs"], meta=node.get("meta")) + ndcube = NDCube(node["data"], + node["wcs"], + meta = node.get("meta"), + mask = node.get("mask")) if "extra_coords" in node: ndcube._extra_coords = node["extra_coords"] if "global_coords" in node: @@ -41,8 +44,10 @@ def to_yaml_tree(self, ndcube, tag, ctx): node["extra_coords"] = ndcube.extra_coords node["global_coords"] = ndcube.global_coords node["meta"] = ndcube.meta + if ndcube.mask is not None: + node["mask"] = ndcube.mask - attributes = ['uncertainty', 'mask', 'unit'] + attributes = ['uncertainty', 'unit'] for attr in attributes: if getattr(ndcube, attr) is not None: warnings.warn(f"Attribute '{attr}' is present but not being saved in ASDF serialization.", UserWarning) diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index 19feefa1b..ad5df5ee7 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -8,7 +8,7 @@ from ndcube.tests.helpers import assert_cubes_equal -@pytest.mark.parametrize("ndc",[("ndcube_gwcs_2d_ln_lt"), +@pytest.mark.parametrize("ndc",[("ndcube_gwcs_2d_ln_lt_mask"), ("ndcube_gwcs_3d_ln_lt_l"), ("ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim"), ("ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc"), diff --git a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml index db10a488e..5529ccbfa 100644 --- a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml @@ -21,6 +21,8 @@ properties: tag: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.*" meta: type: object + mask: + type: object required: [data, wcs] additionalProperties: true diff --git a/ndcube/conftest.py b/ndcube/conftest.py index 56b8a0165..6d0dcf9c5 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -502,10 +502,15 @@ def ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc(gwcs_3d_lt_ln_l): return cube @pytest.fixture -def ndcube_gwcs_2d_ln_lt(gwcs_2d_lt_ln): +def ndcube_gwcs_2d_ln_lt_mask(gwcs_2d_lt_ln): shape = (10, 12) data_cube = data_nd(shape) - return NDCube(data_cube, wcs=gwcs_2d_lt_ln) + mask = np.zeros(shape, dtype=bool) + mask[1, 1] = True + mask[2, 0] = True + mask[3, 3] = True + mask[4:6, :4] = True + return NDCube(data_cube, wcs=gwcs_2d_lt_ln, mask=mask) @pytest.fixture def ndcube_4d_ln_l_t_lt(wcs_4d_lt_t_l_ln): From 6d81ee9581979ccef72070112e5bb04af5a3e1bb Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 22 Aug 2024 14:11:45 +0100 Subject: [PATCH 32/78] Add unit support to asdf --- ndcube/asdf/converters/ndcube_converter.py | 7 +++++-- .../asdf/converters/tests/test_ndcube_converter.py | 13 +++++++------ ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml | 4 ++++ ndcube/conftest.py | 9 +++++++++ 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index c88b8bdff..32e572cfd 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -13,7 +13,8 @@ def from_yaml_tree(self, node, tag, ctx): ndcube = NDCube(node["data"], node["wcs"], meta = node.get("meta"), - mask = node.get("mask")) + mask = node.get("mask"), + unit = node.get("unit")) if "extra_coords" in node: ndcube._extra_coords = node["extra_coords"] if "global_coords" in node: @@ -46,8 +47,10 @@ def to_yaml_tree(self, ndcube, tag, ctx): node["meta"] = ndcube.meta if ndcube.mask is not None: node["mask"] = ndcube.mask + if ndcube.unit is not None: + node["unit"] = ndcube.unit - attributes = ['uncertainty', 'unit'] + attributes = ['uncertainty', 'psf'] for attr in attributes: if getattr(ndcube, attr) is not None: warnings.warn(f"Attribute '{attr}' is present but not being saved in ASDF serialization.", UserWarning) diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index ad5df5ee7..649b08b2e 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -8,12 +8,13 @@ from ndcube.tests.helpers import assert_cubes_equal -@pytest.mark.parametrize("ndc",[("ndcube_gwcs_2d_ln_lt_mask"), - ("ndcube_gwcs_3d_ln_lt_l"), - ("ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim"), - ("ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc"), - ("ndcube_gwcs_3d_rotated"), - ("ndcube_gwcs_4d_ln_lt_l_t") +@pytest.mark.parametrize("ndc",["ndcube_gwcs_2d_ln_lt_mask", + "ndcube_gwcs_3d_ln_lt_l", + "ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim", + "ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc", + "ndcube_gwcs_3d_rotated", + "ndcube_gwcs_4d_ln_lt_l_t", + "ndcube_gwcs_4d_ln_lt_l_t_unit", ], indirect=("ndc",)) @pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") def test_serialization(ndc, tmp_path): diff --git a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml index 5529ccbfa..5ef9a976e 100644 --- a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml @@ -23,6 +23,10 @@ properties: type: object mask: type: object + unit: + anyOf: + - tag: "tag:stsci.edu:asdf/unit/unit-1.*" + - tag: "tag:astropy.org:astropy/units/unit-1.*" required: [data, wcs] additionalProperties: true diff --git a/ndcube/conftest.py b/ndcube/conftest.py index 6d0dcf9c5..2d3cd359f 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -463,6 +463,15 @@ def ndcube_gwcs_4d_ln_lt_l_t(gwcs_4d_t_l_lt_ln): data_cube = data_nd(shape) return NDCube(data_cube, wcs=gwcs_4d_t_l_lt_ln) + +@pytest.fixture +def ndcube_gwcs_4d_ln_lt_l_t_unit(gwcs_4d_t_l_lt_ln): + shape = (5, 8, 10, 12) + gwcs_4d_t_l_lt_ln.array_shape = shape + data_cube = data_nd(shape) + return NDCube(data_cube, wcs=gwcs_4d_t_l_lt_ln, unit=u.DN) + + @pytest.fixture def ndcube_gwcs_3d_ln_lt_l(gwcs_3d_lt_ln_l): shape = (2, 3, 4) From a6a33be09149daa7d2dbbde1170a6bee5ba2e313 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Fri, 16 Aug 2024 12:40:18 +0530 Subject: [PATCH 33/78] Add the serialization support for the ResampledLowlevelWCS ReorderedLowLevelWCS CompoundLowLevelWCS --- .../asdf/converters/compoundwcs_converter.py | 18 ++++ ndcube/asdf/converters/ndcube_converter.py | 7 +- .../asdf/converters/reorderedwcs_converter.py | 21 +++++ ndcube/asdf/converters/resampled_converter.py | 21 +++++ .../tests/test_ndcube_wcs_wrappers.py | 92 +++++++++++++++++++ ndcube/asdf/entry_points.py | 6 ++ .../resources/manifests/ndcube-0.1.0.yaml | 9 ++ .../resources/schemas/compoundwcs-0.1.0.yaml | 25 +++++ .../asdf/resources/schemas/ndcube-0.1.0.yaml | 6 +- .../resources/schemas/reorderedwcs-0.1.0.yaml | 23 +++++ .../resources/schemas/resampledwcs-0.1.0.yaml | 23 +++++ 11 files changed, 249 insertions(+), 2 deletions(-) create mode 100644 ndcube/asdf/converters/compoundwcs_converter.py create mode 100644 ndcube/asdf/converters/reorderedwcs_converter.py create mode 100644 ndcube/asdf/converters/resampled_converter.py create mode 100644 ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py create mode 100644 ndcube/asdf/resources/schemas/compoundwcs-0.1.0.yaml create mode 100644 ndcube/asdf/resources/schemas/reorderedwcs-0.1.0.yaml create mode 100644 ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml diff --git a/ndcube/asdf/converters/compoundwcs_converter.py b/ndcube/asdf/converters/compoundwcs_converter.py new file mode 100644 index 000000000..b812fd051 --- /dev/null +++ b/ndcube/asdf/converters/compoundwcs_converter.py @@ -0,0 +1,18 @@ +from asdf.extension import Converter + + +class CompoundConverter(Converter): + tags = ["tag:sunpy.org:ndcube/compoundwcs-0.1.0"] + types = ["ndcube.wcs.wrappers.compound_wcs.CompoundLowLevelWCS"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.wcs.wrappers import CompoundLowLevelWCS + + return(CompoundLowLevelWCS(*node["wcs"], mapping = node.get("mapping"), pixel_atol = node.get("atol"))) + + def to_yaml_tree(self, compoundwcs, tag, ctx): + node={} + node["wcs"] = compoundwcs._wcs + node["mapping"] = compoundwcs.mapping.mapping + node["atol"] = compoundwcs.atol + return node diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 32e572cfd..f1a78dddf 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -39,9 +39,14 @@ def to_yaml_tree(self, ndcube, tag, ctx): This ensures that users are aware of potentially important information that is not included in the serialized output. """ + from astropy.wcs.wcsapi import HighLevelWCSWrapper + node = {} node["data"] = ndcube.data - node["wcs"] = ndcube.wcs + if isinstance(ndcube.wcs, HighLevelWCSWrapper): + node["wcs"] = ndcube.wcs._low_level_wcs + else: + node["wcs"] = ndcube.wcs node["extra_coords"] = ndcube.extra_coords node["global_coords"] = ndcube.global_coords node["meta"] = ndcube.meta diff --git a/ndcube/asdf/converters/reorderedwcs_converter.py b/ndcube/asdf/converters/reorderedwcs_converter.py new file mode 100644 index 000000000..df86a9804 --- /dev/null +++ b/ndcube/asdf/converters/reorderedwcs_converter.py @@ -0,0 +1,21 @@ +from asdf.extension import Converter + + +class ReorderedConverter(Converter): + tags = ["tag:sunpy.org:ndcube/reorderedwcs-0.1.0"] + types = ["ndcube.wcs.wrappers.reordered_wcs.ReorderedLowLevelWCS"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.wcs.wrappers import ReorderedLowLevelWCS + + reorderedwcs = ReorderedLowLevelWCS(wcs=node["wcs"], + pixel_order = node.get("pixel_order"), + world_order = node.get("world_order") + ) + return reorderedwcs + def to_yaml_tree(self, reorderedwcs, tag, ctx): + node={} + node["wcs"] = reorderedwcs._wcs + node["pixel_order"] = (reorderedwcs._pixel_order) + node["world_order"] = (reorderedwcs._world_order) + return node diff --git a/ndcube/asdf/converters/resampled_converter.py b/ndcube/asdf/converters/resampled_converter.py new file mode 100644 index 000000000..aea976ceb --- /dev/null +++ b/ndcube/asdf/converters/resampled_converter.py @@ -0,0 +1,21 @@ +from asdf.extension import Converter + + +class ResampledConverter(Converter): + tags = ["tag:sunpy.org:ndcube/resampledwcs-0.1.0"] + types = ["ndcube.wcs.wrappers.resampled_wcs.ResampledLowLevelWCS"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.wcs.wrappers import ResampledLowLevelWCS + + resampledwcs = ResampledLowLevelWCS(wcs=node["wcs"], + offset = node.get("offset"), + factor = node.get("factor"), + ) + return resampledwcs + def to_yaml_tree(self, resampledwcs, tag, ctx): + node={} + node["wcs"] = resampledwcs._wcs + node["factor"] = (resampledwcs._factor) + node["offset"] = (resampledwcs._offset) + return node diff --git a/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py b/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py new file mode 100644 index 000000000..fab66d381 --- /dev/null +++ b/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py @@ -0,0 +1,92 @@ +""" +Tests for roundtrip serialization of NDCube with various GWCS types. + +TODO: Add tests for the roundtrip serialization of NDCube with ResampledLowLevelWCS, ReorderedLowLevelWCS, and CompoundLowLevelWCS when using astropy.wcs.WCS. +""" + +import pytest +from gwcs import __version__ as gwcs_version +from packaging.version import Version + +import asdf + +from ndcube import NDCube +from ndcube.conftest import data_nd +from ndcube.tests.helpers import assert_cubes_equal +from ndcube.wcs.wrappers import CompoundLowLevelWCS, ReorderedLowLevelWCS, ResampledLowLevelWCS + + +@pytest.fixture +def create_ndcube_resampledwcs(gwcs_3d_lt_ln_l): + shape = (2, 3, 4) + new_wcs = ResampledLowLevelWCS(wcs = gwcs_3d_lt_ln_l, factor=2 ,offset = 1) + data = data_nd(shape) + return NDCube(data = data, wcs =new_wcs) + + +@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") +def test_serialization_resampled(create_ndcube_resampledwcs, tmp_path): + ndc = create_ndcube_resampledwcs + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube"] = ndc + af.write_to(file_path) + + with asdf.open(file_path) as af: + loaded_ndcube = af["ndcube"] + + loaded_resampledwcs = loaded_ndcube.wcs.low_level_wcs + resampledwcs = ndc.wcs.low_level_wcs + assert (loaded_resampledwcs._factor == resampledwcs._factor).all() + assert (loaded_resampledwcs._offset == resampledwcs._offset).all() + + assert_cubes_equal(loaded_ndcube, ndc) + +@pytest.fixture +def create_ndcube_reorderedwcs(gwcs_3d_lt_ln_l): + shape = (2, 3, 4) + new_wcs = ReorderedLowLevelWCS(wcs = gwcs_3d_lt_ln_l, pixel_order=[1, 2, 0] ,world_order=[2, 0, 1]) + data = data_nd(shape) + return NDCube(data = data, wcs =new_wcs) + + + +@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") +def test_serialization_reordered(create_ndcube_reorderedwcs, tmp_path): + ndc = create_ndcube_reorderedwcs + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube"] = ndc + af.write_to(file_path) + + with asdf.open(file_path) as af: + loaded_ndcube = af["ndcube"] + + loaded_reorderedwcs = loaded_ndcube.wcs.low_level_wcs + reorderedwcs = ndc.wcs.low_level_wcs + assert (loaded_reorderedwcs._pixel_order == reorderedwcs._pixel_order) + assert (loaded_reorderedwcs._world_order == reorderedwcs._world_order) + + assert_cubes_equal(loaded_ndcube, ndc) + +@pytest.fixture +def create_ndcube_compoundwcs(gwcs_2d_lt_ln, time_and_simple_extra_coords_2d): + + shape = (1, 2, 3, 4) + new_wcs = CompoundLowLevelWCS(gwcs_2d_lt_ln, time_and_simple_extra_coords_2d.wcs, mapping = [0, 1, 2, 3]) + data = data_nd(shape) + return NDCube(data = data, wcs = new_wcs) + +@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") +def test_serialization_compoundwcs(create_ndcube_compoundwcs, tmp_path): + ndc = create_ndcube_compoundwcs + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube"] = ndc + af.write_to(file_path) + + with asdf.open(file_path) as af: + loaded_ndcube = af["ndcube"] + assert_cubes_equal(loaded_ndcube, ndc) + assert (loaded_ndcube.wcs.low_level_wcs.mapping.mapping == ndc.wcs.low_level_wcs.mapping.mapping) + assert (loaded_ndcube.wcs.low_level_wcs.atol == ndc.wcs.low_level_wcs.atol) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 55462842e..0919b45f7 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -31,9 +31,12 @@ def get_extensions(): """ Get the list of extensions. """ + from ndcube.asdf.converters.compoundwcs_converter import CompoundConverter from ndcube.asdf.converters.extracoords_converter import ExtraCoordsConverter from ndcube.asdf.converters.globalcoords_converter import GlobalCoordsConverter from ndcube.asdf.converters.ndcube_converter import NDCubeConverter + from ndcube.asdf.converters.reorderedwcs_converter import ReorderedConverter + from ndcube.asdf.converters.resampled_converter import ResampledConverter from ndcube.asdf.converters.tablecoord_converter import ( QuantityTableCoordinateConverter, SkyCoordTableCoordinateConverter, @@ -47,6 +50,9 @@ def get_extensions(): QuantityTableCoordinateConverter(), SkyCoordTableCoordinateConverter(), GlobalCoordsConverter(), + ResampledConverter(), + ReorderedConverter(), + CompoundConverter(), ] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index 4559a5499..d9e3a02bf 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -23,3 +23,12 @@ tags: - tag_uri: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.1.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/global_coords-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/resampledwcs-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/resampledwcs-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/reorderedwcs-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/reorderedwcs-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/compoundwcs-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/compoundwcs-0.1.0" diff --git a/ndcube/asdf/resources/schemas/compoundwcs-0.1.0.yaml b/ndcube/asdf/resources/schemas/compoundwcs-0.1.0.yaml new file mode 100644 index 000000000..24028920e --- /dev/null +++ b/ndcube/asdf/resources/schemas/compoundwcs-0.1.0.yaml @@ -0,0 +1,25 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/Compoundwcs-0.1.0" + +title: + Represents the ndcube CompoundLowLevelWCS object + +description: + Represents the ndcube CompoundLowLevelWCS object + +type: object +properties: + wcs: + type: array + items: + tag: "tag:stsci.edu:gwcs/wcs-1.*" + mapping: + type: array + atol: + type: number + +required: [wcs] +additionalProperties: true +... diff --git a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml index 5ef9a976e..1981f07fb 100644 --- a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml @@ -14,7 +14,11 @@ properties: data: description: "Must be compatible with ASDF serialization/deserialization and supported by NDCube." wcs: - tag: "tag:stsci.edu:gwcs/wcs-1.*" + anyOf: + - tag: "tag:stsci.edu:gwcs/wcs-1.*" + - tag: "tag:sunpy.org:ndcube/resampledwcs-0.1.0" + - tag: "tag:sunpy.org:ndcube/reorderedwcs-0.1.0" + - tag: "tag:sunpy.org:ndcube/compoundwcs-0.1.0" extra_coords: tag: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-0.*" global_coords: diff --git a/ndcube/asdf/resources/schemas/reorderedwcs-0.1.0.yaml b/ndcube/asdf/resources/schemas/reorderedwcs-0.1.0.yaml new file mode 100644 index 000000000..76e54521b --- /dev/null +++ b/ndcube/asdf/resources/schemas/reorderedwcs-0.1.0.yaml @@ -0,0 +1,23 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/resampledwcs-0.1.0" + +title: + Represents the ndcube ReorderedLowLevelWCS object + +description: + Represents the ndcube ReorderedLowLevelWCS object + +type: object +properties: + wcs: + tag: "tag:stsci.edu:gwcs/wcs-1.*" + pixel_order: + type: array + world_order: + type: array + +required: [wcs] +additionalProperties: true +... diff --git a/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml b/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml new file mode 100644 index 000000000..504419de7 --- /dev/null +++ b/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml @@ -0,0 +1,23 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/resampledwcs-0.1.0" + +title: + Represents the ndcube ResampledLowLevelWCS object + +description: + Represents the ndcube ResampledLowLevelWCS object + +type: object +properties: + wcs: + tag: "tag:stsci.edu:gwcs/wcs-1.*" + factor: + type: object + offset: + type: object + +required: [wcs] +additionalProperties: true +... From 95486b01babafa14a6c4823b70a72300efd3c180 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Fri, 23 Aug 2024 22:29:51 +0530 Subject: [PATCH 34/78] Apply suggestions from code review --- ndcube/asdf/converters/reorderedwcs_converter.py | 4 ++-- ndcube/asdf/converters/resampled_converter.py | 5 +++-- ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ndcube/asdf/converters/reorderedwcs_converter.py b/ndcube/asdf/converters/reorderedwcs_converter.py index df86a9804..4fb2264a3 100644 --- a/ndcube/asdf/converters/reorderedwcs_converter.py +++ b/ndcube/asdf/converters/reorderedwcs_converter.py @@ -16,6 +16,6 @@ def from_yaml_tree(self, node, tag, ctx): def to_yaml_tree(self, reorderedwcs, tag, ctx): node={} node["wcs"] = reorderedwcs._wcs - node["pixel_order"] = (reorderedwcs._pixel_order) - node["world_order"] = (reorderedwcs._world_order) + node["pixel_order"] = reorderedwcs._pixel_order + node["world_order"] = reorderedwcs._world_order return node diff --git a/ndcube/asdf/converters/resampled_converter.py b/ndcube/asdf/converters/resampled_converter.py index aea976ceb..da9545545 100644 --- a/ndcube/asdf/converters/resampled_converter.py +++ b/ndcube/asdf/converters/resampled_converter.py @@ -16,6 +16,7 @@ def from_yaml_tree(self, node, tag, ctx): def to_yaml_tree(self, resampledwcs, tag, ctx): node={} node["wcs"] = resampledwcs._wcs - node["factor"] = (resampledwcs._factor) - node["offset"] = (resampledwcs._offset) + node["factor"] = resampledwcs._factor + node["offset"] = resampledwcs._offset + return node diff --git a/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml b/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml index 504419de7..bfecfa023 100644 --- a/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml @@ -14,9 +14,9 @@ properties: wcs: tag: "tag:stsci.edu:gwcs/wcs-1.*" factor: - type: object + tag: "tag:stsci.edu:asdf/core/ndarray-1.0.0" offset: - type: object + tag: "tag:stsci.edu:asdf/core/ndarray-1.0.0" required: [wcs] additionalProperties: true From 4fb52886844e5bcca62788b2da2ab414df7adee2 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sun, 25 Aug 2024 16:06:22 +0530 Subject: [PATCH 35/78] Use Tabular1D, Tabular2D instead of tabular_model --- ndcube/extra_coords/table_coord.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ndcube/extra_coords/table_coord.py b/ndcube/extra_coords/table_coord.py index 503d6f1c7..297ff3ab4 100644 --- a/ndcube/extra_coords/table_coord.py +++ b/ndcube/extra_coords/table_coord.py @@ -10,8 +10,7 @@ import astropy.units as u from astropy.coordinates import SkyCoord from astropy.modeling import models -from astropy.modeling.models import tabular_model -from astropy.modeling.tabular import _Tabular +from astropy.modeling.tabular import Tabular1D, Tabular2D, _Tabular from astropy.time import Time from astropy.wcs.wcsapi.wrappers.sliced_wcs import combine_slices, sanitize_slices @@ -136,7 +135,10 @@ def _generate_tabular(lookup_table, interpolation='linear', points_unit=u.pix, * raise TypeError("lookup_table must be a Quantity.") # pragma: no cover ndim = lookup_table.ndim - TabularND = tabular_model(ndim, name=f"Tabular{ndim}D") + if ndim == 1: + TabularND = Tabular1D + elif ndim == 2: + TabularND = Tabular2D # The integer location is at the centre of the pixel. points = [(np.arange(size) - 0) * points_unit for size in lookup_table.shape] From 6cf9602e9cca42a873670f3b15a3cf99dd820183 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sun, 25 Aug 2024 17:29:53 +0530 Subject: [PATCH 36/78] Add comment and update logic --- ndcube/extra_coords/table_coord.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ndcube/extra_coords/table_coord.py b/ndcube/extra_coords/table_coord.py index 297ff3ab4..13bd0f25b 100644 --- a/ndcube/extra_coords/table_coord.py +++ b/ndcube/extra_coords/table_coord.py @@ -10,6 +10,7 @@ import astropy.units as u from astropy.coordinates import SkyCoord from astropy.modeling import models +from astropy.modeling.models import tabular_model from astropy.modeling.tabular import Tabular1D, Tabular2D, _Tabular from astropy.time import Time from astropy.wcs.wcsapi.wrappers.sliced_wcs import combine_slices, sanitize_slices @@ -135,10 +136,17 @@ def _generate_tabular(lookup_table, interpolation='linear', points_unit=u.pix, * raise TypeError("lookup_table must be a Quantity.") # pragma: no cover ndim = lookup_table.ndim + + # Use existing Tabular1D and Tabular2D classes for 1D and 2D models + # to ensure compatibility with asdf-astropy converters and avoid + # dynamically generated classes that are not recognized by asdf. + # See PR #751 for details on the issue this addresses. if ndim == 1: TabularND = Tabular1D elif ndim == 2: TabularND = Tabular2D + else: + TabularND = tabular_model(ndim, name=f"Tabular{ndim}D") # The integer location is at the centre of the pixel. points = [(np.arange(size) - 0) * points_unit for size in lookup_table.shape] From ddc75bee9a6f5d886fd5f3f0749de0ca4a2cc8ae Mon Sep 17 00:00:00 2001 From: DanRyanIrish Date: Mon, 4 Nov 2024 16:03:01 +0000 Subject: [PATCH 37/78] Apply minor suggestions from code review Co-authored-by: Stuart Mumford --- ndcube/asdf/converters/compoundwcs_converter.py | 4 ++-- ndcube/asdf/converters/reorderedwcs_converter.py | 6 +++--- ndcube/asdf/converters/resampled_converter.py | 6 +++--- ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ndcube/asdf/converters/compoundwcs_converter.py b/ndcube/asdf/converters/compoundwcs_converter.py index b812fd051..3cbd9430b 100644 --- a/ndcube/asdf/converters/compoundwcs_converter.py +++ b/ndcube/asdf/converters/compoundwcs_converter.py @@ -8,10 +8,10 @@ class CompoundConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.wcs.wrappers import CompoundLowLevelWCS - return(CompoundLowLevelWCS(*node["wcs"], mapping = node.get("mapping"), pixel_atol = node.get("atol"))) + return(CompoundLowLevelWCS(*node["wcs"], mapping=node.get("mapping"), pixel_atol=node.get("atol"))) def to_yaml_tree(self, compoundwcs, tag, ctx): - node={} + node = {} node["wcs"] = compoundwcs._wcs node["mapping"] = compoundwcs.mapping.mapping node["atol"] = compoundwcs.atol diff --git a/ndcube/asdf/converters/reorderedwcs_converter.py b/ndcube/asdf/converters/reorderedwcs_converter.py index 4fb2264a3..0ce5ca3e8 100644 --- a/ndcube/asdf/converters/reorderedwcs_converter.py +++ b/ndcube/asdf/converters/reorderedwcs_converter.py @@ -9,12 +9,12 @@ def from_yaml_tree(self, node, tag, ctx): from ndcube.wcs.wrappers import ReorderedLowLevelWCS reorderedwcs = ReorderedLowLevelWCS(wcs=node["wcs"], - pixel_order = node.get("pixel_order"), - world_order = node.get("world_order") + pixel_order=node.get("pixel_order"), + world_order=node.get("world_order") ) return reorderedwcs def to_yaml_tree(self, reorderedwcs, tag, ctx): - node={} + node = {} node["wcs"] = reorderedwcs._wcs node["pixel_order"] = reorderedwcs._pixel_order node["world_order"] = reorderedwcs._world_order diff --git a/ndcube/asdf/converters/resampled_converter.py b/ndcube/asdf/converters/resampled_converter.py index da9545545..ab41c1d70 100644 --- a/ndcube/asdf/converters/resampled_converter.py +++ b/ndcube/asdf/converters/resampled_converter.py @@ -9,12 +9,12 @@ def from_yaml_tree(self, node, tag, ctx): from ndcube.wcs.wrappers import ResampledLowLevelWCS resampledwcs = ResampledLowLevelWCS(wcs=node["wcs"], - offset = node.get("offset"), - factor = node.get("factor"), + offset=node.get("offset"), + factor=node.get("factor"), ) return resampledwcs def to_yaml_tree(self, resampledwcs, tag, ctx): - node={} + node = {} node["wcs"] = resampledwcs._wcs node["factor"] = resampledwcs._factor node["offset"] = resampledwcs._offset diff --git a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml index 1981f07fb..0b2a414e0 100644 --- a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml @@ -16,9 +16,9 @@ properties: wcs: anyOf: - tag: "tag:stsci.edu:gwcs/wcs-1.*" - - tag: "tag:sunpy.org:ndcube/resampledwcs-0.1.0" - - tag: "tag:sunpy.org:ndcube/reorderedwcs-0.1.0" - - tag: "tag:sunpy.org:ndcube/compoundwcs-0.1.0" + - tag: "tag:sunpy.org:ndcube/resampledwcs-*" + - tag: "tag:sunpy.org:ndcube/reorderedwcs-*" + - tag: "tag:sunpy.org:ndcube/compoundwcs-*" extra_coords: tag: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-0.*" global_coords: From e7cf14fd3514d8e1b2c7a83d74f6ee71f6fc11e8 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 6 Nov 2024 09:42:15 +0000 Subject: [PATCH 38/78] More minor tweaks --- ndcube/asdf/converters/ndcube_converter.py | 6 +++--- ndcube/asdf/converters/reorderedwcs_converter.py | 6 +++--- ndcube/asdf/converters/resampled_converter.py | 3 ++- ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py | 6 +++--- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index f1a78dddf..bc855684d 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -39,12 +39,12 @@ def to_yaml_tree(self, ndcube, tag, ctx): This ensures that users are aware of potentially important information that is not included in the serialized output. """ - from astropy.wcs.wcsapi import HighLevelWCSWrapper + from astropy.wcs.wcsapi import BaseHighLevelWCS node = {} node["data"] = ndcube.data - if isinstance(ndcube.wcs, HighLevelWCSWrapper): - node["wcs"] = ndcube.wcs._low_level_wcs + if isinstance(ndcube.wcs, BaseHighLevelWCS): + node["wcs"] = ndcube.wcs.low_level_wcs else: node["wcs"] = ndcube.wcs node["extra_coords"] = ndcube.extra_coords diff --git a/ndcube/asdf/converters/reorderedwcs_converter.py b/ndcube/asdf/converters/reorderedwcs_converter.py index 0ce5ca3e8..715c11e2a 100644 --- a/ndcube/asdf/converters/reorderedwcs_converter.py +++ b/ndcube/asdf/converters/reorderedwcs_converter.py @@ -2,7 +2,7 @@ class ReorderedConverter(Converter): - tags = ["tag:sunpy.org:ndcube/reorderedwcs-0.1.0"] + tags = ["tag:sunpy.org:ndcube/reorderedwcs-*"] types = ["ndcube.wcs.wrappers.reordered_wcs.ReorderedLowLevelWCS"] def from_yaml_tree(self, node, tag, ctx): @@ -10,9 +10,9 @@ def from_yaml_tree(self, node, tag, ctx): reorderedwcs = ReorderedLowLevelWCS(wcs=node["wcs"], pixel_order=node.get("pixel_order"), - world_order=node.get("world_order") - ) + world_order=node.get("world_order")) return reorderedwcs + def to_yaml_tree(self, reorderedwcs, tag, ctx): node = {} node["wcs"] = reorderedwcs._wcs diff --git a/ndcube/asdf/converters/resampled_converter.py b/ndcube/asdf/converters/resampled_converter.py index ab41c1d70..6e23e1616 100644 --- a/ndcube/asdf/converters/resampled_converter.py +++ b/ndcube/asdf/converters/resampled_converter.py @@ -2,7 +2,7 @@ class ResampledConverter(Converter): - tags = ["tag:sunpy.org:ndcube/resampledwcs-0.1.0"] + tags = ["tag:sunpy.org:ndcube/resampledwcs-*"] types = ["ndcube.wcs.wrappers.resampled_wcs.ResampledLowLevelWCS"] def from_yaml_tree(self, node, tag, ctx): @@ -13,6 +13,7 @@ def from_yaml_tree(self, node, tag, ctx): factor=node.get("factor"), ) return resampledwcs + def to_yaml_tree(self, resampledwcs, tag, ctx): node = {} node["wcs"] = resampledwcs._wcs diff --git a/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py b/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py index fab66d381..791c29385 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py +++ b/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py @@ -19,9 +19,9 @@ @pytest.fixture def create_ndcube_resampledwcs(gwcs_3d_lt_ln_l): shape = (2, 3, 4) - new_wcs = ResampledLowLevelWCS(wcs = gwcs_3d_lt_ln_l, factor=2 ,offset = 1) + new_wcs = ResampledLowLevelWCS(wcs=gwcs_3d_lt_ln_l, factor=2 , offset=1) data = data_nd(shape) - return NDCube(data = data, wcs =new_wcs) + return NDCube(data=data, wcs=new_wcs) @pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") @@ -42,6 +42,7 @@ def test_serialization_resampled(create_ndcube_resampledwcs, tmp_path): assert_cubes_equal(loaded_ndcube, ndc) + @pytest.fixture def create_ndcube_reorderedwcs(gwcs_3d_lt_ln_l): shape = (2, 3, 4) @@ -50,7 +51,6 @@ def create_ndcube_reorderedwcs(gwcs_3d_lt_ln_l): return NDCube(data = data, wcs =new_wcs) - @pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") def test_serialization_reordered(create_ndcube_reorderedwcs, tmp_path): ndc = create_ndcube_reorderedwcs From 8610805a9ebd14c1e8fd770db353f80a2e072115 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 5 Jun 2024 03:28:52 +0530 Subject: [PATCH 39/78] Add validation for schema and manifests --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 763e86394..34ebaf8bd 100644 --- a/pytest.ini +++ b/pytest.ini @@ -53,3 +53,4 @@ filterwarnings = ignore:Animating a NDCube does not support transposing the array. The world axes may not display as expected because the array will not be transposed:UserWarning # This is raised by the Windows and mac os build for visualization.rst ignore:FigureCanvasAgg is non-interactive, and thus cannot be shown:UserWarning + From 149944f021618cb58243b4bb842db62f796db898 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 4 Jul 2024 19:13:09 +0530 Subject: [PATCH 40/78] revert small change --- pytest.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 34ebaf8bd..763e86394 100644 --- a/pytest.ini +++ b/pytest.ini @@ -53,4 +53,3 @@ filterwarnings = ignore:Animating a NDCube does not support transposing the array. The world axes may not display as expected because the array will not be transposed:UserWarning # This is raised by the Windows and mac os build for visualization.rst ignore:FigureCanvasAgg is non-interactive, and thus cannot be shown:UserWarning - From 98e633ca2fd664df1570a08475d56c0534ad20a4 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 5 Aug 2024 01:55:41 +0530 Subject: [PATCH 41/78] Add asdf as an optional dep --- pyproject.toml | 9 +++++++-- tox.ini | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8756cc1fd..754bd98ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,11 +54,16 @@ plotting = [ reproject = [ "reproject>=0.7.1", ] +asdf = [ + "asdf>=2.14.4", + "astropy>=5.3.0", + "gwcs>=0.20" +] all = [ - "ndcube[plotting,reproject]", + "ndcube[plotting,reproject,asdf]", ] dev = [ - "ndcube[tests,docs,plotting,reproject]", + "ndcube[tests,docs,plotting,reproject,asdf]", ] [project.urls] repository = "https://docs.sunpy.org/projects/ndcube" diff --git a/tox.ini b/tox.ini index 2655a0b5d..24e66e651 100644 --- a/tox.ini +++ b/tox.ini @@ -63,6 +63,7 @@ deps = figure-!devdeps: scipy # The following indicates which extras_require will be installed extras = + asdf plotting reproject tests From 789e841bd7491688b405510a82c2fa05be1eedfe Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 5 Aug 2024 12:19:08 +0530 Subject: [PATCH 42/78] Remove asdf as an optional dependency --- pyproject.toml | 9 ++------- tox.ini | 1 - 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 754bd98ea..8756cc1fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,16 +54,11 @@ plotting = [ reproject = [ "reproject>=0.7.1", ] -asdf = [ - "asdf>=2.14.4", - "astropy>=5.3.0", - "gwcs>=0.20" -] all = [ - "ndcube[plotting,reproject,asdf]", + "ndcube[plotting,reproject]", ] dev = [ - "ndcube[tests,docs,plotting,reproject,asdf]", + "ndcube[tests,docs,plotting,reproject]", ] [project.urls] repository = "https://docs.sunpy.org/projects/ndcube" diff --git a/tox.ini b/tox.ini index 24e66e651..2655a0b5d 100644 --- a/tox.ini +++ b/tox.ini @@ -63,7 +63,6 @@ deps = figure-!devdeps: scipy # The following indicates which extras_require will be installed extras = - asdf plotting reproject tests From 24af793cc51c0cff8569710b8887655ea699fe12 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sun, 25 Aug 2024 05:13:30 +0530 Subject: [PATCH 43/78] Add serialization logic for the NDCubeSequence and NDCollection --- .../asdf/converters/ndcollection_converter.py | 30 +++++++++++ .../converters/ndcubesequence_converter.py | 25 +++++++++ .../tests/test_ndcollection_converter.py | 51 +++++++++++++++++++ .../tests/test_ndcubesequence_converter.py | 20 ++++++++ ndcube/asdf/entry_points.py | 5 +- .../resources/manifests/ndcube-0.1.0.yaml | 6 +++ .../resources/schemas/ndcollection-0.1.0.yaml | 30 +++++++++++ .../schemas/ndcube_sequence-0.1.0.yaml | 25 +++++++++ 8 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 ndcube/asdf/converters/ndcollection_converter.py create mode 100644 ndcube/asdf/converters/ndcubesequence_converter.py create mode 100644 ndcube/asdf/converters/tests/test_ndcollection_converter.py create mode 100644 ndcube/asdf/converters/tests/test_ndcubesequence_converter.py create mode 100644 ndcube/asdf/resources/schemas/ndcollection-0.1.0.yaml create mode 100644 ndcube/asdf/resources/schemas/ndcube_sequence-0.1.0.yaml diff --git a/ndcube/asdf/converters/ndcollection_converter.py b/ndcube/asdf/converters/ndcollection_converter.py new file mode 100644 index 000000000..921365539 --- /dev/null +++ b/ndcube/asdf/converters/ndcollection_converter.py @@ -0,0 +1,30 @@ +from asdf.extension import Converter + + +class NDCollectionConverter(Converter): + tags = ["tag:sunpy.org:ndcube/ndcube/ndcollection-*"] + types = ["ndcube.ndcollection.NDCollection"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.ndcollection import NDCollection + + key_value_pairs = list(zip(node["keys"], node["value"])) + aligned_axes = list(node.get("aligned_axes").values()) + aligned_axes = tuple(tuple(lst) for lst in aligned_axes) + ndcollection = NDCollection(key_value_pairs, + meta=node.get("meta"), + aligned_axes = aligned_axes) + return ndcollection + + + def to_yaml_tree(self, ndcollection, tag, ctx): + + node = {} + node["keys"] = tuple(ndcollection.keys()) + node["value"] = tuple(ndcollection.values()) + if ndcollection.meta is not None: + node["meta"] = ndcollection.meta + if ndcollection._aligned_axes is not None: + node["aligned_axes"] = ndcollection._aligned_axes + + return node diff --git a/ndcube/asdf/converters/ndcubesequence_converter.py b/ndcube/asdf/converters/ndcubesequence_converter.py new file mode 100644 index 000000000..ba73e45cf --- /dev/null +++ b/ndcube/asdf/converters/ndcubesequence_converter.py @@ -0,0 +1,25 @@ +from asdf.extension import Converter + + +class NDCubeSequenceConverter(Converter): + tags = ["tag:sunpy.org:ndcube/ndcube/ndcube_sequence-*"] + types = ["ndcube.ndcube_sequence.NDCubeSequence"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.ndcube_sequence import NDCubeSequence + + return NDCubeSequence(node["data"], + meta=node.get("meta"), + common_axis=node.get("common_axis")) + + + def to_yaml_tree(self, ndcseq, tag, ctx): + + node = {} + node["data"] = ndcseq.data + if ndcseq.meta is not None: + node["meta"] = ndcseq.meta + if ndcseq._common_axis is not None: + node["common_axis"] = ndcseq._common_axis + + return node diff --git a/ndcube/asdf/converters/tests/test_ndcollection_converter.py b/ndcube/asdf/converters/tests/test_ndcollection_converter.py new file mode 100644 index 000000000..3dff3bf28 --- /dev/null +++ b/ndcube/asdf/converters/tests/test_ndcollection_converter.py @@ -0,0 +1,51 @@ +import pytest +from gwcs import __version__ as gwcs_version +from packaging.version import Version + +import asdf + +from ndcube.ndcollection import NDCollection +from ndcube.ndcube_sequence import NDCubeSequence +from ndcube.tests.helpers import assert_collections_equal + + +@pytest.fixture +def create_ndcollection_cube(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim): + aligned_axes = ((1, 2), (1, 2), (1, 2)) + cube_collection = NDCollection([("cube0", ndcube_gwcs_3d_ln_lt_l), + ("cube1", ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc), + ("cube2", ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim)], + aligned_axes=aligned_axes) + + return cube_collection + + +@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") +def test_serialization_cube(create_ndcollection_cube, tmp_path): + ndcollection = create_ndcollection_cube + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = ndcollection + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_collections_equal(af["ndcube_gwcs"], ndcollection) +@pytest.fixture +def create_ndcollection_sequence(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim): + + sequence02 = NDCubeSequence([ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim]) + sequence20 = NDCubeSequence([ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim, ndcube_gwcs_3d_ln_lt_l]) + seq_collection = NDCollection([("seq0", sequence02), ("seq1", sequence20)], aligned_axes="all") + return seq_collection + + +@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") +def test_serialization_sequence(create_ndcollection_sequence, tmp_path): + ndcollection = create_ndcollection_sequence + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = ndcollection + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_collections_equal(af["ndcube_gwcs"], ndcollection) diff --git a/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py b/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py new file mode 100644 index 000000000..bf2faaa4a --- /dev/null +++ b/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py @@ -0,0 +1,20 @@ +import pytest +from gwcs import __version__ as gwcs_version +from packaging.version import Version + +import asdf + +from ndcube.ndcube_sequence import NDCubeSequence +from ndcube.tests.helpers import assert_cubesequences_equal + + +@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") +def test_serialization(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc, tmp_path): + file_path = tmp_path / "test.asdf" + ndcseq = NDCubeSequence([ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc], common_axis=1) + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = ndcseq + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_cubesequences_equal(af["ndcube_gwcs"], ndcseq) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 0919b45f7..973e49ba6 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -34,15 +34,16 @@ def get_extensions(): from ndcube.asdf.converters.compoundwcs_converter import CompoundConverter from ndcube.asdf.converters.extracoords_converter import ExtraCoordsConverter from ndcube.asdf.converters.globalcoords_converter import GlobalCoordsConverter + from ndcube.asdf.converters.ndcollection_converter import NDCollectionConverter from ndcube.asdf.converters.ndcube_converter import NDCubeConverter from ndcube.asdf.converters.reorderedwcs_converter import ReorderedConverter from ndcube.asdf.converters.resampled_converter import ResampledConverter + from ndcube.asdf.converters.ndcubesequence_converter import NDCubeSequenceConverter from ndcube.asdf.converters.tablecoord_converter import ( QuantityTableCoordinateConverter, SkyCoordTableCoordinateConverter, TimeTableCoordConverter, ) - ndcube_converters = [ NDCubeConverter(), ExtraCoordsConverter(), @@ -53,6 +54,8 @@ def get_extensions(): ResampledConverter(), ReorderedConverter(), CompoundConverter(), + NDCubeSequenceConverter(), + NDCollectionConverter(), ] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index d9e3a02bf..bcb0b8a76 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -27,8 +27,14 @@ tags: - tag_uri: "tag:sunpy.org:ndcube/resampledwcs-0.1.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/resampledwcs-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcube_sequence-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/reorderedwcs-0.1.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/reorderedwcs-0.1.0" - tag_uri: "tag:sunpy.org:ndcube/compoundwcs-0.1.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/compoundwcs-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcollection-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcollection-0.1.0" diff --git a/ndcube/asdf/resources/schemas/ndcollection-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcollection-0.1.0.yaml new file mode 100644 index 000000000..02c76e418 --- /dev/null +++ b/ndcube/asdf/resources/schemas/ndcollection-0.1.0.yaml @@ -0,0 +1,30 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/ndcollection-0.1.0" + +title: + Represents the ndcube.ndcollection.NDCollection object + +description: + Represents the ndcube ndcube.ndcollection.NDCollection object + +type: object +properties: + keys: + type: array + value: + type: array + items: + - type: object + oneOf: + - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" + - tag: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-0.*" + aligned_axes: + anyOf: + - type: object + - type: string + +required: [keys, value] +additionalProperties: true +... diff --git a/ndcube/asdf/resources/schemas/ndcube_sequence-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube_sequence-0.1.0.yaml new file mode 100644 index 000000000..155665502 --- /dev/null +++ b/ndcube/asdf/resources/schemas/ndcube_sequence-0.1.0.yaml @@ -0,0 +1,25 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/ndcube_sequence-0.1.0" + +title: + Represents the ndcube.ndcube_sequence.NDCubeSequence object + +description: + Represents the ndcube.ndcube_sequence.NDCubeSequence object + +type: object +properties: + data: + type: array + items: + tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" + meta: + type: object + common_axis: + type: integer + +required: [data] +additionalProperties: true +... From 494e2c565e10bebc30f7764cd0840613ed35c8c6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:52:08 +0000 Subject: [PATCH 44/78] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ndcube/asdf/entry_points.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 973e49ba6..8fd907977 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -36,9 +36,9 @@ def get_extensions(): from ndcube.asdf.converters.globalcoords_converter import GlobalCoordsConverter from ndcube.asdf.converters.ndcollection_converter import NDCollectionConverter from ndcube.asdf.converters.ndcube_converter import NDCubeConverter + from ndcube.asdf.converters.ndcubesequence_converter import NDCubeSequenceConverter from ndcube.asdf.converters.reorderedwcs_converter import ReorderedConverter from ndcube.asdf.converters.resampled_converter import ResampledConverter - from ndcube.asdf.converters.ndcubesequence_converter import NDCubeSequenceConverter from ndcube.asdf.converters.tablecoord_converter import ( QuantityTableCoordinateConverter, SkyCoordTableCoordinateConverter, From db8f22d609b54be59c2722a647dcf5041ade7856 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 6 Nov 2024 09:55:58 +0000 Subject: [PATCH 45/78] Minor tweaks --- ndcube/asdf/converters/ndcollection_converter.py | 6 ++---- ndcube/asdf/converters/ndcubesequence_converter.py | 2 -- ndcube/asdf/converters/tests/test_ndcollection_converter.py | 2 ++ 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ndcube/asdf/converters/ndcollection_converter.py b/ndcube/asdf/converters/ndcollection_converter.py index 921365539..810bb151c 100644 --- a/ndcube/asdf/converters/ndcollection_converter.py +++ b/ndcube/asdf/converters/ndcollection_converter.py @@ -12,13 +12,11 @@ def from_yaml_tree(self, node, tag, ctx): aligned_axes = list(node.get("aligned_axes").values()) aligned_axes = tuple(tuple(lst) for lst in aligned_axes) ndcollection = NDCollection(key_value_pairs, - meta=node.get("meta"), - aligned_axes = aligned_axes) + meta=node.get("meta"), + aligned_axes = aligned_axes) return ndcollection - def to_yaml_tree(self, ndcollection, tag, ctx): - node = {} node["keys"] = tuple(ndcollection.keys()) node["value"] = tuple(ndcollection.values()) diff --git a/ndcube/asdf/converters/ndcubesequence_converter.py b/ndcube/asdf/converters/ndcubesequence_converter.py index ba73e45cf..419c69497 100644 --- a/ndcube/asdf/converters/ndcubesequence_converter.py +++ b/ndcube/asdf/converters/ndcubesequence_converter.py @@ -12,9 +12,7 @@ def from_yaml_tree(self, node, tag, ctx): meta=node.get("meta"), common_axis=node.get("common_axis")) - def to_yaml_tree(self, ndcseq, tag, ctx): - node = {} node["data"] = ndcseq.data if ndcseq.meta is not None: diff --git a/ndcube/asdf/converters/tests/test_ndcollection_converter.py b/ndcube/asdf/converters/tests/test_ndcollection_converter.py index 3dff3bf28..19158da4f 100644 --- a/ndcube/asdf/converters/tests/test_ndcollection_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcollection_converter.py @@ -30,6 +30,8 @@ def test_serialization_cube(create_ndcollection_cube, tmp_path): with asdf.open(file_path) as af: assert_collections_equal(af["ndcube_gwcs"], ndcollection) + + @pytest.fixture def create_ndcollection_sequence(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim): From 46215b0e6b212304853f70efabb4c54f1c97157a Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 15 May 2025 11:28:51 +0100 Subject: [PATCH 46/78] Update pytest.ini --- pytest.ini | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pytest.ini b/pytest.ini index a9f8ebb0d..cae4a9c0d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -25,12 +25,9 @@ addopts = -p no:threadexception -m "not mpl_image_compare" --doctest-ignore-import-errors + --doctest-continue-on-failure asdf_schema_tests_enabled = true asdf_schema_root = ndcube/asdf/resources -remote_data_strict = True -doctest_subpackage_requires = - docs/explaining_ndcube/* = numpy>=2.0.0 - --doctest-continue-on-failure mpl-results-path = figure_test_images mpl-use-full-test-name = true remote_data_strict = True From 1b72253def08566d368731d0009355a37b17d17a Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 09:19:57 +0100 Subject: [PATCH 47/78] fix pre-commit --- ndcube/asdf/converters/__init__.py | 0 .../asdf/converters/ndcollection_converter.py | 5 +---- .../asdf/converters/reorderedwcs_converter.py | 9 ++++---- ndcube/asdf/converters/resampled_converter.py | 10 ++++----- .../asdf/converters/tablecoord_converter.py | 17 +++++++------- ndcube/asdf/converters/tests/__init__.py | 0 .../tests/test_ndcollection_converter.py | 22 ++++++++++--------- ndcube/conftest.py | 3 +-- 8 files changed, 32 insertions(+), 34 deletions(-) create mode 100644 ndcube/asdf/converters/__init__.py create mode 100644 ndcube/asdf/converters/tests/__init__.py diff --git a/ndcube/asdf/converters/__init__.py b/ndcube/asdf/converters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ndcube/asdf/converters/ndcollection_converter.py b/ndcube/asdf/converters/ndcollection_converter.py index 810bb151c..85b42e523 100644 --- a/ndcube/asdf/converters/ndcollection_converter.py +++ b/ndcube/asdf/converters/ndcollection_converter.py @@ -11,10 +11,7 @@ def from_yaml_tree(self, node, tag, ctx): key_value_pairs = list(zip(node["keys"], node["value"])) aligned_axes = list(node.get("aligned_axes").values()) aligned_axes = tuple(tuple(lst) for lst in aligned_axes) - ndcollection = NDCollection(key_value_pairs, - meta=node.get("meta"), - aligned_axes = aligned_axes) - return ndcollection + return NDCollection(key_value_pairs, meta=node.get("meta"), aligned_axes=aligned_axes) def to_yaml_tree(self, ndcollection, tag, ctx): node = {} diff --git a/ndcube/asdf/converters/reorderedwcs_converter.py b/ndcube/asdf/converters/reorderedwcs_converter.py index 715c11e2a..19d005370 100644 --- a/ndcube/asdf/converters/reorderedwcs_converter.py +++ b/ndcube/asdf/converters/reorderedwcs_converter.py @@ -8,10 +8,11 @@ class ReorderedConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.wcs.wrappers import ReorderedLowLevelWCS - reorderedwcs = ReorderedLowLevelWCS(wcs=node["wcs"], - pixel_order=node.get("pixel_order"), - world_order=node.get("world_order")) - return reorderedwcs + return ReorderedLowLevelWCS( + wcs=node["wcs"], + pixel_order=node.get("pixel_order"), + world_order=node.get("world_order"), + ) def to_yaml_tree(self, reorderedwcs, tag, ctx): node = {} diff --git a/ndcube/asdf/converters/resampled_converter.py b/ndcube/asdf/converters/resampled_converter.py index 6e23e1616..7a838c6f4 100644 --- a/ndcube/asdf/converters/resampled_converter.py +++ b/ndcube/asdf/converters/resampled_converter.py @@ -8,11 +8,11 @@ class ResampledConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.wcs.wrappers import ResampledLowLevelWCS - resampledwcs = ResampledLowLevelWCS(wcs=node["wcs"], - offset=node.get("offset"), - factor=node.get("factor"), - ) - return resampledwcs + return ResampledLowLevelWCS( + wcs=node["wcs"], + offset=node.get("offset"), + factor=node.get("factor"), + ) def to_yaml_tree(self, resampledwcs, tag, ctx): node = {} diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index b07e630e6..c46829866 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -11,10 +11,12 @@ def from_yaml_tree(self, node, tag, ctx): names = node.get("names") physical_types = node.get("physical_types") reference_time = node.get("reference_time") - timetablecoordinate = TimeTableCoordinate( - node["table"], names=names, physical_types=physical_types, reference_time=reference_time) - - return timetablecoordinate + return TimeTableCoordinate( + node["table"], + names=names, + physical_types=physical_types, + reference_time=reference_time, + ) def to_yaml_tree(self, timetablecoordinate, tag, ctx): node = {} @@ -37,8 +39,7 @@ def from_yaml_tree(self, node, tag, ctx): names = node.get("names") mesh = node.get("mesh") physical_types = node.get("physical_types") - quantitytablecoordinate = QuantityTableCoordinate(*node["table"], - names=names, physical_types=physical_types) + quantitytablecoordinate = QuantityTableCoordinate(*node["table"], names=names, physical_types=physical_types) quantitytablecoordinate.unit = node["unit"] quantitytablecoordinate.mesh = mesh return quantitytablecoordinate @@ -65,9 +66,7 @@ def from_yaml_tree(self, node, tag, ctx): names = node.get("names") mesh = node.get("mesh") physical_types = node.get("physical_types") - skycoordinatetablecoordinate = SkyCoordTableCoordinate(node["table"], mesh=mesh, - names=names, physical_types=physical_types) - return skycoordinatetablecoordinate + return SkyCoordTableCoordinate(node["table"], mesh=mesh, names=names, physical_types=physical_types) def to_yaml_tree(self, skycoordinatetablecoordinate, tag, ctx): node = {} diff --git a/ndcube/asdf/converters/tests/__init__.py b/ndcube/asdf/converters/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ndcube/asdf/converters/tests/test_ndcollection_converter.py b/ndcube/asdf/converters/tests/test_ndcollection_converter.py index 19158da4f..bb1ab72bd 100644 --- a/ndcube/asdf/converters/tests/test_ndcollection_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcollection_converter.py @@ -10,14 +10,18 @@ @pytest.fixture -def create_ndcollection_cube(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim): +def create_ndcollection_cube( + ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim +): aligned_axes = ((1, 2), (1, 2), (1, 2)) - cube_collection = NDCollection([("cube0", ndcube_gwcs_3d_ln_lt_l), - ("cube1", ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc), - ("cube2", ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim)], - aligned_axes=aligned_axes) - - return cube_collection + return NDCollection( + [ + ("cube0", ndcube_gwcs_3d_ln_lt_l), + ("cube1", ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc), + ("cube2", ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim), + ], + aligned_axes=aligned_axes, + ) @pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") @@ -34,11 +38,9 @@ def test_serialization_cube(create_ndcollection_cube, tmp_path): @pytest.fixture def create_ndcollection_sequence(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim): - sequence02 = NDCubeSequence([ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim]) sequence20 = NDCubeSequence([ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim, ndcube_gwcs_3d_ln_lt_l]) - seq_collection = NDCollection([("seq0", sequence02), ("seq1", sequence20)], aligned_axes="all") - return seq_collection + return NDCollection([("seq0", sequence02), ("seq1", sequence20)], aligned_axes="all") @pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") diff --git a/ndcube/conftest.py b/ndcube/conftest.py index cd1f17a03..66d1f0f80 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -42,8 +42,7 @@ def time_lut(shape): base_time = Time('2000-01-01', format='fits', scale='utc') - timestamps = Time([base_time + TimeDelta(60 * i, format='sec') for i in range(shape[0])]) - return timestamps + return Time([base_time + TimeDelta(60 * i, format='sec') for i in range(shape[0])]) def skycoord_2d_lut(shape): total_len = np.prod(shape) From e7c4c04ae62a71c117a0a275e1945fb2af04d918 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 09:21:50 +0100 Subject: [PATCH 48/78] Fix schema --- ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml b/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml index bfecfa023..9d63f0692 100644 --- a/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml @@ -14,9 +14,9 @@ properties: wcs: tag: "tag:stsci.edu:gwcs/wcs-1.*" factor: - tag: "tag:stsci.edu:asdf/core/ndarray-1.0.0" + tag: "tag:stsci.edu:asdf/core/ndarray-1.*" offset: - tag: "tag:stsci.edu:asdf/core/ndarray-1.0.0" + tag: "tag:stsci.edu:asdf/core/ndarray-1.*" required: [wcs] additionalProperties: true From 61cc81d1385139e2d77507c91abafab28f94adce Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 15 May 2025 15:59:30 +0100 Subject: [PATCH 49/78] Fix uncertainty, fits wcs etc --- ndcube/asdf/converters/ndcube_converter.py | 25 +++++++++------- .../tests/test_ndcollection_converter.py | 4 ++- .../converters/tests/test_ndcube_converter.py | 29 ++++--------------- .../asdf/resources/schemas/ndcube-0.1.0.yaml | 6 +++- ndcube/conftest.py | 7 +++++ ndcube/tests/helpers.py | 7 ++++- 6 files changed, 41 insertions(+), 37 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index bc855684d..84246cb8e 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -10,11 +10,14 @@ class NDCubeConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.ndcube import NDCube - ndcube = NDCube(node["data"], - node["wcs"], - meta = node.get("meta"), - mask = node.get("mask"), - unit = node.get("unit")) + ndcube = NDCube( + node["data"], + node["wcs"], + meta=node.get("meta"), + mask=node.get("mask"), + unit=node.get("unit"), + uncertainty=node.get("uncertainty"), + ) if "extra_coords" in node: ndcube._extra_coords = node["extra_coords"] if "global_coords" in node: @@ -34,8 +37,8 @@ def to_yaml_tree(self, ndcube, tag, ctx): Warnings -------- UserWarning - Warns if the NDCube object has attributes 'uncertainty', 'mask', - or 'unit' that are present but not being saved in the ASDF serialization. + Warns if the NDCube object has a 'psf' attribute that will not be + saved in the ASDF serialization. This ensures that users are aware of potentially important information that is not included in the serialized output. """ @@ -54,10 +57,10 @@ def to_yaml_tree(self, ndcube, tag, ctx): node["mask"] = ndcube.mask if ndcube.unit is not None: node["unit"] = ndcube.unit + if ndcube.uncertainty is not None: + node["uncertainty"] = ndcube.uncertainty - attributes = ['uncertainty', 'psf'] - for attr in attributes: - if getattr(ndcube, attr) is not None: - warnings.warn(f"Attribute '{attr}' is present but not being saved in ASDF serialization.", UserWarning) + if getattr(ndcube, 'psf') is not None: + warnings.warn("Attribute 'psf' is present but not being saved in ASDF serialization.", UserWarning) return node diff --git a/ndcube/asdf/converters/tests/test_ndcollection_converter.py b/ndcube/asdf/converters/tests/test_ndcollection_converter.py index bb1ab72bd..389ac085a 100644 --- a/ndcube/asdf/converters/tests/test_ndcollection_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcollection_converter.py @@ -11,7 +11,9 @@ @pytest.fixture def create_ndcollection_cube( - ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim + ndcube_gwcs_3d_ln_lt_l, + ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc, + ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim, ): aligned_axes = ((1, 2), (1, 2), (1, 2)) return NDCollection( diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index 649b08b2e..12b15061d 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -8,26 +8,20 @@ from ndcube.tests.helpers import assert_cubes_equal -@pytest.mark.parametrize("ndc",["ndcube_gwcs_2d_ln_lt_mask", - "ndcube_gwcs_3d_ln_lt_l", - "ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim", - "ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc", - "ndcube_gwcs_3d_rotated", - "ndcube_gwcs_4d_ln_lt_l_t", - "ndcube_gwcs_4d_ln_lt_l_t_unit", - ], indirect=("ndc",)) @pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") -def test_serialization(ndc, tmp_path): +def test_serialization(all_ndcubes, tmp_path): + if not isinstance(all_ndcubes.data, np.ndarray): + pytest.skip("Can't save non-numpy array to ASDF.") + file_path = tmp_path / "test.asdf" with asdf.AsdfFile() as af: - af["ndcube_gwcs"] = ndc + af["ndcube"] = all_ndcubes af.write_to(file_path) with asdf.open(file_path) as af: - assert_cubes_equal(af["ndcube_gwcs"], ndc) + assert_cubes_equal(af["ndcube"], all_ndcubes) -@pytest.mark.xfail(reason="Serialization of sliced ndcube not supported") def test_serialization_sliced_ndcube(ndcube_gwcs_3d_ln_lt_l, tmp_path): sndc = ndcube_gwcs_3d_ln_lt_l[np.s_[0, :, :]] file_path = tmp_path / "test.asdf" @@ -37,14 +31,3 @@ def test_serialization_sliced_ndcube(ndcube_gwcs_3d_ln_lt_l, tmp_path): with asdf.open(file_path) as af: assert_cubes_equal(af["ndcube_gwcs"], sndc) - - -@pytest.mark.xfail(reason="Serialization of ndcube with .wcs attribute as astropy.wcs.wcs.WCS not supported") -def test_serialization_ndcube_wcs(ndcube_3d_ln_lt_l, tmp_path): - file_path = tmp_path / "test.asdf" - with asdf.AsdfFile() as af: - af["ndcube"] = ndcube_3d_ln_lt_l - af.write_to(file_path) - - with asdf.open(file_path) as af: - assert_cubes_equal(af["ndcube"], ndcube_3d_ln_lt_l) diff --git a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml index 0b2a414e0..7be82ee3b 100644 --- a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml @@ -15,7 +15,9 @@ properties: description: "Must be compatible with ASDF serialization/deserialization and supported by NDCube." wcs: anyOf: - - tag: "tag:stsci.edu:gwcs/wcs-1.*" + - tag: "tag:stsci.edu:gwcs/wcs-*" + - tag: "tag:astropy.org:astropy/wcs/wcs-*" + - tag: "tag:astropy.org:astropy/wcs/slicedwcs-*" - tag: "tag:sunpy.org:ndcube/resampledwcs-*" - tag: "tag:sunpy.org:ndcube/reorderedwcs-*" - tag: "tag:sunpy.org:ndcube/compoundwcs-*" @@ -31,6 +33,8 @@ properties: anyOf: - tag: "tag:stsci.edu:asdf/unit/unit-1.*" - tag: "tag:astropy.org:astropy/units/unit-1.*" + uncertainty: + tag: tag:astropy.org:astropy/nddata/uncertainty-1.* required: [data, wcs] additionalProperties: true diff --git a/ndcube/conftest.py b/ndcube/conftest.py index 66d1f0f80..b98402aa4 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -1048,6 +1048,13 @@ def ndcube_1d_l(wcs_1d_l): "ndcube_2d_ln_lt_no_unit_no_unc", "ndcube_2d_uncertainty_no_unit", "ndcube_2d_unit_unc", + "ndcube_gwcs_4d_ln_lt_l_t", + "ndcube_gwcs_4d_ln_lt_l_t_unit", + "ndcube_gwcs_3d_ln_lt_l", + "ndcube_gwcs_3d_rotated", + "ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim", + "ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc", + "ndcube_gwcs_2d_ln_lt_mask", ]) def all_ndcubes(request): """ diff --git a/ndcube/tests/helpers.py b/ndcube/tests/helpers.py index 712b9ba3f..0d7708b6e 100644 --- a/ndcube/tests/helpers.py +++ b/ndcube/tests/helpers.py @@ -192,7 +192,12 @@ def assert_wcs_are_equal(wcs1, wcs2): # SlicedLowLevelWCS vs BaseHighLevelWCS don't have the same pixel_to_world method low_level_wcs1 = wcs1.low_level_wcs if isinstance(wcs1, BaseHighLevelWCS) else wcs1 low_level_wcs2 = wcs2.low_level_wcs if isinstance(wcs2, BaseHighLevelWCS) else wcs2 - np.testing.assert_array_equal(low_level_wcs1.pixel_to_world_values(*random_idx.T), low_level_wcs2.pixel_to_world_values(*random_idx.T)) + np.testing.assert_allclose( + low_level_wcs1.pixel_to_world_values(*random_idx.T), + low_level_wcs2.pixel_to_world_values(*random_idx.T), + atol=1e-17, + rtol=1e-23, + ) def create_sliced_wcs(wcs, item, dim): """ From d9b4b809f24a1f78c5b049501ebcc976630f91d1 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 09:36:27 +0100 Subject: [PATCH 50/78] Allow FITS WCS in more places --- ndcube/asdf/resources/schemas/compoundwcs-0.1.0.yaml | 6 ++++-- ndcube/asdf/resources/schemas/reorderedwcs-0.1.0.yaml | 4 +++- ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ndcube/asdf/resources/schemas/compoundwcs-0.1.0.yaml b/ndcube/asdf/resources/schemas/compoundwcs-0.1.0.yaml index 24028920e..37b9a37e0 100644 --- a/ndcube/asdf/resources/schemas/compoundwcs-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/compoundwcs-0.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/Compoundwcs-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/compoundwcs-0.1.0" title: Represents the ndcube CompoundLowLevelWCS object @@ -14,7 +14,9 @@ properties: wcs: type: array items: - tag: "tag:stsci.edu:gwcs/wcs-1.*" + anyOf: + - tag: "tag:astropy.org:astropy/wcs/wcs-*" + - tag: "tag:stsci.edu:gwcs/wcs-*" mapping: type: array atol: diff --git a/ndcube/asdf/resources/schemas/reorderedwcs-0.1.0.yaml b/ndcube/asdf/resources/schemas/reorderedwcs-0.1.0.yaml index 76e54521b..b2acf85bd 100644 --- a/ndcube/asdf/resources/schemas/reorderedwcs-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/reorderedwcs-0.1.0.yaml @@ -12,7 +12,9 @@ description: type: object properties: wcs: - tag: "tag:stsci.edu:gwcs/wcs-1.*" + anyOf: + - tag: "tag:astropy.org:astropy/wcs/wcs-*" + - tag: "tag:stsci.edu:gwcs/wcs-*" pixel_order: type: array world_order: diff --git a/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml b/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml index 9d63f0692..29de01fe8 100644 --- a/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml @@ -12,7 +12,9 @@ description: type: object properties: wcs: - tag: "tag:stsci.edu:gwcs/wcs-1.*" + anyOf: + - tag: "tag:astropy.org:astropy/wcs/wcs-*" + - tag: "tag:stsci.edu:gwcs/wcs-*" factor: tag: "tag:stsci.edu:asdf/core/ndarray-1.*" offset: From ee48a53eda910f5eedd76c49c6a303467c21cd1e Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 09:55:32 +0100 Subject: [PATCH 51/78] Fix tests while awaiting astropy/asdf-astropy#276 --- .../converters/tests/test_ndcube_converter.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index 12b15061d..a6ade02e3 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -1,6 +1,6 @@ +import gwcs import numpy as np import pytest -from gwcs import __version__ as gwcs_version from packaging.version import Version import asdf @@ -8,8 +8,11 @@ from ndcube.tests.helpers import assert_cubes_equal -@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") -def test_serialization(all_ndcubes, tmp_path): +@pytest.mark.skipif(Version(gwcs.__version__) < Version("0.20"), reason="Requires gwcs>=0.20") +def test_serialization(all_ndcubes, tmp_path, all_ndcubes_names): + if all_ndcubes_names in ("ndcube_4d_ln_lt_l_t", "ndcube_3d_ln_lt_l", "ndcube_3d_rotated"): + pytest.importorskip("asdf_astropy", "0.8.0") + if not isinstance(all_ndcubes.data, np.ndarray): pytest.skip("Can't save non-numpy array to ASDF.") @@ -22,8 +25,14 @@ def test_serialization(all_ndcubes, tmp_path): assert_cubes_equal(af["ndcube"], all_ndcubes) -def test_serialization_sliced_ndcube(ndcube_gwcs_3d_ln_lt_l, tmp_path): - sndc = ndcube_gwcs_3d_ln_lt_l[np.s_[0, :, :]] +@pytest.mark.parametrize("expected_cube", + ["ndcube_gwcs_3d_ln_lt_l", "ndcube_3d_ln_lt_l"], + indirect=True) +def test_serialization_sliced_ndcube(expected_cube, tmp_path): + # This needs 0.8.0 of asdf_astropy to be able to save gwcs and to save array_shape on WCS + pytest.importorskip("asdf_astropy", "0.8.0") + + sndc = expected_cube[np.s_[0, :, :]] file_path = tmp_path / "test.asdf" with asdf.AsdfFile() as af: af["ndcube_gwcs"] = sndc From 675bf1daf582a6f1e7d6a31a245ac5c609335b3c Mon Sep 17 00:00:00 2001 From: Drew Leonard Date: Thu, 22 May 2025 14:01:04 +0100 Subject: [PATCH 52/78] Rough start at basic docs --- docs/explaining_ndcube/asdf_serialization.rst | 34 +++++++++++++++++++ docs/explaining_ndcube/index.rst | 1 + 2 files changed, 35 insertions(+) create mode 100644 docs/explaining_ndcube/asdf_serialization.rst diff --git a/docs/explaining_ndcube/asdf_serialization.rst b/docs/explaining_ndcube/asdf_serialization.rst new file mode 100644 index 000000000..497e6c585 --- /dev/null +++ b/docs/explaining_ndcube/asdf_serialization.rst @@ -0,0 +1,34 @@ +.. _asdf_serialization + +************************* +Saving ND objects to ASDF +************************* + +The `Advanced Scientific Data Format (ASDF)`_ is an extensible format for validating and saving complex scientific data along with its metadata. +`ndcube` provides schemas and converters for all the ND objects (NDCube, NDCubeSequence and NDCollection) as well as for various WCS and table objects required by them. +To make use of these, simply save an ND object to an ASDF file and it will be correctly serialized. + +.. code-block:: python + >>> import numpy as np + >>> import asdf + >>> import astropy.wcs + >>> from ndcube import NDCube + >>> + >>> # Define data array. + >>> data = np.random.rand(4, 4, 5) + >>> + >>> # Define WCS transformations in an astropy WCS object. + >>> wcs = astropy.wcs.WCS(naxis=3) + >>> wcs.wcs.ctype = 'WAVE', 'HPLT-TAN', 'HPLN-TAN' + >>> wcs.wcs.cunit = 'Angstrom', 'deg', 'deg' + >>> wcs.wcs.cdelt = 0.2, 0.5, 0.4 + >>> wcs.wcs.crpix = 0, 2, 2 + >>> wcs.wcs.crval = 10, 0.5, 1 + >>> wcs.wcs.cname = 'wavelength', 'HPC lat', 'HPC lon' + >>> + >>> # Now instantiate the NDCube + >>> my_cube = NDCube(data, wcs=wcs) + >>> + >>> # Save the NDCube to an ASDF file + >>> with asdf.AsdfFile(tree={"mycube": my_cube}) as f: + >>> f.write_to("somefile.asdf") diff --git a/docs/explaining_ndcube/index.rst b/docs/explaining_ndcube/index.rst index a28801d00..e9ab9a330 100644 --- a/docs/explaining_ndcube/index.rst +++ b/docs/explaining_ndcube/index.rst @@ -14,3 +14,4 @@ Explaining ``ndcube`` tabular_coordinates reproject visualization + asdf_serialization From 1cf3b1870e7ae4cc4bd83d6879ca7a0310e45e85 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 10:12:16 +0100 Subject: [PATCH 53/78] Update ndcube/asdf/converters/compoundwcs_converter.py --- ndcube/asdf/converters/compoundwcs_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndcube/asdf/converters/compoundwcs_converter.py b/ndcube/asdf/converters/compoundwcs_converter.py index 3cbd9430b..214fe59e6 100644 --- a/ndcube/asdf/converters/compoundwcs_converter.py +++ b/ndcube/asdf/converters/compoundwcs_converter.py @@ -8,7 +8,7 @@ class CompoundConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.wcs.wrappers import CompoundLowLevelWCS - return(CompoundLowLevelWCS(*node["wcs"], mapping=node.get("mapping"), pixel_atol=node.get("atol"))) + return CompoundLowLevelWCS(*node["wcs"], mapping=node.get("mapping"), pixel_atol=node.get("atol")) def to_yaml_tree(self, compoundwcs, tag, ctx): node = {} From e70a736547a1a205bf5a38f26f20134236414a69 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 11:15:11 +0100 Subject: [PATCH 54/78] More details on what's supported --- docs/conf.py | 3 +- docs/explaining_ndcube/asdf_serialization.rst | 51 +++++++++++++++---- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f72cdbce5..806c7923c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -95,7 +95,8 @@ 'sunpy': ('https://docs.sunpy.org/en/stable/', None), 'mpl_animators': ('https://docs.sunpy.org/projects/mpl-animators/en/stable/', None), 'gwcs': ('https://gwcs.readthedocs.io/en/stable/', None), - 'reproject': ("https://reproject.readthedocs.io/en/stable/", None) + 'reproject': ("https://reproject.readthedocs.io/en/stable/", None), + 'asdf': ("https://www.asdf-format.org/projects/asdf/en/stable/", None), } # -- Options for HTML output ------------------------------------------------- diff --git a/docs/explaining_ndcube/asdf_serialization.rst b/docs/explaining_ndcube/asdf_serialization.rst index 497e6c585..c26127259 100644 --- a/docs/explaining_ndcube/asdf_serialization.rst +++ b/docs/explaining_ndcube/asdf_serialization.rst @@ -1,22 +1,26 @@ -.. _asdf_serialization +.. _asdf_serialization: ************************* Saving ND objects to ASDF ************************* -The `Advanced Scientific Data Format (ASDF)`_ is an extensible format for validating and saving complex scientific data along with its metadata. -`ndcube` provides schemas and converters for all the ND objects (NDCube, NDCubeSequence and NDCollection) as well as for various WCS and table objects required by them. +:ref:`asdf` is an extensible format for validating and saving complex scientific data along with its metadata. +`ndcube` provides schemas and converters for all the ND objects (`~ndcube.NDCube`, `~ndcube.NDCubeSequence` and `~ndcube.NDCollection`) as well as for various WCS and table objects required by them. To make use of these, simply save an ND object to an ASDF file and it will be correctly serialized. +ASDF files save a "tree" which is a `dict`. +You can save any number of cubes in your ASDF by adding them to the dictionary. + +.. expanding-code-block:: python + :summary: Click to reveal/hide instantiation of the NDCube. -.. code-block:: python >>> import numpy as np >>> import asdf >>> import astropy.wcs >>> from ndcube import NDCube - >>> + >>> >>> # Define data array. >>> data = np.random.rand(4, 4, 5) - >>> + >>> >>> # Define WCS transformations in an astropy WCS object. >>> wcs = astropy.wcs.WCS(naxis=3) >>> wcs.wcs.ctype = 'WAVE', 'HPLT-TAN', 'HPLN-TAN' @@ -25,10 +29,37 @@ To make use of these, simply save an ND object to an ASDF file and it will be co >>> wcs.wcs.crpix = 0, 2, 2 >>> wcs.wcs.crval = 10, 0.5, 1 >>> wcs.wcs.cname = 'wavelength', 'HPC lat', 'HPC lon' - >>> + >>> >>> # Now instantiate the NDCube >>> my_cube = NDCube(data, wcs=wcs) - >>> - >>> # Save the NDCube to an ASDF file - >>> with asdf.AsdfFile(tree={"mycube": my_cube}) as f: + + +.. code-block:: python + + >>> my_tree = {"mycube": my_cube} + >>> with asdf.AsdfFile(tree=my_tree) as f: >>> f.write_to("somefile.asdf") + + +What's Supported and What isn't +=============================== + +We aim to support all features of `ndcube` when saving and loading to ASDF. +However, because it is possible to create `ndcube` objects with many different components (for example dask arrays) which aren't part of the `ndcube` package these may not be supported. +Many common components of `ndcube` classes are supported in the `asdf_astropy `__ package, such as `astropy.wcs.WCS`, `astropy.wcs.wcsapi.SlicedLowLevelWCS` and uncertainty objects. + +The only component of the `ndcube.NDCube` class which is never saved is the ``.psf`` attribute. + +`ndcube` implements converters and schemas for the following objects: + +* `~ndcube.NDCube` +* `~ndcube.NDCubeSequence` +* `~ndcube.NDCollection` +* `~ndcube.GlobalCoords` +* `~ndcube.ExtraCoords` +* `~ndcube.extra_coords.TimeTableCoordinate` +* `~ndcube.extra_coords.QuantityTableCoordinate` +* `~ndcube.extra_coords.SkyCoordTableCoordinate` +* `~ndcube.wcs.wrappers.ReorderedLowLevelWCS` +* `~ndcube.wcs.wrappers.ResampledLowLevelWCS` +* `~ndcube.wcs.wrappers.CompoundLowLevelWCS` From 1d92daf01db33033e0944dc9a8630a9731636e56 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 11:26:12 +0100 Subject: [PATCH 55/78] Add support for MultipleTableCoordinate --- .../asdf/converters/tablecoord_converter.py | 18 +++++++++++ ndcube/asdf/entry_points.py | 2 ++ .../resources/manifests/ndcube-0.1.0.yaml | 3 ++ .../multipletablecoordinate-1.0.0.yaml | 31 +++++++++++++++++++ ndcube/extra_coords/table_coord.py | 8 ++++- 5 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index c46829866..c008deab9 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -77,3 +77,21 @@ def to_yaml_tree(self, skycoordinatetablecoordinate, tag, ctx): node["physical_types"] = skycoordinatetablecoordinate.physical_types return node + + +class MultipleTableCoordinateConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/multipletablecoordinate-*"] + types = ["ndcube.extra_coords.table_coord.MultipleTableCoordinate"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.table_coord import MultipleTableCoordinate + + mtc = MultipleTableCoordinate(node["table_coords"]) + mtc._dropped_coords = node["dropped_coords"] + return mtc + + def to_yaml_tree(self, multipletablecoordinate, tag, ctx): + node = {} + node["table_coords"] = multipletablecoordinate._table_coords + node["dropped_coords"] = multipletablecoordinate._dropped_coords + return node diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 8fd907977..4d7d906a4 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -40,6 +40,7 @@ def get_extensions(): from ndcube.asdf.converters.reorderedwcs_converter import ReorderedConverter from ndcube.asdf.converters.resampled_converter import ResampledConverter from ndcube.asdf.converters.tablecoord_converter import ( + MultipleTableCoordinateConverter, QuantityTableCoordinateConverter, SkyCoordTableCoordinateConverter, TimeTableCoordConverter, @@ -50,6 +51,7 @@ def get_extensions(): TimeTableCoordConverter(), QuantityTableCoordinateConverter(), SkyCoordTableCoordinateConverter(), + MultipleTableCoordinateConverter(), GlobalCoordsConverter(), ResampledConverter(), ReorderedConverter(), diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index bcb0b8a76..f27a2c647 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -21,6 +21,9 @@ tags: - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-0.1.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/skycoordtablecoordinate-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/multipletablecoordinate-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/multipletablecoordinate-1.0.0" + - tag_uri: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.1.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/global_coords-0.1.0" diff --git a/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml b/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml new file mode 100644 index 000000000..86ae16067 --- /dev/null +++ b/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml @@ -0,0 +1,31 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/multipletablecoordinate-1.0.0" + +title: + Represents the MultipleTableCoordinate class + +description: + Represents the MultipleTableCoordinate class + +type: object +properties: + table_coords: + type: array + items: + anyOf: + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-*" + dropped_coords: + type: array + items: + anyOf: + - tag: "tag:stsci.edu:asdf/unit/quantity-*" + - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" + - tag: "tag:stsci.edu:asdf/time/time-*" + +required: ["table_coords", "dropped_coords"] +additionalProperties: False +... diff --git a/ndcube/extra_coords/table_coord.py b/ndcube/extra_coords/table_coord.py index 25dd58435..e975393d4 100644 --- a/ndcube/extra_coords/table_coord.py +++ b/ndcube/extra_coords/table_coord.py @@ -22,7 +22,13 @@ except ImportError: pass -__all__ = ["BaseTableCoordinate", "MultipleTableCoordinate", 'QuantityTableCoordinate', 'SkyCoordTableCoordinate', 'TimeTableCoordinate'] +__all__ = [ + "BaseTableCoordinate", + "MultipleTableCoordinate", + "QuantityTableCoordinate", + "SkyCoordTableCoordinate", + "TimeTableCoordinate", +] class Length1Tabular(_Tabular): From 50a3cdd2b06ca14d5fb1ee2e585b39ad27a3a756 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 11:33:16 +0100 Subject: [PATCH 56/78] More docs and test fixes --- docs/explaining_ndcube/asdf_serialization.rst | 10 +++++----- ndcube/conftest.py | 8 ++++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/explaining_ndcube/asdf_serialization.rst b/docs/explaining_ndcube/asdf_serialization.rst index c26127259..ac46700cf 100644 --- a/docs/explaining_ndcube/asdf_serialization.rst +++ b/docs/explaining_ndcube/asdf_serialization.rst @@ -17,10 +17,10 @@ You can save any number of cubes in your ASDF by adding them to the dictionary. >>> import asdf >>> import astropy.wcs >>> from ndcube import NDCube - >>> + >>> # Define data array. >>> data = np.random.rand(4, 4, 5) - >>> + >>> # Define WCS transformations in an astropy WCS object. >>> wcs = astropy.wcs.WCS(naxis=3) >>> wcs.wcs.ctype = 'WAVE', 'HPLT-TAN', 'HPLN-TAN' @@ -29,7 +29,7 @@ You can save any number of cubes in your ASDF by adding them to the dictionary. >>> wcs.wcs.crpix = 0, 2, 2 >>> wcs.wcs.crval = 10, 0.5, 1 >>> wcs.wcs.cname = 'wavelength', 'HPC lat', 'HPC lon' - >>> + >>> # Now instantiate the NDCube >>> my_cube = NDCube(data, wcs=wcs) @@ -37,8 +37,8 @@ You can save any number of cubes in your ASDF by adding them to the dictionary. .. code-block:: python >>> my_tree = {"mycube": my_cube} - >>> with asdf.AsdfFile(tree=my_tree) as f: - >>> f.write_to("somefile.asdf") + >>> with asdf.AsdfFile(tree=my_tree) as f: # doctest: +SKIP + ... f.write_to("somefile.asdf") # doctest: +SKIP What's Supported and What isn't diff --git a/ndcube/conftest.py b/ndcube/conftest.py index b98402aa4..9dc7e652f 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -1056,11 +1056,15 @@ def ndcube_1d_l(wcs_1d_l): "ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc", "ndcube_gwcs_2d_ln_lt_mask", ]) -def all_ndcubes(request): +def all_ndcubes_names(request): + return request.param + +@pytest.fixture +def all_ndcubes(request, all_ndcubes_names): """ All the above ndcube fixtures in order. """ - return request.getfixturevalue(request.param) + return request.getfixturevalue(all_ndcubes_names) @pytest.fixture From ce0556d811be8f5be25ca3b43f7d1c259b35cfa9 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 12:24:24 +0100 Subject: [PATCH 57/78] Add support for MTC and tests --- docs/explaining_ndcube/asdf_serialization.rst | 1 + .../asdf/converters/tablecoord_converter.py | 11 ++- .../converters/tests/test_table_coordinate.py | 84 +++++++++++++++++++ .../multipletablecoordinate-1.0.0.yaml | 6 +- ndcube/conftest.py | 64 ++++++++++++++ .../tests/test_lookup_table_coord.py | 62 -------------- 6 files changed, 159 insertions(+), 69 deletions(-) create mode 100644 ndcube/asdf/converters/tests/test_table_coordinate.py diff --git a/docs/explaining_ndcube/asdf_serialization.rst b/docs/explaining_ndcube/asdf_serialization.rst index ac46700cf..94e4641f7 100644 --- a/docs/explaining_ndcube/asdf_serialization.rst +++ b/docs/explaining_ndcube/asdf_serialization.rst @@ -60,6 +60,7 @@ The only component of the `ndcube.NDCube` class which is never saved is the ``.p * `~ndcube.extra_coords.TimeTableCoordinate` * `~ndcube.extra_coords.QuantityTableCoordinate` * `~ndcube.extra_coords.SkyCoordTableCoordinate` +* `~ndcube.extra_coords.MultipleTableCoordinate` * `~ndcube.wcs.wrappers.ReorderedLowLevelWCS` * `~ndcube.wcs.wrappers.ResampledLowLevelWCS` * `~ndcube.wcs.wrappers.CompoundLowLevelWCS` diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py index c008deab9..755c6e2f3 100644 --- a/ndcube/asdf/converters/tablecoord_converter.py +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -21,7 +21,8 @@ def from_yaml_tree(self, node, tag, ctx): def to_yaml_tree(self, timetablecoordinate, tag, ctx): node = {} node["table"] = timetablecoordinate.table - node["names"] = timetablecoordinate.names + if timetablecoordinate.names: + node["names"] = timetablecoordinate.names if timetablecoordinate.physical_types is not None: node["physical_types"] = timetablecoordinate.physical_types node["reference_time"] = timetablecoordinate.reference_time @@ -48,7 +49,8 @@ def to_yaml_tree(self, quantitytablecoordinate, tag, ctx): node = {} node["unit"] = quantitytablecoordinate.unit node["table"] = quantitytablecoordinate.table - node["names"] = quantitytablecoordinate.names + if quantitytablecoordinate.names: + node["names"] = quantitytablecoordinate.names node["mesh"] = quantitytablecoordinate.mesh if quantitytablecoordinate.physical_types is not None: node["physical_types"] = quantitytablecoordinate.physical_types @@ -71,7 +73,8 @@ def from_yaml_tree(self, node, tag, ctx): def to_yaml_tree(self, skycoordinatetablecoordinate, tag, ctx): node = {} node["table"] = skycoordinatetablecoordinate.table - node["names"] = skycoordinatetablecoordinate.names + if skycoordinatetablecoordinate.names: + node["names"] = skycoordinatetablecoordinate.names node["mesh"] = skycoordinatetablecoordinate.mesh if skycoordinatetablecoordinate.physical_types is not None: node["physical_types"] = skycoordinatetablecoordinate.physical_types @@ -86,7 +89,7 @@ class MultipleTableCoordinateConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.extra_coords.table_coord import MultipleTableCoordinate - mtc = MultipleTableCoordinate(node["table_coords"]) + mtc = MultipleTableCoordinate(*node["table_coords"]) mtc._dropped_coords = node["dropped_coords"] return mtc diff --git a/ndcube/asdf/converters/tests/test_table_coordinate.py b/ndcube/asdf/converters/tests/test_table_coordinate.py new file mode 100644 index 000000000..e48a14638 --- /dev/null +++ b/ndcube/asdf/converters/tests/test_table_coordinate.py @@ -0,0 +1,84 @@ +import pytest + +import asdf +import astropy.units as u +from astropy.coordinates import SkyCoord + +from ndcube.extra_coords import MultipleTableCoordinate + + +@pytest.fixture +def lut(request): + return request.getfixturevalue(request.param) + + +def assert_table_coord_equal(test_table, expected_table): + test_table = test_table.table + expected_table = expected_table.table + if not isinstance(expected_table, tuple): + test_table = (test_table,) + expected_table = (expected_table,) + for test_tab, ex_tab in zip(test_table, expected_table): + if ex_tab.isscalar: + assert test_tab == ex_tab + elif isinstance(ex_tab, SkyCoord): + assert u.allclose(ex_tab.spherical.lat, test_tab.spherical.lat) + assert u.allclose(ex_tab.spherical.lon, test_tab.spherical.lon) + else: + assert all(test_tab == ex_tab) + + +@pytest.mark.parametrize("lut", + [ + "lut_1d_distance", + "lut_3d_distance_mesh", + "lut_1d_skycoord_no_mesh", + "lut_2d_skycoord_no_mesh", + "lut_2d_skycoord_mesh", + "lut_3d_skycoord_mesh", + "lut_1d_time", + "lut_1d_wave", + ], indirect=True) +def test_serialize(lut, tmp_path): + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["lut"] = lut + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_table_coord_equal(af["lut"], lut) + + +def assert_mtc_equal(test_mtc, expected_mtc): + assert len(test_mtc._table_coords) == len(expected_mtc._table_coords) + assert len(test_mtc._dropped_coords) == len(expected_mtc._dropped_coords) + + for (test_tc, expected_tc) in zip(test_mtc._table_coords, expected_mtc._table_coords): + assert_table_coord_equal(test_tc, expected_tc) + + for (test_tc, expected_tc) in zip(test_mtc._dropped_coords, expected_mtc._dropped_coords): + assert_table_coord_equal(test_tc, expected_tc) + + +def test_serialize_multiple_coord(lut_1d_distance, lut_1d_time, tmp_path): + mtc = MultipleTableCoordinate(lut_1d_distance, lut_1d_time) + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["lut"] = mtc + af.write_to(file_path) + + with asdf.open(file_path) as af: + new_mtc = af["lut"] + assert_mtc_equal(new_mtc, mtc) + + +def test_serialize_sliced_multiple_coord(lut_1d_distance, lut_1d_time, tmp_path): + mtc = MultipleTableCoordinate(lut_1d_distance, lut_1d_time)[0, :] + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["lut"] = mtc + af.write_to(file_path) + + with asdf.open(file_path) as af: + new_mtc = af["lut"] + assert_mtc_equal(new_mtc, mtc) diff --git a/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml b/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml index 86ae16067..1e58f4d10 100644 --- a/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml @@ -22,9 +22,9 @@ properties: type: array items: anyOf: - - tag: "tag:stsci.edu:asdf/unit/quantity-*" - - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" - - tag: "tag:stsci.edu:asdf/time/time-*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-*" required: ["table_coords", "dropped_coords"] additionalProperties: False diff --git a/ndcube/conftest.py b/ndcube/conftest.py index 9dc7e652f..aa3c71fa8 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -20,6 +20,11 @@ from astropy.wcs import WCS from ndcube import ExtraCoords, GlobalCoords, NDCube, NDCubeSequence, NDMeta +from ndcube.extra_coords.table_coord import ( + QuantityTableCoordinate, + SkyCoordTableCoordinate, + TimeTableCoordinate, +) from ndcube.tests import helpers # Force MPL to use non-gui backends for testing. @@ -1138,6 +1143,65 @@ def ndcubesequence_3c_l_ln_lt_cax1(wcs_3d_lt_ln_l): return NDCubeSequence([cube1, cube2, cube3], common_axis=common_axis) +################################################################################ +# Table Coordinates +################################################################################ + + +@pytest.fixture +def lut_1d_distance(): + lookup_table = u.Quantity(np.arange(10) * u.km) + return QuantityTableCoordinate(lookup_table, names='x') + + +@pytest.fixture +def lut_3d_distance_mesh(): + lookup_table = (u.Quantity(np.arange(10) * u.km), + u.Quantity(np.arange(10, 20) * u.km), + u.Quantity(np.arange(20, 30) * u.km)) + + return QuantityTableCoordinate(*lookup_table, names=['x', 'y', 'z']) + + +@pytest.fixture +def lut_1d_skycoord_no_mesh(): + sc = SkyCoord(range(10), range(10), unit=u.deg) + return SkyCoordTableCoordinate(sc, mesh=False, names=['lon', 'lat']) + + +@pytest.fixture +def lut_2d_skycoord_no_mesh(): + data = np.arange(9).reshape(3, 3), np.arange(9, 18).reshape(3, 3) + sc = SkyCoord(*data, unit=u.deg) + return SkyCoordTableCoordinate(sc, mesh=False) + + +@pytest.fixture +def lut_2d_skycoord_mesh(): + sc = SkyCoord(range(10), range(10), unit=u.deg) + return SkyCoordTableCoordinate(sc, mesh=True) + + +@pytest.fixture +def lut_3d_skycoord_mesh(): + sc = SkyCoord(range(10), range(10), range(10), unit=(u.deg, u.deg, u.AU)) + return SkyCoordTableCoordinate(sc, mesh=True) + + +@pytest.fixture +def lut_1d_time(): + data = Time(["2011-01-01T00:00:00", + "2011-01-01T00:00:10", + "2011-01-01T00:00:20", + "2011-01-01T00:00:30"], format="isot") + return TimeTableCoordinate(data, names='time', physical_types='time') + + +@pytest.fixture +def lut_1d_wave(): + # TODO: Make this into a SpectralCoord object + return QuantityTableCoordinate(range(10) * u.nm) + def pytest_runtest_teardown(item): # Clear the pyplot figure stack if it is not empty after the test diff --git a/ndcube/extra_coords/tests/test_lookup_table_coord.py b/ndcube/extra_coords/tests/test_lookup_table_coord.py index 36974ac3c..35ae13886 100644 --- a/ndcube/extra_coords/tests/test_lookup_table_coord.py +++ b/ndcube/extra_coords/tests/test_lookup_table_coord.py @@ -15,68 +15,6 @@ ) -@pytest.fixture -def lut_1d_distance(): - lookup_table = u.Quantity(np.arange(10) * u.km) - return QuantityTableCoordinate(lookup_table, names='x') - - -@pytest.fixture -def lut_3d_distance_mesh(): - lookup_table = (u.Quantity(np.arange(10) * u.km), - u.Quantity(np.arange(10, 20) * u.km), - u.Quantity(np.arange(20, 30) * u.km)) - - return QuantityTableCoordinate(*lookup_table, names=['x', 'y', 'z']) - - -@pytest.fixture -def lut_2d_distance_no_mesh(): - # Fixture is broken and raises: Currently all tables must be 1-D - lookup_table = np.arange(9).reshape(3, 3) * u.km, np.arange(9, 18).reshape(3, 3) * u.km - return QuantityTableCoordinate(*lookup_table, mesh=False) - - -@pytest.fixture -def lut_1d_skycoord_no_mesh(): - sc = SkyCoord(range(10), range(10), unit=u.deg) - return SkyCoordTableCoordinate(sc, mesh=False, names=['lon', 'lat']) - - -@pytest.fixture -def lut_2d_skycoord_no_mesh(): - data = np.arange(9).reshape(3, 3), np.arange(9, 18).reshape(3, 3) - sc = SkyCoord(*data, unit=u.deg) - return SkyCoordTableCoordinate(sc, mesh=False) - - -@pytest.fixture -def lut_2d_skycoord_mesh(): - sc = SkyCoord(range(10), range(10), unit=u.deg) - return SkyCoordTableCoordinate(sc, mesh=True) - - -@pytest.fixture -def lut_3d_skycoord_mesh(): - sc = SkyCoord(range(10), range(10), range(10), unit=(u.deg, u.deg, u.AU)) - return SkyCoordTableCoordinate(sc, mesh=True) - - -@pytest.fixture -def lut_1d_time(): - data = Time(["2011-01-01T00:00:00", - "2011-01-01T00:00:10", - "2011-01-01T00:00:20", - "2011-01-01T00:00:30"], format="isot") - return TimeTableCoordinate(data, names='time', physical_types='time') - - -@pytest.fixture -def lut_1d_wave(): - # TODO: Make this into a SpectralCoord object - return QuantityTableCoordinate(range(10) * u.nm) - - def test_exceptions(): with pytest.raises(TypeError) as ei: QuantityTableCoordinate(u.Quantity([1, 2, 3], u.nm), [1, 2, 3]) From 350d7ceea59f272010de51d0f4d167353fecb244 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 12:36:31 +0100 Subject: [PATCH 58/78] Bump versions of all schemas etc --- .../asdf/converters/compoundwcs_converter.py | 2 +- ndcube/asdf/entry_points.py | 2 +- .../resources/manifests/ndcube-0.1.0.yaml | 43 ------------------- .../resources/manifests/ndcube-1.0.0.yaml | 43 +++++++++++++++++++ ...dwcs-0.1.0.yaml => compoundwcs-1.0.0.yaml} | 2 +- ...rds-0.1.0.yaml => extra_coords-1.0.0.yaml} | 10 ++--- ...ds-0.1.0.yaml => global_coords-1.0.0.yaml} | 4 +- ...ion-0.1.0.yaml => ndcollection-1.0.0.yaml} | 6 +-- .../{ndcube-0.1.0.yaml => ndcube-1.0.0.yaml} | 6 +-- ...-0.1.0.yaml => ndcube_sequence-1.0.0.yaml} | 4 +- ...aml => quantitytablecoordinate-1.0.0.yaml} | 2 +- ...wcs-0.1.0.yaml => reorderedwcs-1.0.0.yaml} | 2 +- ...wcs-0.1.0.yaml => resampledwcs-1.0.0.yaml} | 2 +- ...aml => skycoordtablecoordinate-1.0.0.yaml} | 2 +- ....0.yaml => timetablecoordinate-1.0.0.yaml} | 2 +- 15 files changed, 66 insertions(+), 66 deletions(-) delete mode 100644 ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml create mode 100644 ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml rename ndcube/asdf/resources/schemas/{compoundwcs-0.1.0.yaml => compoundwcs-1.0.0.yaml} (89%) rename ndcube/asdf/resources/schemas/{extra_coords-0.1.0.yaml => extra_coords-1.0.0.yaml} (78%) rename ndcube/asdf/resources/schemas/{global_coords-0.1.0.yaml => global_coords-1.0.0.yaml} (83%) rename ndcube/asdf/resources/schemas/{ndcollection-0.1.0.yaml => ndcollection-1.0.0.yaml} (81%) rename ndcube/asdf/resources/schemas/{ndcube-0.1.0.yaml => ndcube-1.0.0.yaml} (89%) rename ndcube/asdf/resources/schemas/{ndcube_sequence-0.1.0.yaml => ndcube_sequence-1.0.0.yaml} (77%) rename ndcube/asdf/resources/schemas/{quantitytablecoordinate-0.1.0.yaml => quantitytablecoordinate-1.0.0.yaml} (89%) rename ndcube/asdf/resources/schemas/{reorderedwcs-0.1.0.yaml => reorderedwcs-1.0.0.yaml} (88%) rename ndcube/asdf/resources/schemas/{resampledwcs-0.1.0.yaml => resampledwcs-1.0.0.yaml} (89%) rename ndcube/asdf/resources/schemas/{skycoordtablecoordinate-0.1.0.yaml => skycoordtablecoordinate-1.0.0.yaml} (86%) rename ndcube/asdf/resources/schemas/{timetablecoordinate-0.1.0.yaml => timetablecoordinate-1.0.0.yaml} (86%) diff --git a/ndcube/asdf/converters/compoundwcs_converter.py b/ndcube/asdf/converters/compoundwcs_converter.py index 214fe59e6..1f733fb4a 100644 --- a/ndcube/asdf/converters/compoundwcs_converter.py +++ b/ndcube/asdf/converters/compoundwcs_converter.py @@ -2,7 +2,7 @@ class CompoundConverter(Converter): - tags = ["tag:sunpy.org:ndcube/compoundwcs-0.1.0"] + tags = ["tag:sunpy.org:ndcube/compoundwcs-*"] types = ["ndcube.wcs.wrappers.compound_wcs.CompoundLowLevelWCS"] def from_yaml_tree(self, node, tag, ctx): diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 4d7d906a4..dd175c73b 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -59,7 +59,7 @@ def get_extensions(): NDCubeSequenceConverter(), NDCollectionConverter(), ] - _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" + _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0" return [ ManifestExtension.from_uri(_manifest_uri, converters=ndcube_converters) diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml deleted file mode 100644 index f27a2c647..000000000 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ /dev/null @@ -1,43 +0,0 @@ -%YAML 1.1 ---- -id: asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0 -extension_uri: asdf://sunpy.org/extensions/ndcube-0.1.0 -title: NDCube ASDF Manifest -description: ASDF schemas and tags for NDCube classes. - -tags: - - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcube-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcube-0.1.0" - - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/extra_coords-0.1.0" - - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/timetablecoordinate-0.1.0" - - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/quantitytablecoordinate-0.1.0" - - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/skycoordtablecoordinate-0.1.0" - - - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/multipletablecoordinate-1.0.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/multipletablecoordinate-1.0.0" - - - tag_uri: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/global_coords-0.1.0" - - - tag_uri: "tag:sunpy.org:ndcube/resampledwcs-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/resampledwcs-0.1.0" - - - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcube_sequence-0.1.0" - - - tag_uri: "tag:sunpy.org:ndcube/reorderedwcs-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/reorderedwcs-0.1.0" - - - tag_uri: "tag:sunpy.org:ndcube/compoundwcs-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/compoundwcs-0.1.0" - - - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcollection-0.1.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcollection-0.1.0" diff --git a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml new file mode 100644 index 000000000..65a4bdfb4 --- /dev/null +++ b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml @@ -0,0 +1,43 @@ +%YAML 1.1 +--- +id: asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0 +extension_uri: asdf://sunpy.org/extensions/ndcube-1.0.0 +title: NDCube ASDF Manifest +description: ASDF schemas and tags for NDCube classes. + +tags: + - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcube-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcube-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/extra_coords-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/timetablecoordinate-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/quantitytablecoordinate-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/skycoordtablecoordinate-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/multipletablecoordinate-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/multipletablecoordinate-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/global_coords/globalcoords-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/global_coords-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/resampledwcs-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/resampledwcs-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcube_sequence-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/reorderedwcs-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/reorderedwcs-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/compoundwcs-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/compoundwcs-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcollection-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcollection-1.0.0" diff --git a/ndcube/asdf/resources/schemas/compoundwcs-0.1.0.yaml b/ndcube/asdf/resources/schemas/compoundwcs-1.0.0.yaml similarity index 89% rename from ndcube/asdf/resources/schemas/compoundwcs-0.1.0.yaml rename to ndcube/asdf/resources/schemas/compoundwcs-1.0.0.yaml index 37b9a37e0..a0674a945 100644 --- a/ndcube/asdf/resources/schemas/compoundwcs-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/compoundwcs-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/compoundwcs-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/compoundwcs-1.0.0" title: Represents the ndcube CompoundLowLevelWCS object diff --git a/ndcube/asdf/resources/schemas/extra_coords-0.1.0.yaml b/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml similarity index 78% rename from ndcube/asdf/resources/schemas/extra_coords-0.1.0.yaml rename to ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml index 6b16e3c9c..86b76a308 100644 --- a/ndcube/asdf/resources/schemas/extra_coords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/extra_coords-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/extra_coords-1.0.0" title: Represents the ndcube ExtraCoords object @@ -24,13 +24,13 @@ properties: - type: number - type: array - oneOf: - - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-0.*" - - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-0.*" - - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-0.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-1.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-1.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-1.*" dropped_tables: type: array ndcube: - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" + tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" required: [ndcube] additionalProperties: false diff --git a/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml b/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml similarity index 83% rename from ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml rename to ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml index 09657fede..b099dad61 100644 --- a/ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/global_coords-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/global_coords-1.0.0" title: Represents the ndcube GlobalCoords object @@ -22,7 +22,7 @@ properties: - tag: "tag:stsci.edu:asdf/unit/quantity-*" - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" ndcube: - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" + tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" required: [ndcube] additionalProperties: false diff --git a/ndcube/asdf/resources/schemas/ndcollection-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml similarity index 81% rename from ndcube/asdf/resources/schemas/ndcollection-0.1.0.yaml rename to ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml index 02c76e418..f6c736362 100644 --- a/ndcube/asdf/resources/schemas/ndcollection-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/ndcollection-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/ndcollection-1.0.0" title: Represents the ndcube.ndcollection.NDCollection object @@ -18,8 +18,8 @@ properties: items: - type: object oneOf: - - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" - - tag: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-0.*" + - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" + - tag: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-1.*" aligned_axes: anyOf: - type: object diff --git a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml similarity index 89% rename from ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml rename to ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml index 7be82ee3b..6ae68c9d6 100644 --- a/ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/NDCube-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/NDCube-1.0.0" title: Represents the ndcube NDCube object @@ -22,9 +22,9 @@ properties: - tag: "tag:sunpy.org:ndcube/reorderedwcs-*" - tag: "tag:sunpy.org:ndcube/compoundwcs-*" extra_coords: - tag: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-0.*" + tag: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-1.*" global_coords: - tag: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.*" + tag: "tag:sunpy.org:ndcube/global_coords/globalcoords-1.*" meta: type: object mask: diff --git a/ndcube/asdf/resources/schemas/ndcube_sequence-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml similarity index 77% rename from ndcube/asdf/resources/schemas/ndcube_sequence-0.1.0.yaml rename to ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml index 155665502..fbe4c1e68 100644 --- a/ndcube/asdf/resources/schemas/ndcube_sequence-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/ndcube_sequence-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/ndcube_sequence-1.0.0" title: Represents the ndcube.ndcube_sequence.NDCubeSequence object @@ -14,7 +14,7 @@ properties: data: type: array items: - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" + tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" meta: type: object common_axis: diff --git a/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/quantitytablecoordinate-1.0.0.yaml similarity index 89% rename from ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml rename to ndcube/asdf/resources/schemas/quantitytablecoordinate-1.0.0.yaml index 728d46353..bc8118d02 100644 --- a/ndcube/asdf/resources/schemas/quantitytablecoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/quantitytablecoordinate-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/quantitytablecoordinate-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/quantitytablecoordinate-1.0.0" title: Represents the QuantityTableCoords class diff --git a/ndcube/asdf/resources/schemas/reorderedwcs-0.1.0.yaml b/ndcube/asdf/resources/schemas/reorderedwcs-1.0.0.yaml similarity index 88% rename from ndcube/asdf/resources/schemas/reorderedwcs-0.1.0.yaml rename to ndcube/asdf/resources/schemas/reorderedwcs-1.0.0.yaml index b2acf85bd..9ce7f7752 100644 --- a/ndcube/asdf/resources/schemas/reorderedwcs-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/reorderedwcs-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/resampledwcs-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/resampledwcs-1.0.0" title: Represents the ndcube ReorderedLowLevelWCS object diff --git a/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml b/ndcube/asdf/resources/schemas/resampledwcs-1.0.0.yaml similarity index 89% rename from ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml rename to ndcube/asdf/resources/schemas/resampledwcs-1.0.0.yaml index 29de01fe8..16a94ee7e 100644 --- a/ndcube/asdf/resources/schemas/resampledwcs-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/resampledwcs-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/resampledwcs-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/resampledwcs-1.0.0" title: Represents the ndcube ResampledLowLevelWCS object diff --git a/ndcube/asdf/resources/schemas/skycoordtablecoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/skycoordtablecoordinate-1.0.0.yaml similarity index 86% rename from ndcube/asdf/resources/schemas/skycoordtablecoordinate-0.1.0.yaml rename to ndcube/asdf/resources/schemas/skycoordtablecoordinate-1.0.0.yaml index 97f22843a..4fe59b86a 100644 --- a/ndcube/asdf/resources/schemas/skycoordtablecoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/skycoordtablecoordinate-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/skycoordtablecoordinate-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/skycoordtablecoordinate-1.0.0" title: Represents the SkyCoordTableCoordinate class diff --git a/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml b/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml similarity index 86% rename from ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml rename to ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml index 3c5a828ce..a7b688da8 100644 --- a/ndcube/asdf/resources/schemas/timetablecoordinate-0.1.0.yaml +++ b/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/timetablecoordinate-0.1.0" +id: "asdf://sunpy.org/ndcube/schemas/timetablecoordinate-1.0.0" title: Represents the TimeTableCoords class From f1cec49d9bd48e7c97ad0e5287c4ff9ab71922c8 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 12:45:02 +0100 Subject: [PATCH 59/78] Add changelog --- changelog/776.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/776.feature.rst diff --git a/changelog/776.feature.rst b/changelog/776.feature.rst new file mode 100644 index 000000000..d39280f52 --- /dev/null +++ b/changelog/776.feature.rst @@ -0,0 +1 @@ +Add support for searlising most `ndcube` objects to ASDF files. From f430969fe9ea9fcb19d2334b100142b7ce908585 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 13:26:38 +0100 Subject: [PATCH 60/78] Bump various deps based on SPEC-0 --- changelog/776.breaking.rst | 9 +++++++++ .../tests/test_ndcollection_converter.py | 4 ---- .../asdf/converters/tests/test_ndcube_converter.py | 3 --- .../converters/tests/test_ndcube_wcs_wrappers.py | 6 +----- .../tests/test_ndcubesequence_converter.py | 5 ----- pyproject.toml | 14 +++++++------- tox.ini | 2 +- 7 files changed, 18 insertions(+), 25 deletions(-) create mode 100644 changelog/776.breaking.rst diff --git a/changelog/776.breaking.rst b/changelog/776.breaking.rst new file mode 100644 index 000000000..40274e006 --- /dev/null +++ b/changelog/776.breaking.rst @@ -0,0 +1,9 @@ +The minimum supported version of some dependencies has increased: + +* astropy >= 5.3 +* gwcs >= 0.20 +* numpy >= 1.25 +* scipy >= 1.11 +* matplotlib >= 3.8 +* mpl_animators >= 1.1 +* reproject >= 0.11 diff --git a/ndcube/asdf/converters/tests/test_ndcollection_converter.py b/ndcube/asdf/converters/tests/test_ndcollection_converter.py index 389ac085a..8d57cd2f0 100644 --- a/ndcube/asdf/converters/tests/test_ndcollection_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcollection_converter.py @@ -1,6 +1,4 @@ import pytest -from gwcs import __version__ as gwcs_version -from packaging.version import Version import asdf @@ -26,7 +24,6 @@ def create_ndcollection_cube( ) -@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") def test_serialization_cube(create_ndcollection_cube, tmp_path): ndcollection = create_ndcollection_cube file_path = tmp_path / "test.asdf" @@ -45,7 +42,6 @@ def create_ndcollection_sequence(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ return NDCollection([("seq0", sequence02), ("seq1", sequence20)], aligned_axes="all") -@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") def test_serialization_sequence(create_ndcollection_sequence, tmp_path): ndcollection = create_ndcollection_sequence file_path = tmp_path / "test.asdf" diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index a6ade02e3..c8dbeb2be 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -1,14 +1,11 @@ -import gwcs import numpy as np import pytest -from packaging.version import Version import asdf from ndcube.tests.helpers import assert_cubes_equal -@pytest.mark.skipif(Version(gwcs.__version__) < Version("0.20"), reason="Requires gwcs>=0.20") def test_serialization(all_ndcubes, tmp_path, all_ndcubes_names): if all_ndcubes_names in ("ndcube_4d_ln_lt_l_t", "ndcube_3d_ln_lt_l", "ndcube_3d_rotated"): pytest.importorskip("asdf_astropy", "0.8.0") diff --git a/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py b/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py index 791c29385..44bf3c0c8 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py +++ b/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py @@ -5,8 +5,6 @@ """ import pytest -from gwcs import __version__ as gwcs_version -from packaging.version import Version import asdf @@ -24,7 +22,6 @@ def create_ndcube_resampledwcs(gwcs_3d_lt_ln_l): return NDCube(data=data, wcs=new_wcs) -@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") def test_serialization_resampled(create_ndcube_resampledwcs, tmp_path): ndc = create_ndcube_resampledwcs file_path = tmp_path / "test.asdf" @@ -51,7 +48,6 @@ def create_ndcube_reorderedwcs(gwcs_3d_lt_ln_l): return NDCube(data = data, wcs =new_wcs) -@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") def test_serialization_reordered(create_ndcube_reorderedwcs, tmp_path): ndc = create_ndcube_reorderedwcs file_path = tmp_path / "test.asdf" @@ -77,7 +73,7 @@ def create_ndcube_compoundwcs(gwcs_2d_lt_ln, time_and_simple_extra_coords_2d): data = data_nd(shape) return NDCube(data = data, wcs = new_wcs) -@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") + def test_serialization_compoundwcs(create_ndcube_compoundwcs, tmp_path): ndc = create_ndcube_compoundwcs file_path = tmp_path / "test.asdf" diff --git a/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py b/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py index bf2faaa4a..b76e484e6 100644 --- a/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py @@ -1,14 +1,9 @@ -import pytest -from gwcs import __version__ as gwcs_version -from packaging.version import Version - import asdf from ndcube.ndcube_sequence import NDCubeSequence from ndcube.tests.helpers import assert_cubesequences_equal -@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") def test_serialization(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc, tmp_path): file_path = tmp_path / "test.asdf" ndcseq = NDCubeSequence([ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc], common_axis=1) diff --git a/pyproject.toml b/pyproject.toml index 5fc7beca4..3a33a9d38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,10 +16,10 @@ authors = [ { name = "The SunPy Community", email = "sunpy@googlegroups.com" }, ] dependencies = [ - "astropy>=5.0.6,!=5.1.0", - "gwcs>=0.18", - "numpy>=1.23.0", - "scipy>=1.8.0", + "astropy>=5.3.0", + "gwcs>=0.20", + "numpy>=1.25.0", + "scipy>=1.11.1", ] dynamic = ["version"] @@ -50,11 +50,11 @@ docs = [ "sunpy>=5.0.0", ] plotting = [ - "matplotlib>=3.5.0", - "mpl_animators>=1.0", + "matplotlib>=3.8.0", + "mpl_animators>=1.1", ] reproject = [ - "reproject>=0.7.1", + "reproject>=0.11.0", ] all = [ "ndcube[plotting,reproject]", diff --git a/tox.ini b/tox.ini index 5d5aae52c..03494b25d 100644 --- a/tox.ini +++ b/tox.ini @@ -67,7 +67,7 @@ extras = reproject tests commands_pre = - oldestdeps: minimum_dependencies ndcube --filename requirements-min.txt + oldestdeps: minimum_dependencies ndcube --extras plotting reproject --filename requirements-min.txt oldestdeps: pip install -r requirements-min.txt oldestdeps: python -c "import astropy.time; astropy.time.update_leap_seconds()" pip freeze --all --no-input From 38e2680b5a97a43cba52244a197bac659660f6e2 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 14:04:31 +0100 Subject: [PATCH 61/78] American I guess --- changelog/776.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/776.feature.rst b/changelog/776.feature.rst index d39280f52..5e7fb4512 100644 --- a/changelog/776.feature.rst +++ b/changelog/776.feature.rst @@ -1 +1 @@ -Add support for searlising most `ndcube` objects to ASDF files. +Add support for serialization most `ndcube` objects to ASDF files. From da2b74fece899d302359d1f7380cb00caf199c7f Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 14:05:17 +0100 Subject: [PATCH 62/78] Update 776.feature.rst --- changelog/776.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/776.feature.rst b/changelog/776.feature.rst index 5e7fb4512..9fe85494a 100644 --- a/changelog/776.feature.rst +++ b/changelog/776.feature.rst @@ -1 +1 @@ -Add support for serialization most `ndcube` objects to ASDF files. +Add support for serialization of most `ndcube` objects to ASDF files. From 3754a6193c8b10c0e2beef8acf4448fdfb186cee Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 14:05:47 +0100 Subject: [PATCH 63/78] Update docs/explaining_ndcube/asdf_serialization.rst Co-authored-by: DanRyanIrish --- docs/explaining_ndcube/asdf_serialization.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/explaining_ndcube/asdf_serialization.rst b/docs/explaining_ndcube/asdf_serialization.rst index 94e4641f7..667a85f07 100644 --- a/docs/explaining_ndcube/asdf_serialization.rst +++ b/docs/explaining_ndcube/asdf_serialization.rst @@ -41,7 +41,7 @@ You can save any number of cubes in your ASDF by adding them to the dictionary. ... f.write_to("somefile.asdf") # doctest: +SKIP -What's Supported and What isn't +What's Supported and What Isn't =============================== We aim to support all features of `ndcube` when saving and loading to ASDF. From 5fac69a178fdaf244d2d5b5aa4d1642664cfa7b2 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 14:58:18 +0100 Subject: [PATCH 64/78] More test fixes --- ndcube/asdf/converters/ndcube_converter.py | 3 +- .../converters/tests/test_ndcube_converter.py | 12 +++-- ndcube/conftest.py | 48 +++++++++++++++---- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 84246cb8e..9d4c4dfe2 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -31,8 +31,7 @@ def to_yaml_tree(self, ndcube, tag, ctx): ----- This methods serializes the primary components of the NDCube object, including the `data`, `wcs`, `extra_coords`, and `global_coords` attributes. - Issues a warning if unsupported attributes (uncertainty, mask, meta, unit) are present, - which are not currently serialized to ASDF. + Issues a warning if unsupported attributes are present. Warnings -------- diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index c8dbeb2be..2fc389489 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -1,14 +1,18 @@ +import asdf_astropy import numpy as np import pytest +from packaging.version import Version import asdf +import astropy.wcs from ndcube.tests.helpers import assert_cubes_equal def test_serialization(all_ndcubes, tmp_path, all_ndcubes_names): - if all_ndcubes_names in ("ndcube_4d_ln_lt_l_t", "ndcube_3d_ln_lt_l", "ndcube_3d_rotated"): - pytest.importorskip("asdf_astropy", "0.8.0") + # asdf_astropy doesn't save _naxis before this PR: https://github.com/astropy/asdf-astropy/pull/276 + if Version(asdf_astropy.__version__) < Version("0.8.0") and isinstance(all_ndcubes.wcs, astropy.wcs.WCS): + all_ndcubes.wcs._naxis = [0, 0] if not isinstance(all_ndcubes.data, np.ndarray): pytest.skip("Can't save non-numpy array to ASDF.") @@ -22,9 +26,7 @@ def test_serialization(all_ndcubes, tmp_path, all_ndcubes_names): assert_cubes_equal(af["ndcube"], all_ndcubes) -@pytest.mark.parametrize("expected_cube", - ["ndcube_gwcs_3d_ln_lt_l", "ndcube_3d_ln_lt_l"], - indirect=True) +@pytest.mark.parametrize("expected_cube", ["ndcube_gwcs_3d_ln_lt_l", "ndcube_3d_ln_lt_l"], indirect=True) def test_serialization_sliced_ndcube(expected_cube, tmp_path): # This needs 0.8.0 of asdf_astropy to be able to save gwcs and to save array_shape on WCS pytest.importorskip("asdf_astropy", "0.8.0") diff --git a/ndcube/conftest.py b/ndcube/conftest.py index aa3c71fa8..f6cc4c153 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -562,6 +562,7 @@ def extra_coords_sharing_axis(): ################################################################################ # NDCube Fixtures +# NOTE: If you add more fixtures please add to the all_ndcubes fixture ################################################################################ @pytest.fixture @@ -587,6 +588,7 @@ def ndcube_gwcs_3d_ln_lt_l(gwcs_3d_lt_ln_l): data_cube = data_nd(shape) return NDCube(data_cube, wcs=gwcs_3d_lt_ln_l) + @pytest.fixture def ndcube_gwcs_3d_rotated(gwcs_3d_lt_ln_l, simple_extra_coords_3d): data_rotated = np.array([[[1, 2, 3, 4, 6], [2, 4, 5, 3, 1], [0, -1, 2, 4, 2], [3, 5, 1, 2, 0]], @@ -597,6 +599,7 @@ def ndcube_gwcs_3d_rotated(gwcs_3d_lt_ln_l, simple_extra_coords_3d): cube._extra_coords = simple_extra_coords_3d return cube + @pytest.fixture def ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim(gwcs_3d_lt_ln_l, time_and_simple_extra_coords_2d): shape = (2, 3, 4) @@ -606,6 +609,7 @@ def ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim(gwcs_3d_lt_ln_l, time_and_simple_extra cube._extra_coords = time_and_simple_extra_coords_2d[0] return cube + @pytest.fixture def ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc(gwcs_3d_lt_ln_l): shape = (3, 3, 4) @@ -618,6 +622,7 @@ def ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc(gwcs_3d_lt_ln_l): cube.extra_coords.add("exposure_lut", 1, range(shape[1]) * u.s) return cube + @pytest.fixture def ndcube_gwcs_2d_ln_lt_mask(gwcs_2d_lt_ln): shape = (10, 12) @@ -629,6 +634,7 @@ def ndcube_gwcs_2d_ln_lt_mask(gwcs_2d_lt_ln): mask[4:6, :4] = True return NDCube(data_cube, wcs=gwcs_2d_lt_ln, mask=mask) + @pytest.fixture def ndcube_4d_ln_l_t_lt(wcs_4d_lt_t_l_ln): shape = (5, 10, 12, 8) @@ -1039,27 +1045,49 @@ def ndcube_1d_l(wcs_1d_l): @pytest.fixture(params=[ + "ndcube_gwcs_4d_ln_lt_l_t", + "ndcube_gwcs_4d_ln_lt_l_t_unit", + "ndcube_gwcs_3d_ln_lt_l", + "ndcube_gwcs_3d_rotated", + "ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim", + "ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc", + "ndcube_gwcs_2d_ln_lt_mask", + "ndcube_4d_ln_l_t_lt", "ndcube_4d_ln_lt_l_t", + "ndcube_4d_axis_aware_meta", "ndcube_4d_uncertainty", "ndcube_4d_mask", "ndcube_4d_extra_coords", "ndcube_4d_unit_uncertainty", "ndcube_3d_ln_lt_l", + "ndcube_3d_ln_lt_l_ec_all_axes", + "ndcube_3d_ln_lt_l_ec_sharing_axis", + "ndcube_3d_ln_lt_l_ec_time", + "ndcube_3d_wave_lt_ln_ec_time", "ndcube_3d_rotated", + "ndcube_3d_coupled", + "ndcube_3d_coupled_time", + "ndcube_3d_l_ln_lt_ectime", "ndcube_2d_ln_lt", + "ndcube_2d_ln_lt_uncert", + "ndcube_2d_ln_lt_mask_uncert", + "ndcube_2d_ln_lt_mask_uncert_unit_mask_false", + "ndcube_2d_ln_lt_mask_uncert_unit_one_maskele_true", + "ndcube_2d_ln_lt_mask_uncert_unit_one_maskele_true_expected_unmask_false", + "ndcube_2d_ln_lt_mask_uncert_unit_one_maskele_true_expected_unmask_true", + "ndcube_2d_ln_lt_mask_uncert_unit_mask_true", + "ndcube_2d_ln_lt_mask_uncert_unit_mask_true_expected_unmask_true", + "ndcube_2d_ln_lt_mask_uncert_unit_mask_true_expected_unmask_false", + "ndcube_2d_ln_lt_uncert_ec", "ndcube_2d_ln_lt_units", - "ndcube_2d_dask", - "ndcube_1d_l", "ndcube_2d_ln_lt_no_unit_no_unc", - "ndcube_2d_uncertainty_no_unit", "ndcube_2d_unit_unc", - "ndcube_gwcs_4d_ln_lt_l_t", - "ndcube_gwcs_4d_ln_lt_l_t_unit", - "ndcube_gwcs_3d_ln_lt_l", - "ndcube_gwcs_3d_rotated", - "ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim", - "ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc", - "ndcube_gwcs_2d_ln_lt_mask", + "ndcube_2d_uncertainty_no_unit", + "ndcube_2d_ln_lt_mask", + "ndcube_2d_ln_lt_mask2", + "ndcube_2d_ln_lt_nomask", + "ndcube_2d_dask", + "ndcube_1d_l", ]) def all_ndcubes_names(request): return request.param From 81677e91deaffb5010f8827cdee0b677a694dc0b Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 14:58:40 +0100 Subject: [PATCH 65/78] Allow bool for mask --- ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml index 6ae68c9d6..1b56c70a9 100644 --- a/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml @@ -28,7 +28,9 @@ properties: meta: type: object mask: - type: object + anyOf: + - tag: "tag:stsci.edu:asdf/core/ndarray-1.*" + - type: boolean unit: anyOf: - tag: "tag:stsci.edu:asdf/unit/unit-1.*" From b90a2ff43432cd1b0ee5090456ba7c836519a08c Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 15:19:11 +0100 Subject: [PATCH 66/78] Add support for NDMeta --- docs/explaining_ndcube/asdf_serialization.rst | 1 + ndcube/asdf/converters/ndmeta_converter.py | 24 +++++++++++++++++ ndcube/asdf/entry_points.py | 2 ++ .../resources/manifests/ndcube-1.0.0.yaml | 3 +++ .../asdf/resources/schemas/ndcube-1.0.0.yaml | 2 +- .../asdf/resources/schemas/ndmeta-1.0.0.yaml | 27 +++++++++++++++++++ ndcube/meta.py | 4 +-- ndcube/tests/helpers.py | 6 +++-- 8 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 ndcube/asdf/converters/ndmeta_converter.py create mode 100644 ndcube/asdf/resources/schemas/ndmeta-1.0.0.yaml diff --git a/docs/explaining_ndcube/asdf_serialization.rst b/docs/explaining_ndcube/asdf_serialization.rst index 667a85f07..4d48570d6 100644 --- a/docs/explaining_ndcube/asdf_serialization.rst +++ b/docs/explaining_ndcube/asdf_serialization.rst @@ -55,6 +55,7 @@ The only component of the `ndcube.NDCube` class which is never saved is the ``.p * `~ndcube.NDCube` * `~ndcube.NDCubeSequence` * `~ndcube.NDCollection` +* `~ndcube.NDMeta` * `~ndcube.GlobalCoords` * `~ndcube.ExtraCoords` * `~ndcube.extra_coords.TimeTableCoordinate` diff --git a/ndcube/asdf/converters/ndmeta_converter.py b/ndcube/asdf/converters/ndmeta_converter.py new file mode 100644 index 000000000..f8fa011b3 --- /dev/null +++ b/ndcube/asdf/converters/ndmeta_converter.py @@ -0,0 +1,24 @@ +import numpy as np + +from asdf.extension import Converter + + +class NDMetaConverter(Converter): + tags = ["tag:sunpy.org:ndcube/meta/ndmeta-*"] + types = ["ndcube.meta.NDMeta"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.meta import NDMeta + axes = {k: np.array(v) for k, v in node["axes"].items()} + meta = NDMeta(node["meta"], node["key_comments"], axes, node["data_shape"]) + meta._original_meta = node["original_meta"] + return meta + + def to_yaml_tree(self, meta, tag, ctx): + node = {} + node["meta"] = dict(meta) + node["key_comments"] = meta.key_comments + node["axes"] = meta.axes + node["data_shape"] = meta.data_shape + node["original_meta"] = meta._original_meta # not the MappingProxy object + return node diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index dd175c73b..ade044f41 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -37,6 +37,7 @@ def get_extensions(): from ndcube.asdf.converters.ndcollection_converter import NDCollectionConverter from ndcube.asdf.converters.ndcube_converter import NDCubeConverter from ndcube.asdf.converters.ndcubesequence_converter import NDCubeSequenceConverter + from ndcube.asdf.converters.ndmeta_converter import NDMetaConverter from ndcube.asdf.converters.reorderedwcs_converter import ReorderedConverter from ndcube.asdf.converters.resampled_converter import ResampledConverter from ndcube.asdf.converters.tablecoord_converter import ( @@ -58,6 +59,7 @@ def get_extensions(): CompoundConverter(), NDCubeSequenceConverter(), NDCollectionConverter(), + NDMetaConverter(), ] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0" diff --git a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml index 65a4bdfb4..5fc219630 100644 --- a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml @@ -41,3 +41,6 @@ tags: - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcollection-1.0.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcollection-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/meta/ndmeta-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndmeta-1.0.0" diff --git a/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml index 1b56c70a9..780fdbaa4 100644 --- a/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/NDCube-1.0.0" +id: "asdf://sunpy.org/ndcube/schemas/ndcube-1.0.0" title: Represents the ndcube NDCube object diff --git a/ndcube/asdf/resources/schemas/ndmeta-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndmeta-1.0.0.yaml new file mode 100644 index 000000000..175e14405 --- /dev/null +++ b/ndcube/asdf/resources/schemas/ndmeta-1.0.0.yaml @@ -0,0 +1,27 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/ndmeta-1.0.0" + +title: + Represents the ndcube NDMeta object + +description: + Represents the ndcube NDMeta object + +type: object +properties: + meta: + type: object + key_comments: + type: object + axes: + type: object + data_shape: + tag: "tag:stsci.edu:asdf/core/ndarray-1.*" + original_meta: + type: object + +required: [meta, key_comments, axes, data_shape, original_meta] +additionalProperties: true +... diff --git a/ndcube/meta.py b/ndcube/meta.py index 00255de67..5a10b113c 100644 --- a/ndcube/meta.py +++ b/ndcube/meta.py @@ -184,11 +184,11 @@ def _sanitize_axis_value(self, axis, value, key): if isinstance(axis, numbers.Integral): axis = (axis,) if len(axis) == 0: - return ValueError(axis_err_msg) + raise ValueError(axis_err_msg) # Verify each entry in axes is an iterable of ints or a scalar. if not (isinstance(axis, collections.abc.Iterable) and all(isinstance(i, numbers.Integral) for i in axis)): - return ValueError(axis_err_msg) + raise ValueError(axis_err_msg) # If metadata's axis/axes include axis beyond current data shape, extend it. data_shape = self.data_shape if max(axis) >= len(data_shape): diff --git a/ndcube/tests/helpers.py b/ndcube/tests/helpers.py index 0d7708b6e..d34a29b4e 100644 --- a/ndcube/tests/helpers.py +++ b/ndcube/tests/helpers.py @@ -107,10 +107,12 @@ def assert_metas_equal(test_input, expected_output): else: assert np.allclose(test_input.data_shape, expected_output.data_shape) - for test_value, expected_value in zip(test_input.values(), expected_output.values()): + for key in test_input.keys(): + test_value = test_input[key] + expected_value = expected_output[key] try: assert test_value == expected_value - except ValueError as err: # noqa: PERF203 + except ValueError as err: if multi_element_msg in err.args[0]: assert np.allclose(test_value, expected_value) for key in test_input.axes.keys(): From ad258e5bf84384e900f59b15d6284f2a2e5c09db Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 26 May 2025 15:56:24 +0100 Subject: [PATCH 67/78] Allow FITS WCS in ExtraCoords ASDF schema --- ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml b/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml index 86b76a308..a3b283649 100644 --- a/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml @@ -12,7 +12,9 @@ description: type: object properties: wcs: - tag: "tag:stsci.edu:gwcs/wcs-1.*" + anyOf: + - tag: "tag:stsci.edu:gwcs/wcs-*" + - tag: "tag:astropy.org:astropy/wcs/wcs-*" mapping: type: array lookup_tables: From e2c24f21f5c239cb9e10b8b18264d84797232e04 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Tue, 27 May 2025 11:22:03 +0100 Subject: [PATCH 68/78] Change NDCollection to save a dict not kv pairs --- .../asdf/converters/ndcollection_converter.py | 6 ++---- .../resources/schemas/ndcollection-1.0.0.yaml | 18 ++++++++---------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/ndcube/asdf/converters/ndcollection_converter.py b/ndcube/asdf/converters/ndcollection_converter.py index 85b42e523..6cd2f5ff8 100644 --- a/ndcube/asdf/converters/ndcollection_converter.py +++ b/ndcube/asdf/converters/ndcollection_converter.py @@ -8,15 +8,13 @@ class NDCollectionConverter(Converter): def from_yaml_tree(self, node, tag, ctx): from ndcube.ndcollection import NDCollection - key_value_pairs = list(zip(node["keys"], node["value"])) aligned_axes = list(node.get("aligned_axes").values()) aligned_axes = tuple(tuple(lst) for lst in aligned_axes) - return NDCollection(key_value_pairs, meta=node.get("meta"), aligned_axes=aligned_axes) + return NDCollection(node["items"], meta=node.get("meta"), aligned_axes=aligned_axes) def to_yaml_tree(self, ndcollection, tag, ctx): node = {} - node["keys"] = tuple(ndcollection.keys()) - node["value"] = tuple(ndcollection.values()) + node["items"] = dict(ndcollection) if ndcollection.meta is not None: node["meta"] = ndcollection.meta if ndcollection._aligned_axes is not None: diff --git a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml index f6c736362..9dfc874c6 100644 --- a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml @@ -11,20 +11,18 @@ description: type: object properties: - keys: - type: array - value: - type: array - items: - - type: object - oneOf: - - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" - - tag: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-1.*" + items: + type: object + patternProperties: + ".*": + oneOf: + - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-*" + - tag: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-*" aligned_axes: anyOf: - type: object - type: string -required: [keys, value] +required: [items] additionalProperties: true ... From 85f93f71c9f12a9ef38c446157b04093a8643dcd Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Tue, 27 May 2025 15:19:52 +0100 Subject: [PATCH 69/78] Apply suggestions from code review Co-authored-by: Brett Graham --- ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml b/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml index a7b688da8..075a92f6c 100644 --- a/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml @@ -12,13 +12,13 @@ description: type: object properties: table: - tag: "tag:stsci.edu:asdf/time/time-1*" + tag: "tag:stsci.edu:asdf/time/time-*" names: type: array physical_types: type: array reference_time: - tag: "tag:stsci.edu:asdf/time/time-1*" + tag: "tag:stsci.edu:asdf/time/time-*" required: ["table"] additionalProperties: False From b56c70280d8938d3c5c6389d996cb0bb4ed4188e Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 2 Jun 2025 10:35:04 +0100 Subject: [PATCH 70/78] Apply suggestions from code review --- .../asdf/resources/schemas/compoundwcs-1.0.0.yaml | 2 +- .../asdf/resources/schemas/extra_coords-1.0.0.yaml | 4 ++-- .../resources/schemas/global_coords-1.0.0.yaml | 2 +- .../schemas/multipletablecoordinate-1.0.0.yaml | 14 +++++++------- .../asdf/resources/schemas/ndcollection-1.0.0.yaml | 8 +++++--- ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml | 8 ++++---- .../schemas/quantitytablecoordinate-1.0.0.yaml | 4 ++-- .../asdf/resources/schemas/reorderedwcs-1.0.0.yaml | 2 +- .../asdf/resources/schemas/resampledwcs-1.0.0.yaml | 2 +- .../schemas/skycoordtablecoordinate-1.0.0.yaml | 2 +- .../schemas/timetablecoordinate-1.0.0.yaml | 2 +- pyproject.toml | 6 +++++- 12 files changed, 31 insertions(+), 25 deletions(-) diff --git a/ndcube/asdf/resources/schemas/compoundwcs-1.0.0.yaml b/ndcube/asdf/resources/schemas/compoundwcs-1.0.0.yaml index a0674a945..09eaa6dfe 100644 --- a/ndcube/asdf/resources/schemas/compoundwcs-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/compoundwcs-1.0.0.yaml @@ -23,5 +23,5 @@ properties: type: number required: [wcs] -additionalProperties: true +additionalProperties: false ... diff --git a/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml b/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml index a3b283649..ce7f156ad 100644 --- a/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml @@ -22,10 +22,10 @@ properties: items: type: array items: - - oneOf: + - anyOf: - type: number - type: array - - oneOf: + - anyOf: - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-1.*" - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-1.*" - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-1.*" diff --git a/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml b/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml index b099dad61..32a2e8d9e 100644 --- a/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml @@ -18,7 +18,7 @@ properties: items: - type: string - type: object - oneOf: + anyOf: - tag: "tag:stsci.edu:asdf/unit/quantity-*" - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" ndcube: diff --git a/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml b/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml index 1e58f4d10..d355303b3 100644 --- a/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml @@ -15,17 +15,17 @@ properties: type: array items: anyOf: - - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-*" - - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-*" - - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-1.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-1.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-1.*" dropped_coords: type: array items: anyOf: - - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-*" - - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-*" - - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-1.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-1.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-1.*" required: ["table_coords", "dropped_coords"] -additionalProperties: False +additionalProperties: false ... diff --git a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml index 9dfc874c6..08a50c2b4 100644 --- a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml @@ -15,9 +15,11 @@ properties: type: object patternProperties: ".*": - oneOf: - - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-*" - - tag: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-*" + anyOf: + - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" + - tag: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-1.*" + # Allow any other objects here as we don't want to restrict what people can save + - {} aligned_axes: anyOf: - type: object diff --git a/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml index 780fdbaa4..be97001a0 100644 --- a/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml @@ -29,14 +29,14 @@ properties: type: object mask: anyOf: - - tag: "tag:stsci.edu:asdf/core/ndarray-1.*" + - tag: "tag:stsci.edu:asdf/core/ndarray-*" - type: boolean unit: anyOf: - - tag: "tag:stsci.edu:asdf/unit/unit-1.*" - - tag: "tag:astropy.org:astropy/units/unit-1.*" + - tag: "tag:stsci.edu:asdf/unit/unit-*" + - tag: "tag:astropy.org:astropy/units/unit-*" uncertainty: - tag: tag:astropy.org:astropy/nddata/uncertainty-1.* + tag: tag:astropy.org:astropy/nddata/uncertainty-* required: [data, wcs] additionalProperties: true diff --git a/ndcube/asdf/resources/schemas/quantitytablecoordinate-1.0.0.yaml b/ndcube/asdf/resources/schemas/quantitytablecoordinate-1.0.0.yaml index bc8118d02..5cd0e7501 100644 --- a/ndcube/asdf/resources/schemas/quantitytablecoordinate-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/quantitytablecoordinate-1.0.0.yaml @@ -14,7 +14,7 @@ properties: unit: anyOf: - tag: "tag:stsci.edu:asdf/unit/unit-*" - - tag: "tag:astropy.org:astropy/units/unit-1.*" + - tag: "tag:astropy.org:astropy/units/unit-*" table: type: array items: @@ -27,5 +27,5 @@ properties: type: array required: ["table", "unit"] -additionalProperties: False +additionalProperties: false ... diff --git a/ndcube/asdf/resources/schemas/reorderedwcs-1.0.0.yaml b/ndcube/asdf/resources/schemas/reorderedwcs-1.0.0.yaml index 9ce7f7752..f01244466 100644 --- a/ndcube/asdf/resources/schemas/reorderedwcs-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/reorderedwcs-1.0.0.yaml @@ -21,5 +21,5 @@ properties: type: array required: [wcs] -additionalProperties: true +additionalProperties: false ... diff --git a/ndcube/asdf/resources/schemas/resampledwcs-1.0.0.yaml b/ndcube/asdf/resources/schemas/resampledwcs-1.0.0.yaml index 16a94ee7e..106707a06 100644 --- a/ndcube/asdf/resources/schemas/resampledwcs-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/resampledwcs-1.0.0.yaml @@ -21,5 +21,5 @@ properties: tag: "tag:stsci.edu:asdf/core/ndarray-1.*" required: [wcs] -additionalProperties: true +additionalProperties: false ... diff --git a/ndcube/asdf/resources/schemas/skycoordtablecoordinate-1.0.0.yaml b/ndcube/asdf/resources/schemas/skycoordtablecoordinate-1.0.0.yaml index 4fe59b86a..1607e637a 100644 --- a/ndcube/asdf/resources/schemas/skycoordtablecoordinate-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/skycoordtablecoordinate-1.0.0.yaml @@ -21,5 +21,5 @@ properties: type: array required: ["table"] -additionalProperties: False +additionalProperties: false ... diff --git a/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml b/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml index 075a92f6c..1e60d3ebf 100644 --- a/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml @@ -21,5 +21,5 @@ properties: tag: "tag:stsci.edu:asdf/time/time-*" required: ["table"] -additionalProperties: False +additionalProperties: false ... diff --git a/pyproject.toml b/pyproject.toml index 3a33a9d38..f5b71b2b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,8 +56,12 @@ plotting = [ reproject = [ "reproject>=0.11.0", ] +asdf = [ + "asdf>=2.15.0", + "asdf-astropy>=0.7.0", +] all = [ - "ndcube[plotting,reproject]", + "ndcube[plotting,reproject,asdf]", ] dev = [ "ndcube[tests,docs,plotting,reproject]", From 0762b8d5280558e839a05e9a7f9e9ae54bd5421e Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 2 Jun 2025 10:47:50 +0100 Subject: [PATCH 71/78] Only add meta, global and extra to ndcube if populated --- ndcube/asdf/converters/ndcube_converter.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 9d4c4dfe2..db0107ad7 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -49,9 +49,12 @@ def to_yaml_tree(self, ndcube, tag, ctx): node["wcs"] = ndcube.wcs.low_level_wcs else: node["wcs"] = ndcube.wcs - node["extra_coords"] = ndcube.extra_coords - node["global_coords"] = ndcube.global_coords - node["meta"] = ndcube.meta + if not ndcube.extra_coords.is_empty: + node["extra_coords"] = ndcube.extra_coords + if ndcube.global_coords._all_coords: + node["global_coords"] = ndcube.global_coords + if ndcube.meta: + node["meta"] = ndcube.meta if ndcube.mask is not None: node["mask"] = ndcube.mask if ndcube.unit is not None: From eb0c6f3615dbbea89da0d4218ae7e8616761f1f9 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Tue, 3 Jun 2025 17:01:11 +0100 Subject: [PATCH 72/78] More lax schemas --- ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml | 5 ++++- ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml index 08a50c2b4..02c9bba0f 100644 --- a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml @@ -11,6 +11,8 @@ description: type: object properties: + meta: + type: object items: type: object patternProperties: @@ -19,7 +21,7 @@ properties: - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" - tag: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-1.*" # Allow any other objects here as we don't want to restrict what people can save - - {} + - type: object aligned_axes: anyOf: - type: object @@ -27,4 +29,5 @@ properties: required: [items] additionalProperties: true +propertyOrder: ["meta", "aligned_axes", "items"] ... diff --git a/ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml index fbe4c1e68..9387c9a3c 100644 --- a/ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml @@ -14,7 +14,10 @@ properties: data: type: array items: - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" + anyOf: + - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" + # Allow any other objects here as we don't want to restrict what people can save + - type: object meta: type: object common_axis: From 25590577e4fe7927d71432acae66060c7c801d0e Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 30 Jun 2025 17:02:05 +0100 Subject: [PATCH 73/78] no property order --- ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml index 02c9bba0f..80b60444a 100644 --- a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml @@ -29,5 +29,4 @@ properties: required: [items] additionalProperties: true -propertyOrder: ["meta", "aligned_axes", "items"] ... From ff4b79f05c2ea57efb76d9c73a5726ece8a59a2f Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 20 Aug 2025 16:10:35 +0100 Subject: [PATCH 74/78] Fix tag URIs --- ndcube/asdf/converters/ndcollection_converter.py | 4 ++-- ndcube/asdf/converters/ndcube_converter.py | 6 +++--- ndcube/asdf/converters/ndcubesequence_converter.py | 4 ++-- ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml | 6 +++--- ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml | 2 +- ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml | 2 +- ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml | 4 ++-- ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ndcube/asdf/converters/ndcollection_converter.py b/ndcube/asdf/converters/ndcollection_converter.py index 6cd2f5ff8..3506ee8c2 100644 --- a/ndcube/asdf/converters/ndcollection_converter.py +++ b/ndcube/asdf/converters/ndcollection_converter.py @@ -2,11 +2,11 @@ class NDCollectionConverter(Converter): - tags = ["tag:sunpy.org:ndcube/ndcube/ndcollection-*"] + tags = ["tag:sunpy.org:ndcube/ndcollection-*"] types = ["ndcube.ndcollection.NDCollection"] def from_yaml_tree(self, node, tag, ctx): - from ndcube.ndcollection import NDCollection + from ndcube.ndcollection import NDCollection # noqa: PLC0415 aligned_axes = list(node.get("aligned_axes").values()) aligned_axes = tuple(tuple(lst) for lst in aligned_axes) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index db0107ad7..67dbed1a7 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -4,11 +4,11 @@ class NDCubeConverter(Converter): - tags = ["tag:sunpy.org:ndcube/ndcube/ndcube-*"] + tags = ["tag:sunpy.org:ndcube/ndcube-*"] types = ["ndcube.ndcube.NDCube"] def from_yaml_tree(self, node, tag, ctx): - from ndcube.ndcube import NDCube + from ndcube.ndcube import NDCube # noqa: PLC0415 ndcube = NDCube( node["data"], @@ -41,7 +41,7 @@ def to_yaml_tree(self, ndcube, tag, ctx): This ensures that users are aware of potentially important information that is not included in the serialized output. """ - from astropy.wcs.wcsapi import BaseHighLevelWCS + from astropy.wcs.wcsapi import BaseHighLevelWCS # noqa: PLC0415 node = {} node["data"] = ndcube.data diff --git a/ndcube/asdf/converters/ndcubesequence_converter.py b/ndcube/asdf/converters/ndcubesequence_converter.py index 419c69497..51cda0243 100644 --- a/ndcube/asdf/converters/ndcubesequence_converter.py +++ b/ndcube/asdf/converters/ndcubesequence_converter.py @@ -2,11 +2,11 @@ class NDCubeSequenceConverter(Converter): - tags = ["tag:sunpy.org:ndcube/ndcube/ndcube_sequence-*"] + tags = ["tag:sunpy.org:ndcube/ndcube_sequence-*"] types = ["ndcube.ndcube_sequence.NDCubeSequence"] def from_yaml_tree(self, node, tag, ctx): - from ndcube.ndcube_sequence import NDCubeSequence + from ndcube.ndcube_sequence import NDCubeSequence # noqa: PLC0415 return NDCubeSequence(node["data"], meta=node.get("meta"), diff --git a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml index 5fc219630..9a88d51d4 100644 --- a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml @@ -6,7 +6,7 @@ title: NDCube ASDF Manifest description: ASDF schemas and tags for NDCube classes. tags: - - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcube-1.0.0" + - tag_uri: "tag:sunpy.org:ndcube/ndcube-1.0.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcube-1.0.0" - tag_uri: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-1.0.0" @@ -30,7 +30,7 @@ tags: - tag_uri: "tag:sunpy.org:ndcube/resampledwcs-1.0.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/resampledwcs-1.0.0" - - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-1.0.0" + - tag_uri: "tag:sunpy.org:ndcube_sequence-1.0.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcube_sequence-1.0.0" - tag_uri: "tag:sunpy.org:ndcube/reorderedwcs-1.0.0" @@ -39,7 +39,7 @@ tags: - tag_uri: "tag:sunpy.org:ndcube/compoundwcs-1.0.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/compoundwcs-1.0.0" - - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcollection-1.0.0" + - tag_uri: "tag:sunpy.org:ndcube/ndcollection-1.0.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcollection-1.0.0" - tag_uri: "tag:sunpy.org:ndcube/meta/ndmeta-1.0.0" diff --git a/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml b/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml index ce7f156ad..7470473ee 100644 --- a/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml @@ -32,7 +32,7 @@ properties: dropped_tables: type: array ndcube: - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" + tag: "tag:sunpy.org:ndcube/ndcube-1.*" required: [ndcube] additionalProperties: false diff --git a/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml b/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml index 32a2e8d9e..2e4c00e14 100644 --- a/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml @@ -22,7 +22,7 @@ properties: - tag: "tag:stsci.edu:asdf/unit/quantity-*" - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" ndcube: - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" + tag: "tag:sunpy.org:ndcube/ndcube-1.*" required: [ndcube] additionalProperties: false diff --git a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml index 80b60444a..25099fd48 100644 --- a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml @@ -18,8 +18,8 @@ properties: patternProperties: ".*": anyOf: - - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" - - tag: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-1.*" + - tag: "tag:sunpy.org:ndcube/ndcube-1.*" + - tag: "tag:sunpy.org:ndcube/ndcube_sequence-1.*" # Allow any other objects here as we don't want to restrict what people can save - type: object aligned_axes: diff --git a/ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml index 9387c9a3c..5a67ae721 100644 --- a/ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml @@ -15,7 +15,7 @@ properties: type: array items: anyOf: - - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-1.*" + - tag: "tag:sunpy.org:ndcube/ndcube-1.*" # Allow any other objects here as we don't want to restrict what people can save - type: object meta: From 3ead8ad6ef6299b7697217a66e493324b3d79388 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 8 Sep 2025 14:45:57 +0100 Subject: [PATCH 75/78] Exclude all asdf files from import-at-top ruff --- .ruff.toml | 5 +++++ ndcube/asdf/converters/ndcollection_converter.py | 2 +- ndcube/asdf/converters/ndcube_converter.py | 4 ++-- ndcube/asdf/converters/ndcubesequence_converter.py | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index e38d68441..6e61f920c 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -82,6 +82,11 @@ extend-ignore = [ "test_*.py" = [ "E402", # Module level import not at top of cell ] +# ASDF converters delay imports for performance reasons +"ndcube/asdf/**.py" = [ + "E402", # Module level import not at top of cell + "PLC0415", # `import` should be at the top-level of a file +] [lint.pydocstyle] convention = "numpy" diff --git a/ndcube/asdf/converters/ndcollection_converter.py b/ndcube/asdf/converters/ndcollection_converter.py index 3506ee8c2..58adc2027 100644 --- a/ndcube/asdf/converters/ndcollection_converter.py +++ b/ndcube/asdf/converters/ndcollection_converter.py @@ -6,7 +6,7 @@ class NDCollectionConverter(Converter): types = ["ndcube.ndcollection.NDCollection"] def from_yaml_tree(self, node, tag, ctx): - from ndcube.ndcollection import NDCollection # noqa: PLC0415 + from ndcube.ndcollection import NDCollection aligned_axes = list(node.get("aligned_axes").values()) aligned_axes = tuple(tuple(lst) for lst in aligned_axes) diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py index 67dbed1a7..febcd918a 100644 --- a/ndcube/asdf/converters/ndcube_converter.py +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -8,7 +8,7 @@ class NDCubeConverter(Converter): types = ["ndcube.ndcube.NDCube"] def from_yaml_tree(self, node, tag, ctx): - from ndcube.ndcube import NDCube # noqa: PLC0415 + from ndcube.ndcube import NDCube ndcube = NDCube( node["data"], @@ -41,7 +41,7 @@ def to_yaml_tree(self, ndcube, tag, ctx): This ensures that users are aware of potentially important information that is not included in the serialized output. """ - from astropy.wcs.wcsapi import BaseHighLevelWCS # noqa: PLC0415 + from astropy.wcs.wcsapi import BaseHighLevelWCS node = {} node["data"] = ndcube.data diff --git a/ndcube/asdf/converters/ndcubesequence_converter.py b/ndcube/asdf/converters/ndcubesequence_converter.py index 51cda0243..94e0a4d18 100644 --- a/ndcube/asdf/converters/ndcubesequence_converter.py +++ b/ndcube/asdf/converters/ndcubesequence_converter.py @@ -6,7 +6,7 @@ class NDCubeSequenceConverter(Converter): types = ["ndcube.ndcube_sequence.NDCubeSequence"] def from_yaml_tree(self, node, tag, ctx): - from ndcube.ndcube_sequence import NDCubeSequence # noqa: PLC0415 + from ndcube.ndcube_sequence import NDCubeSequence return NDCubeSequence(node["data"], meta=node.get("meta"), From b467dabb0fa7b1402617a266ef2bbf808be93582 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 8 Sep 2025 14:54:09 +0100 Subject: [PATCH 76/78] Attempt to fix ndcube sequence schema --- ndcube/asdf/converters/ndcubesequence_converter.py | 2 +- ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml | 4 ++-- ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml | 2 +- .../{ndcube_sequence-1.0.0.yaml => ndcubesequence-1.0.0.yaml} | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename ndcube/asdf/resources/schemas/{ndcube_sequence-1.0.0.yaml => ndcubesequence-1.0.0.yaml} (90%) diff --git a/ndcube/asdf/converters/ndcubesequence_converter.py b/ndcube/asdf/converters/ndcubesequence_converter.py index 94e0a4d18..64896dbaf 100644 --- a/ndcube/asdf/converters/ndcubesequence_converter.py +++ b/ndcube/asdf/converters/ndcubesequence_converter.py @@ -2,7 +2,7 @@ class NDCubeSequenceConverter(Converter): - tags = ["tag:sunpy.org:ndcube/ndcube_sequence-*"] + tags = ["tag:sunpy.org:ndcube/ndcubesequence-*"] types = ["ndcube.ndcube_sequence.NDCubeSequence"] def from_yaml_tree(self, node, tag, ctx): diff --git a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml index 9a88d51d4..9867abc4e 100644 --- a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml @@ -30,8 +30,8 @@ tags: - tag_uri: "tag:sunpy.org:ndcube/resampledwcs-1.0.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/resampledwcs-1.0.0" - - tag_uri: "tag:sunpy.org:ndcube_sequence-1.0.0" - schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcube_sequence-1.0.0" + - tag_uri: "tag:sunpy.org:ndcube/ndcubesequence-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcubesequence-1.0.0" - tag_uri: "tag:sunpy.org:ndcube/reorderedwcs-1.0.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/reorderedwcs-1.0.0" diff --git a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml index 25099fd48..90c6afb6f 100644 --- a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml @@ -19,7 +19,7 @@ properties: ".*": anyOf: - tag: "tag:sunpy.org:ndcube/ndcube-1.*" - - tag: "tag:sunpy.org:ndcube/ndcube_sequence-1.*" + - tag: "tag:sunpy.org:ndcube/ndcubesequence-1.*" # Allow any other objects here as we don't want to restrict what people can save - type: object aligned_axes: diff --git a/ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcubesequence-1.0.0.yaml similarity index 90% rename from ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml rename to ndcube/asdf/resources/schemas/ndcubesequence-1.0.0.yaml index 5a67ae721..f7f75e49f 100644 --- a/ndcube/asdf/resources/schemas/ndcube_sequence-1.0.0.yaml +++ b/ndcube/asdf/resources/schemas/ndcubesequence-1.0.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "asdf://sunpy.org/ndcube/schemas/ndcube_sequence-1.0.0" +id: "asdf://sunpy.org/ndcube/schemas/ndcubesequence-1.0.0" title: Represents the ndcube.ndcube_sequence.NDCubeSequence object From 771c1a2a2cd4a419e45d9c466d2ea2da28df372f Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 8 Sep 2025 15:12:59 +0100 Subject: [PATCH 77/78] Finesse some deps --- pyproject.toml | 14 ++++++++------ tox.ini | 7 +++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f5b71b2b6..90dd1e11a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,8 +24,9 @@ dependencies = [ dynamic = ["version"] [project.optional-dependencies] -tests = [ +tests-only = [ "dask", + "pytest-asdf-plugin", "pytest-astropy", "pytest-cov", "pytest-doctestplus", @@ -33,17 +34,18 @@ tests = [ "pytest-xdist", "pytest", "pytest-memray; sys_platform != 'win32'", - "scipy", "specutils", "sunpy>=5.0.0", ] +tests = [ + "ndcube[plotting,reproject,asdf,tests-only]", +] docs = [ + "ndcube[plotting]", "sphinx", "sphinx-automodapi", "sunpy-sphinx-theme", "packaging", - "matplotlib", - "mpl-animators>=1.0", "sphinx-changelog>=1.1.0", "sphinx-gallery", "sphinxext-opengraph", @@ -61,10 +63,10 @@ asdf = [ "asdf-astropy>=0.7.0", ] all = [ - "ndcube[plotting,reproject,asdf]", + "ndcube[plotting,reproject]", ] dev = [ - "ndcube[tests,docs,plotting,reproject]", + "ndcube[tests-only,docs,plotting,reproject,asdf]", ] [project.urls] diff --git a/tox.ini b/tox.ini index b473d1e83..6ab3a47e6 100644 --- a/tox.ini +++ b/tox.ini @@ -63,11 +63,10 @@ deps = figure-!devdeps: scipy # The following indicates which extras_require will be installed extras = - plotting - reproject - tests + !oldestdeps: tests + oldestdeps: tests-only commands_pre = - oldestdeps: minimum_dependencies ndcube --extras plotting reproject --filename requirements-min.txt + oldestdeps: minimum_dependencies ndcube --extras plotting reproject asdf --filename requirements-min.txt oldestdeps: pip install -r requirements-min.txt oldestdeps: python -c "import astropy.time; astropy.time.update_leap_seconds()" pip freeze --all --no-input From a59c901fb4dae80ae3f1175dd143a25b956a50a0 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 8 Sep 2025 15:26:29 +0100 Subject: [PATCH 78/78] Fix tests with dev asdf-astropy --- .../asdf/converters/tests/test_ndcube_converter.py | 4 ++-- ndcube/tests/helpers.py | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py index 2fc389489..6e0fad3f3 100644 --- a/ndcube/asdf/converters/tests/test_ndcube_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -23,7 +23,7 @@ def test_serialization(all_ndcubes, tmp_path, all_ndcubes_names): af.write_to(file_path) with asdf.open(file_path) as af: - assert_cubes_equal(af["ndcube"], all_ndcubes) + assert_cubes_equal(af["ndcube"], all_ndcubes, rtol=1e-12) @pytest.mark.parametrize("expected_cube", ["ndcube_gwcs_3d_ln_lt_l", "ndcube_3d_ln_lt_l"], indirect=True) @@ -38,4 +38,4 @@ def test_serialization_sliced_ndcube(expected_cube, tmp_path): af.write_to(file_path) with asdf.open(file_path) as af: - assert_cubes_equal(af["ndcube_gwcs"], sndc) + assert_cubes_equal(af["ndcube_gwcs"], sndc, rtol=1e-12) diff --git a/ndcube/tests/helpers.py b/ndcube/tests/helpers.py index af5c0ea98..7d6c5409b 100644 --- a/ndcube/tests/helpers.py +++ b/ndcube/tests/helpers.py @@ -128,7 +128,7 @@ def assert_metas_equal(test_input, expected_output): assert test_input[key] == expected_output[key] -def assert_cubes_equal(test_input, expected_cube, check_data=True, check_uncertainty_values=False): +def assert_cubes_equal(test_input, expected_cube, check_data=True, check_uncertainty_values=False, rtol=None, atol=None): assert isinstance(test_input, type(expected_cube)) if isinstance(test_input.mask, bool): if not isinstance(expected_cube.mask, bool): @@ -138,7 +138,7 @@ def assert_cubes_equal(test_input, expected_cube, check_data=True, check_uncerta assert np.all(test_input.mask == expected_cube.mask) if check_data: np.testing.assert_array_equal(test_input.data, expected_cube.data) - assert_wcs_are_equal(test_input.wcs, expected_cube.wcs) + assert_wcs_are_equal(test_input.wcs, expected_cube.wcs, rtol=rtol, atol=atol) if check_uncertainty_values: # Check output and expected uncertainty are of same type. Remember they could be None. # If the uncertainties are not None,... @@ -171,7 +171,7 @@ def assert_cubesequences_equal(test_input, expected_sequence, check_data=True): assert_cubes_equal(cube, expected_sequence.data[i], check_data=check_data) -def assert_wcs_are_equal(wcs1, wcs2): +def assert_wcs_are_equal(wcs1, wcs2, *, rtol=None, atol=None): """ Assert function for testing two wcs object. @@ -179,6 +179,8 @@ def assert_wcs_are_equal(wcs1, wcs2): Also checks if both the wcs objects are instance of `~astropy.wcs.wcsapi.SlicedLowLevelWCS`. """ + atol = atol or 1e-23 + rtol = rtol or 1e-17 if not isinstance(wcs1, BaseLowLevelWCS): wcs1 = wcs1.low_level_wcs @@ -201,8 +203,8 @@ def assert_wcs_are_equal(wcs1, wcs2): np.testing.assert_allclose( low_level_wcs1.pixel_to_world_values(*random_idx.T), low_level_wcs2.pixel_to_world_values(*random_idx.T), - atol=1e-17, - rtol=1e-23, + atol=atol, + rtol=rtol, ) def create_sliced_wcs(wcs, item, dim):