diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9efeef115..1a0d1dc06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,7 @@ jobs: - windows: py311 - macos: py312 - linux: py310-oldestdeps + - linux: asdf_schemas secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.ruff.toml b/.ruff.toml index e38d68441..6e61f920c 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -82,6 +82,11 @@ extend-ignore = [ "test_*.py" = [ "E402", # Module level import not at top of cell ] +# ASDF converters delay imports for performance reasons +"ndcube/asdf/**.py" = [ + "E402", # Module level import not at top of cell + "PLC0415", # `import` should be at the top-level of a file +] [lint.pydocstyle] convention = "numpy" diff --git a/changelog/776.breaking.rst b/changelog/776.breaking.rst new file mode 100644 index 000000000..40274e006 --- /dev/null +++ b/changelog/776.breaking.rst @@ -0,0 +1,9 @@ +The minimum supported version of some dependencies has increased: + +* astropy >= 5.3 +* gwcs >= 0.20 +* numpy >= 1.25 +* scipy >= 1.11 +* matplotlib >= 3.8 +* mpl_animators >= 1.1 +* reproject >= 0.11 diff --git a/changelog/776.feature.rst b/changelog/776.feature.rst new file mode 100644 index 000000000..9fe85494a --- /dev/null +++ b/changelog/776.feature.rst @@ -0,0 +1 @@ +Add support for serialization of most `ndcube` objects to ASDF files. diff --git a/docs/conf.py b/docs/conf.py index 7c93801b2..f03512217 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -94,7 +94,8 @@ 'sunpy': ('https://docs.sunpy.org/en/stable/', None), 'mpl_animators': ('https://docs.sunpy.org/projects/mpl-animators/en/stable/', None), 'gwcs': ('https://gwcs.readthedocs.io/en/stable/', None), - 'reproject': ("https://reproject.readthedocs.io/en/stable/", None) + 'reproject': ("https://reproject.readthedocs.io/en/stable/", None), + 'asdf': ("https://www.asdf-format.org/projects/asdf/en/stable/", None), } # -- Options for HTML output ------------------------------------------------- diff --git a/docs/explaining_ndcube/asdf_serialization.rst b/docs/explaining_ndcube/asdf_serialization.rst new file mode 100644 index 000000000..4d48570d6 --- /dev/null +++ b/docs/explaining_ndcube/asdf_serialization.rst @@ -0,0 +1,67 @@ +.. _asdf_serialization: + +************************* +Saving ND objects to ASDF +************************* + +:ref:`asdf` is an extensible format for validating and saving complex scientific data along with its metadata. +`ndcube` provides schemas and converters for all the ND objects (`~ndcube.NDCube`, `~ndcube.NDCubeSequence` and `~ndcube.NDCollection`) as well as for various WCS and table objects required by them. +To make use of these, simply save an ND object to an ASDF file and it will be correctly serialized. +ASDF files save a "tree" which is a `dict`. +You can save any number of cubes in your ASDF by adding them to the dictionary. + +.. expanding-code-block:: python + :summary: Click to reveal/hide instantiation of the NDCube. + + >>> import numpy as np + >>> import asdf + >>> import astropy.wcs + >>> from ndcube import NDCube + + >>> # Define data array. + >>> data = np.random.rand(4, 4, 5) + + >>> # Define WCS transformations in an astropy WCS object. + >>> wcs = astropy.wcs.WCS(naxis=3) + >>> wcs.wcs.ctype = 'WAVE', 'HPLT-TAN', 'HPLN-TAN' + >>> wcs.wcs.cunit = 'Angstrom', 'deg', 'deg' + >>> wcs.wcs.cdelt = 0.2, 0.5, 0.4 + >>> wcs.wcs.crpix = 0, 2, 2 + >>> wcs.wcs.crval = 10, 0.5, 1 + >>> wcs.wcs.cname = 'wavelength', 'HPC lat', 'HPC lon' + + >>> # Now instantiate the NDCube + >>> my_cube = NDCube(data, wcs=wcs) + + +.. code-block:: python + + >>> my_tree = {"mycube": my_cube} + >>> with asdf.AsdfFile(tree=my_tree) as f: # doctest: +SKIP + ... f.write_to("somefile.asdf") # doctest: +SKIP + + +What's Supported and What Isn't +=============================== + +We aim to support all features of `ndcube` when saving and loading to ASDF. +However, because it is possible to create `ndcube` objects with many different components (for example dask arrays) which aren't part of the `ndcube` package these may not be supported. +Many common components of `ndcube` classes are supported in the `asdf_astropy `__ package, such as `astropy.wcs.WCS`, `astropy.wcs.wcsapi.SlicedLowLevelWCS` and uncertainty objects. + +The only component of the `ndcube.NDCube` class which is never saved is the ``.psf`` attribute. + +`ndcube` implements converters and schemas for the following objects: + +* `~ndcube.NDCube` +* `~ndcube.NDCubeSequence` +* `~ndcube.NDCollection` +* `~ndcube.NDMeta` +* `~ndcube.GlobalCoords` +* `~ndcube.ExtraCoords` +* `~ndcube.extra_coords.TimeTableCoordinate` +* `~ndcube.extra_coords.QuantityTableCoordinate` +* `~ndcube.extra_coords.SkyCoordTableCoordinate` +* `~ndcube.extra_coords.MultipleTableCoordinate` +* `~ndcube.wcs.wrappers.ReorderedLowLevelWCS` +* `~ndcube.wcs.wrappers.ResampledLowLevelWCS` +* `~ndcube.wcs.wrappers.CompoundLowLevelWCS` diff --git a/docs/explaining_ndcube/index.rst b/docs/explaining_ndcube/index.rst index a28801d00..e9ab9a330 100644 --- a/docs/explaining_ndcube/index.rst +++ b/docs/explaining_ndcube/index.rst @@ -14,3 +14,4 @@ Explaining ``ndcube`` tabular_coordinates reproject visualization + asdf_serialization diff --git a/ndcube/asdf/__init__.py b/ndcube/asdf/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ndcube/asdf/converters/__init__.py b/ndcube/asdf/converters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ndcube/asdf/converters/compoundwcs_converter.py b/ndcube/asdf/converters/compoundwcs_converter.py new file mode 100644 index 000000000..1f733fb4a --- /dev/null +++ b/ndcube/asdf/converters/compoundwcs_converter.py @@ -0,0 +1,18 @@ +from asdf.extension import Converter + + +class CompoundConverter(Converter): + tags = ["tag:sunpy.org:ndcube/compoundwcs-*"] + types = ["ndcube.wcs.wrappers.compound_wcs.CompoundLowLevelWCS"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.wcs.wrappers import CompoundLowLevelWCS + + return CompoundLowLevelWCS(*node["wcs"], mapping=node.get("mapping"), pixel_atol=node.get("atol")) + + def to_yaml_tree(self, compoundwcs, tag, ctx): + node = {} + node["wcs"] = compoundwcs._wcs + node["mapping"] = compoundwcs.mapping.mapping + node["atol"] = compoundwcs.atol + return node diff --git a/ndcube/asdf/converters/extracoords_converter.py b/ndcube/asdf/converters/extracoords_converter.py new file mode 100644 index 000000000..c94795943 --- /dev/null +++ b/ndcube/asdf/converters/extracoords_converter.py @@ -0,0 +1,28 @@ +from asdf.extension import Converter + + +class ExtraCoordsConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-*"] + types = ["ndcube.extra_coords.extra_coords.ExtraCoords"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.extra_coords import ExtraCoords + extra_coords = ExtraCoords() + extra_coords._wcs = node.get("wcs") + extra_coords._mapping = node.get("mapping") + extra_coords._lookup_tables = node.get("lookup_tables", []) + extra_coords._dropped_tables = node.get("dropped_tables") + extra_coords._ndcube = node.get("ndcube") + return extra_coords + + def to_yaml_tree(self, extracoords, tag, ctx): + node = {} + if extracoords._wcs is not None: + node["wcs"] = extracoords._wcs + if extracoords._mapping is not None: + node["mapping"] = extracoords._mapping + if extracoords._lookup_tables: + node["lookup_tables"] = extracoords._lookup_tables + node["dropped_tables"] = extracoords._dropped_tables + node["ndcube"] = extracoords._ndcube + return node diff --git a/ndcube/asdf/converters/globalcoords_converter.py b/ndcube/asdf/converters/globalcoords_converter.py new file mode 100644 index 000000000..9700a5890 --- /dev/null +++ b/ndcube/asdf/converters/globalcoords_converter.py @@ -0,0 +1,24 @@ +from asdf.extension import Converter + + +class GlobalCoordsConverter(Converter): + tags = ["tag:sunpy.org:ndcube/global_coords/globalcoords-*"] + types = ["ndcube.global_coords.GlobalCoords"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.global_coords import GlobalCoords + + globalcoords = GlobalCoords() + if "internal_coords" in node: + globalcoords._internal_coords = node["internal_coords"] + globalcoords._ndcube = node["ndcube"] + + return globalcoords + + def to_yaml_tree(self, globalcoords, tag, ctx): + node = {} + node["ndcube"] = globalcoords._ndcube + if globalcoords._internal_coords: + node["internal_coords"] = globalcoords._internal_coords + + return node diff --git a/ndcube/asdf/converters/ndcollection_converter.py b/ndcube/asdf/converters/ndcollection_converter.py new file mode 100644 index 000000000..58adc2027 --- /dev/null +++ b/ndcube/asdf/converters/ndcollection_converter.py @@ -0,0 +1,23 @@ +from asdf.extension import Converter + + +class NDCollectionConverter(Converter): + tags = ["tag:sunpy.org:ndcube/ndcollection-*"] + types = ["ndcube.ndcollection.NDCollection"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.ndcollection import NDCollection + + aligned_axes = list(node.get("aligned_axes").values()) + aligned_axes = tuple(tuple(lst) for lst in aligned_axes) + return NDCollection(node["items"], meta=node.get("meta"), aligned_axes=aligned_axes) + + def to_yaml_tree(self, ndcollection, tag, ctx): + node = {} + node["items"] = dict(ndcollection) + if ndcollection.meta is not None: + node["meta"] = ndcollection.meta + if ndcollection._aligned_axes is not None: + node["aligned_axes"] = ndcollection._aligned_axes + + return node diff --git a/ndcube/asdf/converters/ndcube_converter.py b/ndcube/asdf/converters/ndcube_converter.py new file mode 100644 index 000000000..febcd918a --- /dev/null +++ b/ndcube/asdf/converters/ndcube_converter.py @@ -0,0 +1,68 @@ +import warnings + +from asdf.extension import Converter + + +class NDCubeConverter(Converter): + tags = ["tag:sunpy.org:ndcube/ndcube-*"] + types = ["ndcube.ndcube.NDCube"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.ndcube import NDCube + + ndcube = NDCube( + node["data"], + node["wcs"], + meta=node.get("meta"), + mask=node.get("mask"), + unit=node.get("unit"), + uncertainty=node.get("uncertainty"), + ) + if "extra_coords" in node: + ndcube._extra_coords = node["extra_coords"] + if "global_coords" in node: + ndcube._global_coords = node["global_coords"] + + return ndcube + + def to_yaml_tree(self, ndcube, tag, ctx): + """ + Notes + ----- + This methods serializes the primary components of the NDCube object, + including the `data`, `wcs`, `extra_coords`, and `global_coords` attributes. + Issues a warning if unsupported attributes are present. + + Warnings + -------- + UserWarning + Warns if the NDCube object has a 'psf' attribute that will not be + saved in the ASDF serialization. + This ensures that users are aware of potentially important information + that is not included in the serialized output. + """ + from astropy.wcs.wcsapi import BaseHighLevelWCS + + node = {} + node["data"] = ndcube.data + if isinstance(ndcube.wcs, BaseHighLevelWCS): + node["wcs"] = ndcube.wcs.low_level_wcs + else: + node["wcs"] = ndcube.wcs + if not ndcube.extra_coords.is_empty: + node["extra_coords"] = ndcube.extra_coords + if ndcube.global_coords._all_coords: + node["global_coords"] = ndcube.global_coords + if ndcube.meta: + node["meta"] = ndcube.meta + if ndcube.mask is not None: + node["mask"] = ndcube.mask + if ndcube.unit is not None: + node["unit"] = ndcube.unit + if ndcube.uncertainty is not None: + node["uncertainty"] = ndcube.uncertainty + + if getattr(ndcube, 'psf') is not None: + warnings.warn("Attribute 'psf' is present but not being saved in ASDF serialization.", UserWarning) + + return node diff --git a/ndcube/asdf/converters/ndcubesequence_converter.py b/ndcube/asdf/converters/ndcubesequence_converter.py new file mode 100644 index 000000000..64896dbaf --- /dev/null +++ b/ndcube/asdf/converters/ndcubesequence_converter.py @@ -0,0 +1,23 @@ +from asdf.extension import Converter + + +class NDCubeSequenceConverter(Converter): + tags = ["tag:sunpy.org:ndcube/ndcubesequence-*"] + types = ["ndcube.ndcube_sequence.NDCubeSequence"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.ndcube_sequence import NDCubeSequence + + return NDCubeSequence(node["data"], + meta=node.get("meta"), + common_axis=node.get("common_axis")) + + def to_yaml_tree(self, ndcseq, tag, ctx): + node = {} + node["data"] = ndcseq.data + if ndcseq.meta is not None: + node["meta"] = ndcseq.meta + if ndcseq._common_axis is not None: + node["common_axis"] = ndcseq._common_axis + + return node diff --git a/ndcube/asdf/converters/ndmeta_converter.py b/ndcube/asdf/converters/ndmeta_converter.py new file mode 100644 index 000000000..f8fa011b3 --- /dev/null +++ b/ndcube/asdf/converters/ndmeta_converter.py @@ -0,0 +1,24 @@ +import numpy as np + +from asdf.extension import Converter + + +class NDMetaConverter(Converter): + tags = ["tag:sunpy.org:ndcube/meta/ndmeta-*"] + types = ["ndcube.meta.NDMeta"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.meta import NDMeta + axes = {k: np.array(v) for k, v in node["axes"].items()} + meta = NDMeta(node["meta"], node["key_comments"], axes, node["data_shape"]) + meta._original_meta = node["original_meta"] + return meta + + def to_yaml_tree(self, meta, tag, ctx): + node = {} + node["meta"] = dict(meta) + node["key_comments"] = meta.key_comments + node["axes"] = meta.axes + node["data_shape"] = meta.data_shape + node["original_meta"] = meta._original_meta # not the MappingProxy object + return node diff --git a/ndcube/asdf/converters/reorderedwcs_converter.py b/ndcube/asdf/converters/reorderedwcs_converter.py new file mode 100644 index 000000000..19d005370 --- /dev/null +++ b/ndcube/asdf/converters/reorderedwcs_converter.py @@ -0,0 +1,22 @@ +from asdf.extension import Converter + + +class ReorderedConverter(Converter): + tags = ["tag:sunpy.org:ndcube/reorderedwcs-*"] + types = ["ndcube.wcs.wrappers.reordered_wcs.ReorderedLowLevelWCS"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.wcs.wrappers import ReorderedLowLevelWCS + + return ReorderedLowLevelWCS( + wcs=node["wcs"], + pixel_order=node.get("pixel_order"), + world_order=node.get("world_order"), + ) + + def to_yaml_tree(self, reorderedwcs, tag, ctx): + node = {} + node["wcs"] = reorderedwcs._wcs + node["pixel_order"] = reorderedwcs._pixel_order + node["world_order"] = reorderedwcs._world_order + return node diff --git a/ndcube/asdf/converters/resampled_converter.py b/ndcube/asdf/converters/resampled_converter.py new file mode 100644 index 000000000..7a838c6f4 --- /dev/null +++ b/ndcube/asdf/converters/resampled_converter.py @@ -0,0 +1,23 @@ +from asdf.extension import Converter + + +class ResampledConverter(Converter): + tags = ["tag:sunpy.org:ndcube/resampledwcs-*"] + types = ["ndcube.wcs.wrappers.resampled_wcs.ResampledLowLevelWCS"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.wcs.wrappers import ResampledLowLevelWCS + + return ResampledLowLevelWCS( + wcs=node["wcs"], + offset=node.get("offset"), + factor=node.get("factor"), + ) + + def to_yaml_tree(self, resampledwcs, tag, ctx): + node = {} + node["wcs"] = resampledwcs._wcs + node["factor"] = resampledwcs._factor + node["offset"] = resampledwcs._offset + + return node diff --git a/ndcube/asdf/converters/tablecoord_converter.py b/ndcube/asdf/converters/tablecoord_converter.py new file mode 100644 index 000000000..755c6e2f3 --- /dev/null +++ b/ndcube/asdf/converters/tablecoord_converter.py @@ -0,0 +1,100 @@ +from asdf.extension import Converter + + +class TimeTableCoordConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-*"] + types = ["ndcube.extra_coords.table_coord.TimeTableCoordinate"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.table_coord import TimeTableCoordinate + + names = node.get("names") + physical_types = node.get("physical_types") + reference_time = node.get("reference_time") + return TimeTableCoordinate( + node["table"], + names=names, + physical_types=physical_types, + reference_time=reference_time, + ) + + def to_yaml_tree(self, timetablecoordinate, tag, ctx): + node = {} + node["table"] = timetablecoordinate.table + if timetablecoordinate.names: + node["names"] = timetablecoordinate.names + if timetablecoordinate.physical_types is not None: + node["physical_types"] = timetablecoordinate.physical_types + node["reference_time"] = timetablecoordinate.reference_time + + return node + + +class QuantityTableCoordinateConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-*"] + types = ["ndcube.extra_coords.table_coord.QuantityTableCoordinate"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.table_coord import QuantityTableCoordinate + + names = node.get("names") + mesh = node.get("mesh") + physical_types = node.get("physical_types") + quantitytablecoordinate = QuantityTableCoordinate(*node["table"], names=names, physical_types=physical_types) + quantitytablecoordinate.unit = node["unit"] + quantitytablecoordinate.mesh = mesh + return quantitytablecoordinate + + def to_yaml_tree(self, quantitytablecoordinate, tag, ctx): + node = {} + node["unit"] = quantitytablecoordinate.unit + node["table"] = quantitytablecoordinate.table + if quantitytablecoordinate.names: + node["names"] = quantitytablecoordinate.names + node["mesh"] = quantitytablecoordinate.mesh + if quantitytablecoordinate.physical_types is not None: + node["physical_types"] = quantitytablecoordinate.physical_types + + return node + + +class SkyCoordTableCoordinateConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-*"] + types = ["ndcube.extra_coords.table_coord.SkyCoordTableCoordinate"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.table_coord import SkyCoordTableCoordinate + + names = node.get("names") + mesh = node.get("mesh") + physical_types = node.get("physical_types") + return SkyCoordTableCoordinate(node["table"], mesh=mesh, names=names, physical_types=physical_types) + + def to_yaml_tree(self, skycoordinatetablecoordinate, tag, ctx): + node = {} + node["table"] = skycoordinatetablecoordinate.table + if skycoordinatetablecoordinate.names: + node["names"] = skycoordinatetablecoordinate.names + node["mesh"] = skycoordinatetablecoordinate.mesh + if skycoordinatetablecoordinate.physical_types is not None: + node["physical_types"] = skycoordinatetablecoordinate.physical_types + + return node + + +class MultipleTableCoordinateConverter(Converter): + tags = ["tag:sunpy.org:ndcube/extra_coords/table_coord/multipletablecoordinate-*"] + types = ["ndcube.extra_coords.table_coord.MultipleTableCoordinate"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.extra_coords.table_coord import MultipleTableCoordinate + + mtc = MultipleTableCoordinate(*node["table_coords"]) + mtc._dropped_coords = node["dropped_coords"] + return mtc + + def to_yaml_tree(self, multipletablecoordinate, tag, ctx): + node = {} + node["table_coords"] = multipletablecoordinate._table_coords + node["dropped_coords"] = multipletablecoordinate._dropped_coords + return node diff --git a/ndcube/asdf/converters/tests/__init__.py b/ndcube/asdf/converters/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ndcube/asdf/converters/tests/test_ndcollection_converter.py b/ndcube/asdf/converters/tests/test_ndcollection_converter.py new file mode 100644 index 000000000..8d57cd2f0 --- /dev/null +++ b/ndcube/asdf/converters/tests/test_ndcollection_converter.py @@ -0,0 +1,53 @@ +import pytest + +import asdf + +from ndcube.ndcollection import NDCollection +from ndcube.ndcube_sequence import NDCubeSequence +from ndcube.tests.helpers import assert_collections_equal + + +@pytest.fixture +def create_ndcollection_cube( + ndcube_gwcs_3d_ln_lt_l, + ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc, + ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim, +): + aligned_axes = ((1, 2), (1, 2), (1, 2)) + return NDCollection( + [ + ("cube0", ndcube_gwcs_3d_ln_lt_l), + ("cube1", ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc), + ("cube2", ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim), + ], + aligned_axes=aligned_axes, + ) + + +def test_serialization_cube(create_ndcollection_cube, tmp_path): + ndcollection = create_ndcollection_cube + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = ndcollection + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_collections_equal(af["ndcube_gwcs"], ndcollection) + + +@pytest.fixture +def create_ndcollection_sequence(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim): + sequence02 = NDCubeSequence([ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim]) + sequence20 = NDCubeSequence([ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim, ndcube_gwcs_3d_ln_lt_l]) + return NDCollection([("seq0", sequence02), ("seq1", sequence20)], aligned_axes="all") + + +def test_serialization_sequence(create_ndcollection_sequence, tmp_path): + ndcollection = create_ndcollection_sequence + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = ndcollection + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_collections_equal(af["ndcube_gwcs"], ndcollection) diff --git a/ndcube/asdf/converters/tests/test_ndcube_converter.py b/ndcube/asdf/converters/tests/test_ndcube_converter.py new file mode 100644 index 000000000..6e0fad3f3 --- /dev/null +++ b/ndcube/asdf/converters/tests/test_ndcube_converter.py @@ -0,0 +1,41 @@ +import asdf_astropy +import numpy as np +import pytest +from packaging.version import Version + +import asdf +import astropy.wcs + +from ndcube.tests.helpers import assert_cubes_equal + + +def test_serialization(all_ndcubes, tmp_path, all_ndcubes_names): + # asdf_astropy doesn't save _naxis before this PR: https://github.com/astropy/asdf-astropy/pull/276 + if Version(asdf_astropy.__version__) < Version("0.8.0") and isinstance(all_ndcubes.wcs, astropy.wcs.WCS): + all_ndcubes.wcs._naxis = [0, 0] + + if not isinstance(all_ndcubes.data, np.ndarray): + pytest.skip("Can't save non-numpy array to ASDF.") + + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube"] = all_ndcubes + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_cubes_equal(af["ndcube"], all_ndcubes, rtol=1e-12) + + +@pytest.mark.parametrize("expected_cube", ["ndcube_gwcs_3d_ln_lt_l", "ndcube_3d_ln_lt_l"], indirect=True) +def test_serialization_sliced_ndcube(expected_cube, tmp_path): + # This needs 0.8.0 of asdf_astropy to be able to save gwcs and to save array_shape on WCS + pytest.importorskip("asdf_astropy", "0.8.0") + + sndc = expected_cube[np.s_[0, :, :]] + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = sndc + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_cubes_equal(af["ndcube_gwcs"], sndc, rtol=1e-12) diff --git a/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py b/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py new file mode 100644 index 000000000..44bf3c0c8 --- /dev/null +++ b/ndcube/asdf/converters/tests/test_ndcube_wcs_wrappers.py @@ -0,0 +1,88 @@ +""" +Tests for roundtrip serialization of NDCube with various GWCS types. + +TODO: Add tests for the roundtrip serialization of NDCube with ResampledLowLevelWCS, ReorderedLowLevelWCS, and CompoundLowLevelWCS when using astropy.wcs.WCS. +""" + +import pytest + +import asdf + +from ndcube import NDCube +from ndcube.conftest import data_nd +from ndcube.tests.helpers import assert_cubes_equal +from ndcube.wcs.wrappers import CompoundLowLevelWCS, ReorderedLowLevelWCS, ResampledLowLevelWCS + + +@pytest.fixture +def create_ndcube_resampledwcs(gwcs_3d_lt_ln_l): + shape = (2, 3, 4) + new_wcs = ResampledLowLevelWCS(wcs=gwcs_3d_lt_ln_l, factor=2 , offset=1) + data = data_nd(shape) + return NDCube(data=data, wcs=new_wcs) + + +def test_serialization_resampled(create_ndcube_resampledwcs, tmp_path): + ndc = create_ndcube_resampledwcs + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube"] = ndc + af.write_to(file_path) + + with asdf.open(file_path) as af: + loaded_ndcube = af["ndcube"] + + loaded_resampledwcs = loaded_ndcube.wcs.low_level_wcs + resampledwcs = ndc.wcs.low_level_wcs + assert (loaded_resampledwcs._factor == resampledwcs._factor).all() + assert (loaded_resampledwcs._offset == resampledwcs._offset).all() + + assert_cubes_equal(loaded_ndcube, ndc) + + +@pytest.fixture +def create_ndcube_reorderedwcs(gwcs_3d_lt_ln_l): + shape = (2, 3, 4) + new_wcs = ReorderedLowLevelWCS(wcs = gwcs_3d_lt_ln_l, pixel_order=[1, 2, 0] ,world_order=[2, 0, 1]) + data = data_nd(shape) + return NDCube(data = data, wcs =new_wcs) + + +def test_serialization_reordered(create_ndcube_reorderedwcs, tmp_path): + ndc = create_ndcube_reorderedwcs + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube"] = ndc + af.write_to(file_path) + + with asdf.open(file_path) as af: + loaded_ndcube = af["ndcube"] + + loaded_reorderedwcs = loaded_ndcube.wcs.low_level_wcs + reorderedwcs = ndc.wcs.low_level_wcs + assert (loaded_reorderedwcs._pixel_order == reorderedwcs._pixel_order) + assert (loaded_reorderedwcs._world_order == reorderedwcs._world_order) + + assert_cubes_equal(loaded_ndcube, ndc) + +@pytest.fixture +def create_ndcube_compoundwcs(gwcs_2d_lt_ln, time_and_simple_extra_coords_2d): + + shape = (1, 2, 3, 4) + new_wcs = CompoundLowLevelWCS(gwcs_2d_lt_ln, time_and_simple_extra_coords_2d.wcs, mapping = [0, 1, 2, 3]) + data = data_nd(shape) + return NDCube(data = data, wcs = new_wcs) + + +def test_serialization_compoundwcs(create_ndcube_compoundwcs, tmp_path): + ndc = create_ndcube_compoundwcs + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube"] = ndc + af.write_to(file_path) + + with asdf.open(file_path) as af: + loaded_ndcube = af["ndcube"] + assert_cubes_equal(loaded_ndcube, ndc) + assert (loaded_ndcube.wcs.low_level_wcs.mapping.mapping == ndc.wcs.low_level_wcs.mapping.mapping) + assert (loaded_ndcube.wcs.low_level_wcs.atol == ndc.wcs.low_level_wcs.atol) diff --git a/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py b/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py new file mode 100644 index 000000000..b76e484e6 --- /dev/null +++ b/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py @@ -0,0 +1,15 @@ +import asdf + +from ndcube.ndcube_sequence import NDCubeSequence +from ndcube.tests.helpers import assert_cubesequences_equal + + +def test_serialization(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc, tmp_path): + file_path = tmp_path / "test.asdf" + ndcseq = NDCubeSequence([ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc], common_axis=1) + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = ndcseq + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_cubesequences_equal(af["ndcube_gwcs"], ndcseq) diff --git a/ndcube/asdf/converters/tests/test_table_coordinate.py b/ndcube/asdf/converters/tests/test_table_coordinate.py new file mode 100644 index 000000000..e48a14638 --- /dev/null +++ b/ndcube/asdf/converters/tests/test_table_coordinate.py @@ -0,0 +1,84 @@ +import pytest + +import asdf +import astropy.units as u +from astropy.coordinates import SkyCoord + +from ndcube.extra_coords import MultipleTableCoordinate + + +@pytest.fixture +def lut(request): + return request.getfixturevalue(request.param) + + +def assert_table_coord_equal(test_table, expected_table): + test_table = test_table.table + expected_table = expected_table.table + if not isinstance(expected_table, tuple): + test_table = (test_table,) + expected_table = (expected_table,) + for test_tab, ex_tab in zip(test_table, expected_table): + if ex_tab.isscalar: + assert test_tab == ex_tab + elif isinstance(ex_tab, SkyCoord): + assert u.allclose(ex_tab.spherical.lat, test_tab.spherical.lat) + assert u.allclose(ex_tab.spherical.lon, test_tab.spherical.lon) + else: + assert all(test_tab == ex_tab) + + +@pytest.mark.parametrize("lut", + [ + "lut_1d_distance", + "lut_3d_distance_mesh", + "lut_1d_skycoord_no_mesh", + "lut_2d_skycoord_no_mesh", + "lut_2d_skycoord_mesh", + "lut_3d_skycoord_mesh", + "lut_1d_time", + "lut_1d_wave", + ], indirect=True) +def test_serialize(lut, tmp_path): + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["lut"] = lut + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_table_coord_equal(af["lut"], lut) + + +def assert_mtc_equal(test_mtc, expected_mtc): + assert len(test_mtc._table_coords) == len(expected_mtc._table_coords) + assert len(test_mtc._dropped_coords) == len(expected_mtc._dropped_coords) + + for (test_tc, expected_tc) in zip(test_mtc._table_coords, expected_mtc._table_coords): + assert_table_coord_equal(test_tc, expected_tc) + + for (test_tc, expected_tc) in zip(test_mtc._dropped_coords, expected_mtc._dropped_coords): + assert_table_coord_equal(test_tc, expected_tc) + + +def test_serialize_multiple_coord(lut_1d_distance, lut_1d_time, tmp_path): + mtc = MultipleTableCoordinate(lut_1d_distance, lut_1d_time) + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["lut"] = mtc + af.write_to(file_path) + + with asdf.open(file_path) as af: + new_mtc = af["lut"] + assert_mtc_equal(new_mtc, mtc) + + +def test_serialize_sliced_multiple_coord(lut_1d_distance, lut_1d_time, tmp_path): + mtc = MultipleTableCoordinate(lut_1d_distance, lut_1d_time)[0, :] + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["lut"] = mtc + af.write_to(file_path) + + with asdf.open(file_path) as af: + new_mtc = af["lut"] + assert_mtc_equal(new_mtc, mtc) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py new file mode 100644 index 000000000..ade044f41 --- /dev/null +++ b/ndcube/asdf/entry_points.py @@ -0,0 +1,68 @@ +""" +This file contains the entry points for asdf. +""" +import importlib.resources as importlib_resources + +from asdf.extension import ManifestExtension +from asdf.resource import DirectoryResourceMapping + + +def get_resource_mappings(): + """ + Get the resource mapping instances for myschemas + and manifests. This method is registered with the + asdf.resource_mappings entry point. + + Returns + ------- + list of collections.abc.Mapping + """ + from ndcube.asdf import resources + resources_root = importlib_resources.files(resources) + return [ + DirectoryResourceMapping( + resources_root / "schemas", "asdf://sunpy.org/ndcube/schemas/"), + DirectoryResourceMapping( + resources_root / "manifests", "asdf://sunpy.org/ndcube/manifests/"), + ] + + +def get_extensions(): + """ + Get the list of extensions. + """ + from ndcube.asdf.converters.compoundwcs_converter import CompoundConverter + from ndcube.asdf.converters.extracoords_converter import ExtraCoordsConverter + from ndcube.asdf.converters.globalcoords_converter import GlobalCoordsConverter + from ndcube.asdf.converters.ndcollection_converter import NDCollectionConverter + from ndcube.asdf.converters.ndcube_converter import NDCubeConverter + from ndcube.asdf.converters.ndcubesequence_converter import NDCubeSequenceConverter + from ndcube.asdf.converters.ndmeta_converter import NDMetaConverter + from ndcube.asdf.converters.reorderedwcs_converter import ReorderedConverter + from ndcube.asdf.converters.resampled_converter import ResampledConverter + from ndcube.asdf.converters.tablecoord_converter import ( + MultipleTableCoordinateConverter, + QuantityTableCoordinateConverter, + SkyCoordTableCoordinateConverter, + TimeTableCoordConverter, + ) + ndcube_converters = [ + NDCubeConverter(), + ExtraCoordsConverter(), + TimeTableCoordConverter(), + QuantityTableCoordinateConverter(), + SkyCoordTableCoordinateConverter(), + MultipleTableCoordinateConverter(), + GlobalCoordsConverter(), + ResampledConverter(), + ReorderedConverter(), + CompoundConverter(), + NDCubeSequenceConverter(), + NDCollectionConverter(), + NDMetaConverter(), + ] + _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0" + + return [ + ManifestExtension.from_uri(_manifest_uri, converters=ndcube_converters) + ] diff --git a/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml new file mode 100644 index 000000000..9867abc4e --- /dev/null +++ b/ndcube/asdf/resources/manifests/ndcube-1.0.0.yaml @@ -0,0 +1,46 @@ +%YAML 1.1 +--- +id: asdf://sunpy.org/ndcube/manifests/ndcube-1.0.0 +extension_uri: asdf://sunpy.org/extensions/ndcube-1.0.0 +title: NDCube ASDF Manifest +description: ASDF schemas and tags for NDCube classes. + +tags: + - tag_uri: "tag:sunpy.org:ndcube/ndcube-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcube-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/extra_coords-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/timetablecoordinate-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/quantitytablecoordinate-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/skycoordtablecoordinate-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/extra_coords/table_coord/multipletablecoordinate-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/multipletablecoordinate-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/global_coords/globalcoords-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/global_coords-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/resampledwcs-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/resampledwcs-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/ndcubesequence-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcubesequence-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/reorderedwcs-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/reorderedwcs-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/compoundwcs-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/compoundwcs-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/ndcollection-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcollection-1.0.0" + + - tag_uri: "tag:sunpy.org:ndcube/meta/ndmeta-1.0.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndmeta-1.0.0" diff --git a/ndcube/asdf/resources/schemas/compoundwcs-1.0.0.yaml b/ndcube/asdf/resources/schemas/compoundwcs-1.0.0.yaml new file mode 100644 index 000000000..09eaa6dfe --- /dev/null +++ b/ndcube/asdf/resources/schemas/compoundwcs-1.0.0.yaml @@ -0,0 +1,27 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/compoundwcs-1.0.0" + +title: + Represents the ndcube CompoundLowLevelWCS object + +description: + Represents the ndcube CompoundLowLevelWCS object + +type: object +properties: + wcs: + type: array + items: + anyOf: + - tag: "tag:astropy.org:astropy/wcs/wcs-*" + - tag: "tag:stsci.edu:gwcs/wcs-*" + mapping: + type: array + atol: + type: number + +required: [wcs] +additionalProperties: false +... diff --git a/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml b/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml new file mode 100644 index 000000000..7470473ee --- /dev/null +++ b/ndcube/asdf/resources/schemas/extra_coords-1.0.0.yaml @@ -0,0 +1,39 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/extra_coords-1.0.0" + +title: + Represents the ndcube ExtraCoords object + +description: + Represents the ndcube ExtraCoords object + +type: object +properties: + wcs: + anyOf: + - tag: "tag:stsci.edu:gwcs/wcs-*" + - tag: "tag:astropy.org:astropy/wcs/wcs-*" + mapping: + type: array + lookup_tables: + type: array + items: + type: array + items: + - anyOf: + - type: number + - type: array + - anyOf: + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-1.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-1.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-1.*" + dropped_tables: + type: array + ndcube: + tag: "tag:sunpy.org:ndcube/ndcube-1.*" + +required: [ndcube] +additionalProperties: false +... diff --git a/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml b/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml new file mode 100644 index 000000000..2e4c00e14 --- /dev/null +++ b/ndcube/asdf/resources/schemas/global_coords-1.0.0.yaml @@ -0,0 +1,29 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/global_coords-1.0.0" + +title: + Represents the ndcube GlobalCoords object + +description: + Represents the ndcube GlobalCoords object + +type: object +properties: + internal_coords: + type: object + additionalProperties: + type: array + items: + - type: string + - type: object + anyOf: + - tag: "tag:stsci.edu:asdf/unit/quantity-*" + - tag: "tag:astropy.org:astropy/coordinates/skycoord-*" + ndcube: + tag: "tag:sunpy.org:ndcube/ndcube-1.*" + +required: [ndcube] +additionalProperties: false +... diff --git a/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml b/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml new file mode 100644 index 000000000..d355303b3 --- /dev/null +++ b/ndcube/asdf/resources/schemas/multipletablecoordinate-1.0.0.yaml @@ -0,0 +1,31 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/multipletablecoordinate-1.0.0" + +title: + Represents the MultipleTableCoordinate class + +description: + Represents the MultipleTableCoordinate class + +type: object +properties: + table_coords: + type: array + items: + anyOf: + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-1.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-1.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-1.*" + dropped_coords: + type: array + items: + anyOf: + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/timetablecoordinate-1.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/quantitytablecoordinate-1.*" + - tag: "tag:sunpy.org:ndcube/extra_coords/table_coord/skycoordtablecoordinate-1.*" + +required: ["table_coords", "dropped_coords"] +additionalProperties: false +... diff --git a/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml new file mode 100644 index 000000000..90c6afb6f --- /dev/null +++ b/ndcube/asdf/resources/schemas/ndcollection-1.0.0.yaml @@ -0,0 +1,32 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/ndcollection-1.0.0" + +title: + Represents the ndcube.ndcollection.NDCollection object + +description: + Represents the ndcube ndcube.ndcollection.NDCollection object + +type: object +properties: + meta: + type: object + items: + type: object + patternProperties: + ".*": + anyOf: + - tag: "tag:sunpy.org:ndcube/ndcube-1.*" + - tag: "tag:sunpy.org:ndcube/ndcubesequence-1.*" + # Allow any other objects here as we don't want to restrict what people can save + - type: object + aligned_axes: + anyOf: + - type: object + - type: string + +required: [items] +additionalProperties: true +... diff --git a/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml new file mode 100644 index 000000000..be97001a0 --- /dev/null +++ b/ndcube/asdf/resources/schemas/ndcube-1.0.0.yaml @@ -0,0 +1,43 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/ndcube-1.0.0" + +title: + Represents the ndcube NDCube object + +description: + Represents the ndcube NDCube object + +type: object +properties: + data: + description: "Must be compatible with ASDF serialization/deserialization and supported by NDCube." + wcs: + anyOf: + - tag: "tag:stsci.edu:gwcs/wcs-*" + - tag: "tag:astropy.org:astropy/wcs/wcs-*" + - tag: "tag:astropy.org:astropy/wcs/slicedwcs-*" + - tag: "tag:sunpy.org:ndcube/resampledwcs-*" + - tag: "tag:sunpy.org:ndcube/reorderedwcs-*" + - tag: "tag:sunpy.org:ndcube/compoundwcs-*" + extra_coords: + tag: "tag:sunpy.org:ndcube/extra_coords/extra_coords/extracoords-1.*" + global_coords: + tag: "tag:sunpy.org:ndcube/global_coords/globalcoords-1.*" + meta: + type: object + mask: + anyOf: + - tag: "tag:stsci.edu:asdf/core/ndarray-*" + - type: boolean + unit: + anyOf: + - tag: "tag:stsci.edu:asdf/unit/unit-*" + - tag: "tag:astropy.org:astropy/units/unit-*" + uncertainty: + tag: tag:astropy.org:astropy/nddata/uncertainty-* + +required: [data, wcs] +additionalProperties: true +... diff --git a/ndcube/asdf/resources/schemas/ndcubesequence-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndcubesequence-1.0.0.yaml new file mode 100644 index 000000000..f7f75e49f --- /dev/null +++ b/ndcube/asdf/resources/schemas/ndcubesequence-1.0.0.yaml @@ -0,0 +1,28 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/ndcubesequence-1.0.0" + +title: + Represents the ndcube.ndcube_sequence.NDCubeSequence object + +description: + Represents the ndcube.ndcube_sequence.NDCubeSequence object + +type: object +properties: + data: + type: array + items: + anyOf: + - tag: "tag:sunpy.org:ndcube/ndcube-1.*" + # Allow any other objects here as we don't want to restrict what people can save + - type: object + meta: + type: object + common_axis: + type: integer + +required: [data] +additionalProperties: true +... diff --git a/ndcube/asdf/resources/schemas/ndmeta-1.0.0.yaml b/ndcube/asdf/resources/schemas/ndmeta-1.0.0.yaml new file mode 100644 index 000000000..175e14405 --- /dev/null +++ b/ndcube/asdf/resources/schemas/ndmeta-1.0.0.yaml @@ -0,0 +1,27 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/ndmeta-1.0.0" + +title: + Represents the ndcube NDMeta object + +description: + Represents the ndcube NDMeta object + +type: object +properties: + meta: + type: object + key_comments: + type: object + axes: + type: object + data_shape: + tag: "tag:stsci.edu:asdf/core/ndarray-1.*" + original_meta: + type: object + +required: [meta, key_comments, axes, data_shape, original_meta] +additionalProperties: true +... diff --git a/ndcube/asdf/resources/schemas/quantitytablecoordinate-1.0.0.yaml b/ndcube/asdf/resources/schemas/quantitytablecoordinate-1.0.0.yaml new file mode 100644 index 000000000..5cd0e7501 --- /dev/null +++ b/ndcube/asdf/resources/schemas/quantitytablecoordinate-1.0.0.yaml @@ -0,0 +1,31 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/quantitytablecoordinate-1.0.0" + +title: + Represents the QuantityTableCoords class + +description: + Represents the QuantityTableCoords class + +type: object +properties: + unit: + anyOf: + - tag: "tag:stsci.edu:asdf/unit/unit-*" + - tag: "tag:astropy.org:astropy/units/unit-*" + table: + type: array + items: + tag: "tag:stsci.edu:asdf/unit/quantity-*" + names: + type: array + mesh: + type: boolean + physical_types: + type: array + +required: ["table", "unit"] +additionalProperties: false +... diff --git a/ndcube/asdf/resources/schemas/reorderedwcs-1.0.0.yaml b/ndcube/asdf/resources/schemas/reorderedwcs-1.0.0.yaml new file mode 100644 index 000000000..f01244466 --- /dev/null +++ b/ndcube/asdf/resources/schemas/reorderedwcs-1.0.0.yaml @@ -0,0 +1,25 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/resampledwcs-1.0.0" + +title: + Represents the ndcube ReorderedLowLevelWCS object + +description: + Represents the ndcube ReorderedLowLevelWCS object + +type: object +properties: + wcs: + anyOf: + - tag: "tag:astropy.org:astropy/wcs/wcs-*" + - tag: "tag:stsci.edu:gwcs/wcs-*" + pixel_order: + type: array + world_order: + type: array + +required: [wcs] +additionalProperties: false +... diff --git a/ndcube/asdf/resources/schemas/resampledwcs-1.0.0.yaml b/ndcube/asdf/resources/schemas/resampledwcs-1.0.0.yaml new file mode 100644 index 000000000..106707a06 --- /dev/null +++ b/ndcube/asdf/resources/schemas/resampledwcs-1.0.0.yaml @@ -0,0 +1,25 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/resampledwcs-1.0.0" + +title: + Represents the ndcube ResampledLowLevelWCS object + +description: + Represents the ndcube ResampledLowLevelWCS object + +type: object +properties: + wcs: + anyOf: + - tag: "tag:astropy.org:astropy/wcs/wcs-*" + - tag: "tag:stsci.edu:gwcs/wcs-*" + factor: + tag: "tag:stsci.edu:asdf/core/ndarray-1.*" + offset: + tag: "tag:stsci.edu:asdf/core/ndarray-1.*" + +required: [wcs] +additionalProperties: false +... diff --git a/ndcube/asdf/resources/schemas/skycoordtablecoordinate-1.0.0.yaml b/ndcube/asdf/resources/schemas/skycoordtablecoordinate-1.0.0.yaml new file mode 100644 index 000000000..1607e637a --- /dev/null +++ b/ndcube/asdf/resources/schemas/skycoordtablecoordinate-1.0.0.yaml @@ -0,0 +1,25 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/skycoordtablecoordinate-1.0.0" + +title: + Represents the SkyCoordTableCoordinate class + +description: + Represents the SkyCoordTableCoordinate class + +type: object +properties: + table: + tag: "tag:astropy.org:astropy/coordinates/skycoord-*" + names: + type: array + mesh: + type: boolean + physical_types: + type: array + +required: ["table"] +additionalProperties: false +... diff --git a/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml b/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml new file mode 100644 index 000000000..1e60d3ebf --- /dev/null +++ b/ndcube/asdf/resources/schemas/timetablecoordinate-1.0.0.yaml @@ -0,0 +1,25 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/timetablecoordinate-1.0.0" + +title: + Represents the TimeTableCoords class + +description: + Represents the TimeTableCoords class + +type: object +properties: + table: + tag: "tag:stsci.edu:asdf/time/time-*" + names: + type: array + physical_types: + type: array + reference_time: + tag: "tag:stsci.edu:asdf/time/time-*" + +required: ["table"] +additionalProperties: false +... diff --git a/ndcube/conftest.py b/ndcube/conftest.py index e3ee125c4..157ae74b4 100644 --- a/ndcube/conftest.py +++ b/ndcube/conftest.py @@ -7,15 +7,24 @@ import dask.array import numpy as np import pytest +from gwcs import coordinate_frames as cf +from gwcs import wcs import astropy.nddata import astropy.units as u +from astropy import coordinates as coord from astropy.coordinates import SkyCoord +from astropy.modeling import models from astropy.nddata import StdDevUncertainty from astropy.time import Time, TimeDelta from astropy.wcs import WCS from ndcube import ExtraCoords, GlobalCoords, NDCube, NDCubeSequence, NDMeta +from ndcube.extra_coords.table_coord import ( + QuantityTableCoordinate, + SkyCoordTableCoordinate, + TimeTableCoordinate, +) from ndcube.tests import helpers # Force MPL to use non-gui backends for testing. @@ -36,6 +45,9 @@ # Helper Functions ################################################################################ +def time_lut(shape): + base_time = Time('2000-01-01', format='fits', scale='utc') + return Time([base_time + TimeDelta(60 * i, format='sec') for i in range(shape[0])]) def skycoord_2d_lut(shape): total_len = np.prod(shape) @@ -82,6 +94,136 @@ def gen_ndcube_3d_l_ln_lt_ectime(wcs_3d_lt_ln_l, time_axis, time_base, global_co # WCS Fixtures ################################################################################ +@pytest.fixture +def gwcs_4d_t_l_lt_ln(): + """ + Creates a 4D GWCS object with time, wavelength, and celestial coordinates. + + - Time: Axis 0 + - Wavelength: Axis 1 + - Sky: Axes 2 and 3 + + Returns: + wcs.WCS: 4D GWCS object. + """ + + time_model = models.Identity(1) + time_frame = cf.TemporalFrame(axes_order=(0, ), unit=u.s, + reference_frame=Time("2000-01-01T00:00:00")) + + wave_frame = cf.SpectralFrame(axes_order=(1, ), unit=u.m, axes_names=('wavelength',)) + wave_model = models.Scale(0.2) + + shift = models.Shift(-5) & models.Shift(0) + scale = models.Scale(5) & models.Scale(20) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial(0, 0, 180) + cel_model = shift | scale | tan | celestial_rotation + sky_frame = cf.CelestialFrame(axes_order=(2, 3), name='icrs', + reference_frame=coord.ICRS(), + axes_names=("longitude", "latitude")) + + transform = time_model & wave_model & cel_model + + frame = cf.CompositeFrame([time_frame, wave_frame, sky_frame]) + detector_frame = cf.CoordinateFrame(name="detector", naxes=4, + axes_order=(0, 1, 2, 3), + axes_type=("pixel", "pixel", "pixel", "pixel"), + unit=(u.pix, u.pix, u.pix, u.pix)) + + return (wcs.WCS(forward_transform=transform, output_frame=frame, input_frame=detector_frame)) + +@pytest.fixture +def gwcs_3d_lt_ln_l(): + """ + Creates a 3D GWCS object with celestial coordinates and wavelength. + + - Sky: Axes 0 and 1 + - Wavelength: Axis 2 + + Returns: + wcs.WCS: 3D GWCS object. + """ + + shift = models.Shift(-5) & models.Identity(1) + scale = models.Scale(5) & models.Scale(10) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial(0, 0, 180) + cel_model = shift | scale | tan | celestial_rotation + sky_frame = cf.CelestialFrame(axes_order=(0, 1), name='icrs', + reference_frame=coord.ICRS(), + axes_names=("longitude", "latitude")) + + wave_model = models.Identity(1) | models.Scale(0.2) | models.Shift(10) + wave_frame = cf.SpectralFrame(axes_order=(2, ), unit=u.nm, axes_names=("wavelength",)) + + transform = cel_model & wave_model + + frame = cf.CompositeFrame([sky_frame, wave_frame]) + detector_frame = cf.CoordinateFrame(name="detector", naxes=3, + axes_order=(0, 1, 2), + axes_type=("pixel", "pixel", "pixel"), + axes_names=("x", "y", "z"), unit=(u.pix, u.pix, u.pix)) + + return (wcs.WCS(forward_transform=transform, output_frame=frame, input_frame=detector_frame)) + +@pytest.fixture +def gwcs_3d_ln_lt_t_rotated(): + """ + Creates a 3D GWCS object with celestial coordinates and wavelength, including rotation. + + - Sky: Axes 0 and 1 + - Wavelength: Axis 2 + + Returns: + wcs.WCS: 3D GWCS object with rotation. + """ + shift = models.Shift(-5) & models.Identity(1) + scale = models.Scale(5) & models.Scale(10) + matrix = np.array([[1.290551569736E-05, 5.9525007864732E-06], + [5.0226382102765E-06 , -1.2644844123757E-05]]) + rotation = models.AffineTransformation2D(matrix) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial(0, 0, 180) + cel_model = shift | scale| rotation | tan | celestial_rotation + sky_frame = cf.CelestialFrame(axes_order=(0, 1), name='icrs', + reference_frame=coord.ICRS(), + axes_names=("longitude", "latitude")) + + wave_model = models.Identity(1) | models.Scale(0.2) | models.Shift(10) + wave_frame = cf.SpectralFrame(axes_order=(2, ), unit=u.nm, axes_names=("wavelength",)) + + transform = cel_model & wave_model + + frame = cf.CompositeFrame([sky_frame, wave_frame]) + detector_frame = cf.CoordinateFrame(name="detector", naxes=3, + axes_order=(0, 1, 2), + axes_type=("pixel", "pixel", "pixel"), + axes_names=("x", "y", "z"), unit=(u.pix, u.pix, u.pix)) + + return (wcs.WCS(forward_transform=transform, output_frame=frame, input_frame=detector_frame)) + +@pytest.fixture +def gwcs_2d_lt_ln(): + """ + Creates a 2D GWCS object with celestial coordinates. + + - Sky: Axes 0 and 1 + + Returns: + wcs.WCS: 2D GWCS object. + """ + shift = models.Shift(-5) & models.Shift(-5) + scale = models.Scale(2) & models.Scale(4) + tan = models.Pix2Sky_TAN() + celestial_rotation = models.RotateNative2Celestial(0, 0, 180) + cel_model = shift | scale | tan | celestial_rotation + input_frame = cf.Frame2D(name="detector", axes_names=("x", "y")) + sky_frame = cf.CelestialFrame(axes_order=(0, 1), name='icrs', + reference_frame=coord.ICRS(), + axes_names=("longitude", "latitude")) + + return (wcs.WCS(forward_transform=cel_model, output_frame=sky_frame, input_frame=input_frame)) @pytest.fixture def wcs_4d_t_l_lt_ln(): @@ -420,8 +562,79 @@ def extra_coords_sharing_axis(): ################################################################################ # NDCube Fixtures +# NOTE: If you add more fixtures please add to the all_ndcubes fixture ################################################################################ +@pytest.fixture +def ndcube_gwcs_4d_ln_lt_l_t(gwcs_4d_t_l_lt_ln): + shape = (5, 8, 10, 12) + gwcs_4d_t_l_lt_ln.array_shape = shape + data_cube = data_nd(shape) + return NDCube(data_cube, wcs=gwcs_4d_t_l_lt_ln) + + +@pytest.fixture +def ndcube_gwcs_4d_ln_lt_l_t_unit(gwcs_4d_t_l_lt_ln): + shape = (5, 8, 10, 12) + gwcs_4d_t_l_lt_ln.array_shape = shape + data_cube = data_nd(shape) + return NDCube(data_cube, wcs=gwcs_4d_t_l_lt_ln, unit=u.DN) + + +@pytest.fixture +def ndcube_gwcs_3d_ln_lt_l(gwcs_3d_lt_ln_l): + shape = (2, 3, 4) + gwcs_3d_lt_ln_l.array_shape = shape + data_cube = data_nd(shape) + return NDCube(data_cube, wcs=gwcs_3d_lt_ln_l) + + +@pytest.fixture +def ndcube_gwcs_3d_rotated(gwcs_3d_lt_ln_l, simple_extra_coords_3d): + data_rotated = np.array([[[1, 2, 3, 4, 6], [2, 4, 5, 3, 1], [0, -1, 2, 4, 2], [3, 5, 1, 2, 0]], + [[2, 4, 5, 1, 3], [1, 5, 2, 2, 4], [2, 3, 4, 0, 5], [0, 1, 2, 3, 4]]]) + cube = NDCube( + data_rotated, + wcs=gwcs_3d_lt_ln_l) + cube._extra_coords = simple_extra_coords_3d + return cube + + +@pytest.fixture +def ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim(gwcs_3d_lt_ln_l, time_and_simple_extra_coords_2d): + shape = (2, 3, 4) + gwcs_3d_lt_ln_l.array_shape = shape + data_cube = data_nd(shape) + cube = NDCube(data_cube, wcs=gwcs_3d_lt_ln_l) + cube._extra_coords = time_and_simple_extra_coords_2d[0] + return cube + + +@pytest.fixture +def ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc(gwcs_3d_lt_ln_l): + shape = (3, 3, 4) + gwcs_3d_lt_ln_l.array_shape = shape + data_cube = data_nd(shape) + cube = NDCube(data_cube, wcs=gwcs_3d_lt_ln_l) + coord1 = 1 * u.m + cube.global_coords.add('name1', 'custom:physical_type1', coord1) + cube.extra_coords.add("time", 0, time_lut(shape)) + cube.extra_coords.add("exposure_lut", 1, range(shape[1]) * u.s) + return cube + + +@pytest.fixture +def ndcube_gwcs_2d_ln_lt_mask(gwcs_2d_lt_ln): + shape = (10, 12) + data_cube = data_nd(shape) + mask = np.zeros(shape, dtype=bool) + mask[1, 1] = True + mask[2, 0] = True + mask[3, 3] = True + mask[4:6, :4] = True + return NDCube(data_cube, wcs=gwcs_2d_lt_ln, mask=mask) + + @pytest.fixture def ndcube_4d_ln_l_t_lt(wcs_4d_lt_t_l_ln): shape = (5, 10, 12, 8) @@ -849,26 +1062,59 @@ def ndcube_1d_l(wcs_1d_l): @pytest.fixture(params=[ + "ndcube_gwcs_4d_ln_lt_l_t", + "ndcube_gwcs_4d_ln_lt_l_t_unit", + "ndcube_gwcs_3d_ln_lt_l", + "ndcube_gwcs_3d_rotated", + "ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim", + "ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc", + "ndcube_gwcs_2d_ln_lt_mask", + "ndcube_4d_ln_l_t_lt", "ndcube_4d_ln_lt_l_t", + "ndcube_4d_axis_aware_meta", "ndcube_4d_uncertainty", "ndcube_4d_mask", "ndcube_4d_extra_coords", "ndcube_4d_unit_uncertainty", "ndcube_3d_ln_lt_l", + "ndcube_3d_ln_lt_l_ec_all_axes", + "ndcube_3d_ln_lt_l_ec_sharing_axis", + "ndcube_3d_ln_lt_l_ec_time", + "ndcube_3d_wave_lt_ln_ec_time", "ndcube_3d_rotated", + "ndcube_3d_coupled", + "ndcube_3d_coupled_time", + "ndcube_3d_l_ln_lt_ectime", "ndcube_2d_ln_lt", + "ndcube_2d_ln_lt_uncert", + "ndcube_2d_ln_lt_mask_uncert", + "ndcube_2d_ln_lt_mask_uncert_unit_mask_false", + "ndcube_2d_ln_lt_mask_uncert_unit_one_maskele_true", + "ndcube_2d_ln_lt_mask_uncert_unit_one_maskele_true_expected_unmask_false", + "ndcube_2d_ln_lt_mask_uncert_unit_one_maskele_true_expected_unmask_true", + "ndcube_2d_ln_lt_mask_uncert_unit_mask_true", + "ndcube_2d_ln_lt_mask_uncert_unit_mask_true_expected_unmask_true", + "ndcube_2d_ln_lt_mask_uncert_unit_mask_true_expected_unmask_false", + "ndcube_2d_ln_lt_uncert_ec", "ndcube_2d_ln_lt_units", - "ndcube_2d_dask", - "ndcube_1d_l", "ndcube_2d_ln_lt_no_unit_no_unc", - "ndcube_2d_uncertainty_no_unit", "ndcube_2d_unit_unc", + "ndcube_2d_uncertainty_no_unit", + "ndcube_2d_ln_lt_mask", + "ndcube_2d_ln_lt_mask2", + "ndcube_2d_ln_lt_nomask", + "ndcube_2d_dask", + "ndcube_1d_l", ]) -def all_ndcubes(request): +def all_ndcubes_names(request): + return request.param + +@pytest.fixture +def all_ndcubes(request, all_ndcubes_names): """ All the above ndcube fixtures in order. """ - return request.getfixturevalue(request.param) + return request.getfixturevalue(all_ndcubes_names) @pytest.fixture @@ -942,6 +1188,65 @@ def ndcubesequence_3c_l_ln_lt_cax1(wcs_3d_lt_ln_l): return NDCubeSequence([cube1, cube2, cube3], common_axis=common_axis) +################################################################################ +# Table Coordinates +################################################################################ + + +@pytest.fixture +def lut_1d_distance(): + lookup_table = u.Quantity(np.arange(10) * u.km) + return QuantityTableCoordinate(lookup_table, names='x') + + +@pytest.fixture +def lut_3d_distance_mesh(): + lookup_table = (u.Quantity(np.arange(10) * u.km), + u.Quantity(np.arange(10, 20) * u.km), + u.Quantity(np.arange(20, 30) * u.km)) + + return QuantityTableCoordinate(*lookup_table, names=['x', 'y', 'z']) + + +@pytest.fixture +def lut_1d_skycoord_no_mesh(): + sc = SkyCoord(range(10), range(10), unit=u.deg) + return SkyCoordTableCoordinate(sc, mesh=False, names=['lon', 'lat']) + + +@pytest.fixture +def lut_2d_skycoord_no_mesh(): + data = np.arange(9).reshape(3, 3), np.arange(9, 18).reshape(3, 3) + sc = SkyCoord(*data, unit=u.deg) + return SkyCoordTableCoordinate(sc, mesh=False) + + +@pytest.fixture +def lut_2d_skycoord_mesh(): + sc = SkyCoord(range(10), range(10), unit=u.deg) + return SkyCoordTableCoordinate(sc, mesh=True) + + +@pytest.fixture +def lut_3d_skycoord_mesh(): + sc = SkyCoord(range(10), range(10), range(10), unit=(u.deg, u.deg, u.AU)) + return SkyCoordTableCoordinate(sc, mesh=True) + + +@pytest.fixture +def lut_1d_time(): + data = Time(["2011-01-01T00:00:00", + "2011-01-01T00:00:10", + "2011-01-01T00:00:20", + "2011-01-01T00:00:30"], format="isot") + return TimeTableCoordinate(data, names='time', physical_types='time') + + +@pytest.fixture +def lut_1d_wave(): + # TODO: Make this into a SpectralCoord object + return QuantityTableCoordinate(range(10) * u.nm) + def pytest_runtest_teardown(item): # Clear the pyplot figure stack if it is not empty after the test diff --git a/ndcube/extra_coords/table_coord.py b/ndcube/extra_coords/table_coord.py index c1290ed98..e975393d4 100644 --- a/ndcube/extra_coords/table_coord.py +++ b/ndcube/extra_coords/table_coord.py @@ -13,7 +13,7 @@ from astropy.coordinates import SkyCoord from astropy.modeling import models from astropy.modeling.models import tabular_model -from astropy.modeling.tabular import _Tabular +from astropy.modeling.tabular import Tabular1D, Tabular2D, _Tabular from astropy.time import Time from astropy.wcs.wcsapi.wrappers.sliced_wcs import combine_slices, sanitize_slices @@ -22,7 +22,13 @@ except ImportError: pass -__all__ = ["BaseTableCoordinate", "MultipleTableCoordinate", 'QuantityTableCoordinate', 'SkyCoordTableCoordinate', 'TimeTableCoordinate'] +__all__ = [ + "BaseTableCoordinate", + "MultipleTableCoordinate", + "QuantityTableCoordinate", + "SkyCoordTableCoordinate", + "TimeTableCoordinate", +] class Length1Tabular(_Tabular): @@ -142,7 +148,17 @@ def _generate_tabular(lookup_table, interpolation='linear', points_unit=u.pix, * raise TypeError("lookup_table must be a Quantity.") # pragma: no cover ndim = lookup_table.ndim - TabularND = tabular_model(ndim, name=f"Tabular{ndim}D") + + # Use existing Tabular1D and Tabular2D classes for 1D and 2D models + # to ensure compatibility with asdf-astropy converters and avoid + # dynamically generated classes that are not recognized by asdf. + # See PR #751 for details on the issue this addresses. + if ndim == 1: + TabularND = Tabular1D + elif ndim == 2: + TabularND = Tabular2D + else: + TabularND = tabular_model(ndim, name=f"Tabular{ndim}D") # The integer location is at the centre of the pixel. points = [(np.arange(size) - 0) * points_unit for size in lookup_table.shape] diff --git a/ndcube/extra_coords/tests/test_lookup_table_coord.py b/ndcube/extra_coords/tests/test_lookup_table_coord.py index 36974ac3c..35ae13886 100644 --- a/ndcube/extra_coords/tests/test_lookup_table_coord.py +++ b/ndcube/extra_coords/tests/test_lookup_table_coord.py @@ -15,68 +15,6 @@ ) -@pytest.fixture -def lut_1d_distance(): - lookup_table = u.Quantity(np.arange(10) * u.km) - return QuantityTableCoordinate(lookup_table, names='x') - - -@pytest.fixture -def lut_3d_distance_mesh(): - lookup_table = (u.Quantity(np.arange(10) * u.km), - u.Quantity(np.arange(10, 20) * u.km), - u.Quantity(np.arange(20, 30) * u.km)) - - return QuantityTableCoordinate(*lookup_table, names=['x', 'y', 'z']) - - -@pytest.fixture -def lut_2d_distance_no_mesh(): - # Fixture is broken and raises: Currently all tables must be 1-D - lookup_table = np.arange(9).reshape(3, 3) * u.km, np.arange(9, 18).reshape(3, 3) * u.km - return QuantityTableCoordinate(*lookup_table, mesh=False) - - -@pytest.fixture -def lut_1d_skycoord_no_mesh(): - sc = SkyCoord(range(10), range(10), unit=u.deg) - return SkyCoordTableCoordinate(sc, mesh=False, names=['lon', 'lat']) - - -@pytest.fixture -def lut_2d_skycoord_no_mesh(): - data = np.arange(9).reshape(3, 3), np.arange(9, 18).reshape(3, 3) - sc = SkyCoord(*data, unit=u.deg) - return SkyCoordTableCoordinate(sc, mesh=False) - - -@pytest.fixture -def lut_2d_skycoord_mesh(): - sc = SkyCoord(range(10), range(10), unit=u.deg) - return SkyCoordTableCoordinate(sc, mesh=True) - - -@pytest.fixture -def lut_3d_skycoord_mesh(): - sc = SkyCoord(range(10), range(10), range(10), unit=(u.deg, u.deg, u.AU)) - return SkyCoordTableCoordinate(sc, mesh=True) - - -@pytest.fixture -def lut_1d_time(): - data = Time(["2011-01-01T00:00:00", - "2011-01-01T00:00:10", - "2011-01-01T00:00:20", - "2011-01-01T00:00:30"], format="isot") - return TimeTableCoordinate(data, names='time', physical_types='time') - - -@pytest.fixture -def lut_1d_wave(): - # TODO: Make this into a SpectralCoord object - return QuantityTableCoordinate(range(10) * u.nm) - - def test_exceptions(): with pytest.raises(TypeError) as ei: QuantityTableCoordinate(u.Quantity([1, 2, 3], u.nm), [1, 2, 3]) diff --git a/ndcube/meta.py b/ndcube/meta.py index 00255de67..5a10b113c 100644 --- a/ndcube/meta.py +++ b/ndcube/meta.py @@ -184,11 +184,11 @@ def _sanitize_axis_value(self, axis, value, key): if isinstance(axis, numbers.Integral): axis = (axis,) if len(axis) == 0: - return ValueError(axis_err_msg) + raise ValueError(axis_err_msg) # Verify each entry in axes is an iterable of ints or a scalar. if not (isinstance(axis, collections.abc.Iterable) and all(isinstance(i, numbers.Integral) for i in axis)): - return ValueError(axis_err_msg) + raise ValueError(axis_err_msg) # If metadata's axis/axes include axis beyond current data shape, extend it. data_shape = self.data_shape if max(axis) >= len(data_shape): diff --git a/ndcube/tests/helpers.py b/ndcube/tests/helpers.py index 5381fdeca..7d6c5409b 100644 --- a/ndcube/tests/helpers.py +++ b/ndcube/tests/helpers.py @@ -107,10 +107,12 @@ def assert_metas_equal(test_input, expected_output): else: assert np.allclose(test_input.data_shape, expected_output.data_shape) - for test_value, expected_value in zip(test_input.values(), expected_output.values()): + for key in test_input.keys(): + test_value = test_input[key] + expected_value = expected_output[key] try: assert test_value == expected_value - except ValueError as err: # noqa: PERF203 + except ValueError as err: if multi_element_msg in err.args[0]: if test_value.dtype.kind in ('S', 'U'): # If the values are strings, we can compare them as arrays. @@ -126,7 +128,7 @@ def assert_metas_equal(test_input, expected_output): assert test_input[key] == expected_output[key] -def assert_cubes_equal(test_input, expected_cube, check_data=True, check_uncertainty_values=False): +def assert_cubes_equal(test_input, expected_cube, check_data=True, check_uncertainty_values=False, rtol=None, atol=None): assert isinstance(test_input, type(expected_cube)) if isinstance(test_input.mask, bool): if not isinstance(expected_cube.mask, bool): @@ -136,7 +138,7 @@ def assert_cubes_equal(test_input, expected_cube, check_data=True, check_uncerta assert np.all(test_input.mask == expected_cube.mask) if check_data: np.testing.assert_array_equal(test_input.data, expected_cube.data) - assert_wcs_are_equal(test_input.wcs, expected_cube.wcs) + assert_wcs_are_equal(test_input.wcs, expected_cube.wcs, rtol=rtol, atol=atol) if check_uncertainty_values: # Check output and expected uncertainty are of same type. Remember they could be None. # If the uncertainties are not None,... @@ -169,7 +171,7 @@ def assert_cubesequences_equal(test_input, expected_sequence, check_data=True): assert_cubes_equal(cube, expected_sequence.data[i], check_data=check_data) -def assert_wcs_are_equal(wcs1, wcs2): +def assert_wcs_are_equal(wcs1, wcs2, *, rtol=None, atol=None): """ Assert function for testing two wcs object. @@ -177,6 +179,8 @@ def assert_wcs_are_equal(wcs1, wcs2): Also checks if both the wcs objects are instance of `~astropy.wcs.wcsapi.SlicedLowLevelWCS`. """ + atol = atol or 1e-23 + rtol = rtol or 1e-17 if not isinstance(wcs1, BaseLowLevelWCS): wcs1 = wcs1.low_level_wcs @@ -196,7 +200,12 @@ def assert_wcs_are_equal(wcs1, wcs2): # SlicedLowLevelWCS vs BaseHighLevelWCS don't have the same pixel_to_world method low_level_wcs1 = wcs1.low_level_wcs if isinstance(wcs1, BaseHighLevelWCS) else wcs1 low_level_wcs2 = wcs2.low_level_wcs if isinstance(wcs2, BaseHighLevelWCS) else wcs2 - np.testing.assert_array_equal(low_level_wcs1.pixel_to_world_values(*random_idx.T), low_level_wcs2.pixel_to_world_values(*random_idx.T)) + np.testing.assert_allclose( + low_level_wcs1.pixel_to_world_values(*random_idx.T), + low_level_wcs2.pixel_to_world_values(*random_idx.T), + atol=atol, + rtol=rtol, + ) def create_sliced_wcs(wcs, item, dim): """ diff --git a/pyproject.toml b/pyproject.toml index 0fb95c798..90dd1e11a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,16 +16,17 @@ authors = [ { name = "The SunPy Community", email = "sunpy@googlegroups.com" }, ] dependencies = [ - "astropy>=5.0.6,!=5.1.0", - "gwcs>=0.18", - "numpy>=1.23.0", - "scipy>=1.8.0", + "astropy>=5.3.0", + "gwcs>=0.20", + "numpy>=1.25.0", + "scipy>=1.11.1", ] dynamic = ["version"] [project.optional-dependencies] -tests = [ +tests-only = [ "dask", + "pytest-asdf-plugin", "pytest-astropy", "pytest-cov", "pytest-doctestplus", @@ -33,34 +34,39 @@ tests = [ "pytest-xdist", "pytest", "pytest-memray; sys_platform != 'win32'", - "scipy", "specutils", "sunpy>=5.0.0", ] +tests = [ + "ndcube[plotting,reproject,asdf,tests-only]", +] docs = [ + "ndcube[plotting]", "sphinx", "sphinx-automodapi", "sunpy-sphinx-theme", "packaging", - "matplotlib", - "mpl-animators>=1.0", "sphinx-changelog>=1.1.0", "sphinx-gallery", "sphinxext-opengraph", "sunpy>=5.0.0", ] plotting = [ - "matplotlib>=3.5.0", - "mpl_animators>=1.0", + "matplotlib>=3.8.0", + "mpl_animators>=1.1", ] reproject = [ - "reproject>=0.7.1", + "reproject>=0.11.0", +] +asdf = [ + "asdf>=2.15.0", + "asdf-astropy>=0.7.0", ] all = [ "ndcube[plotting,reproject]", ] dev = [ - "ndcube[tests,docs,plotting,reproject]", + "ndcube[tests-only,docs,plotting,reproject,asdf]", ] [project.urls] @@ -115,6 +121,14 @@ version_file = "ndcube/_version.py" number_incorrect_long = "The number in the changelog file you added does not match the number of this pull request. Please rename the file." + +[project.entry-points."asdf.resource_mappings"] +ndcube = "ndcube.asdf.entry_points:get_resource_mappings" + +[project.entry-points."asdf.extensions"] +ndcube = "ndcube.asdf.entry_points:get_extensions" + + # TODO: This should be in towncrier.toml but Giles currently only works looks in # pyproject.toml we should move this back when it's fixed. [tool.towncrier] diff --git a/pytest.ini b/pytest.ini index cd1290262..cae4a9c0d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -26,6 +26,8 @@ addopts = -m "not mpl_image_compare" --doctest-ignore-import-errors --doctest-continue-on-failure +asdf_schema_tests_enabled = true +asdf_schema_root = ndcube/asdf/resources mpl-results-path = figure_test_images mpl-use-full-test-name = true remote_data_strict = True diff --git a/tox.ini b/tox.ini index e7e613ffe..6ab3a47e6 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = py310-oldestdeps codestyle build_docs + asdf_schemas [testenv] # We use bash in some of our environments so we have to whitelist it. @@ -62,11 +63,10 @@ deps = figure-!devdeps: scipy # The following indicates which extras_require will be installed extras = - plotting - reproject - tests + !oldestdeps: tests + oldestdeps: tests-only commands_pre = - oldestdeps: minimum_dependencies ndcube --filename requirements-min.txt + oldestdeps: minimum_dependencies ndcube --extras plotting reproject asdf --filename requirements-min.txt oldestdeps: pip install -r requirements-min.txt oldestdeps: python -c "import astropy.time; astropy.time.update_leap_seconds()" pip freeze --all --no-input @@ -94,6 +94,16 @@ commands = {toxinidir}/docs \ {posargs} +[testenv:asdf_schemas] +description = Run schema tests +deps = + pytest + asdf +set_env = + asdf_schema_root = {toxinidir}/ndcube/asdf/resources +commands = + pytest {env:asdf_schema_root} + [testenv:build_docs] change_dir = docs description = Invoke sphinx-build to build the HTML docs