From f3a53833c1e2b04495a853b6d27d7cb7f5c195a4 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 16 Sep 2025 20:30:56 -0500 Subject: [PATCH 1/7] Initial support for Zarr v3 --- build.gradle | 1 + .../bioformats2raw/Converter.java | 389 ++++++++++-------- .../bioformats2raw/ZarrTypes.java | 164 ++++++++ 3 files changed, 388 insertions(+), 166 deletions(-) create mode 100644 src/main/java/com/glencoesoftware/bioformats2raw/ZarrTypes.java diff --git a/build.gradle b/build.gradle index c124bc2..b93e7a8 100644 --- a/build.gradle +++ b/build.gradle @@ -41,6 +41,7 @@ dependencies { implementation 'info.picocli:picocli:4.7.5' implementation 'com.univocity:univocity-parsers:2.8.4' implementation 'dev.zarr:jzarr:0.4.2' + implementation 'dev.zarr:zarr-java:0.0.4' // implementation 'org.carlspring.cloud.aws:s3fs-nio:1.0-SNAPSHOT' // implementation 'io.nextflow:nxf-s3fs:1.1.0' implementation 'org.lasersonlab:s3fs:2.2.3' diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index 29c9dd0..15aaae9 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -81,12 +81,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.bc.zarr.ArrayParams; -import com.bc.zarr.CompressorFactory; -import com.bc.zarr.DataType; -import com.bc.zarr.DimensionSeparator; -import com.bc.zarr.ZarrArray; -import com.bc.zarr.ZarrGroup; import com.glencoesoftware.bioformats2raw.MiraxReader.TilePointer; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -100,6 +94,27 @@ import picocli.CommandLine.Parameters; import ucar.ma2.InvalidRangeException; +// jzarr: v2 + +import com.bc.zarr.ArrayParams; +import com.bc.zarr.CompressorFactory; +import com.bc.zarr.DataType; +import com.bc.zarr.DimensionSeparator; +import com.bc.zarr.ZarrArray; +import com.bc.zarr.ZarrGroup; + +// zarr-java: v3 + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; +//import dev.zarr.zarrjava.store.Store; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.utils.Utils; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.Group; +//import dev.zarr.zarrjava.v3.Node; +//import dev.zarr.zarrjava.v3.codec.CodecBuilder; + /** * Command line tool for converting whole slide imaging files to Zarr. */ @@ -126,6 +141,7 @@ public class Converter implements Callable { /** NGFF specification version.*/ public static final String NGFF_VERSION = "0.4"; + public static final String NGFF_VERSION_V3 = "0.5"; private volatile Path inputPath; private volatile String outputLocation; @@ -145,6 +161,9 @@ public class Converter implements Callable { private volatile boolean help = false; private volatile boolean originalMetadata = true; + private volatile boolean v3 = false; + private volatile FilesystemStore v3Store = null; + private volatile int maxWorkers; private volatile int maxCachedTiles; private volatile ZarrCompression compressionType; @@ -678,6 +697,21 @@ public void setAdditionalScaleFormatCSV(Path scaleFormatCSV) { additionalScaleFormatStringArgsCsv = scaleFormatCSV; } + /** + * Set whether or not to use Zarr v3. + * By default, v2 is used. + * + * @param useV3 true if Zarr v3 should be written + */ + @Option( + names = "--v3", + description = "Write Zarr v3 data", + defaultValue = "false" + ) + public void setV3(boolean useV3) { + v3 = useV3; + } + /** * Set the directory for memo (metadata cache) files. * @@ -1104,6 +1138,13 @@ public Path getAdditionalScaleFormatCSV() { return additionalScaleFormatStringArgsCsv; } + /** + * @return true if Zarr v3 data should be written + */ + public boolean getV3() { + return v3; + } + /** * @return directory containing memo (metadata cache) files */ @@ -1353,7 +1394,7 @@ public Integer call() throws Exception { */ public void convert() throws FormatException, IOException, InterruptedException, - EnumerationException + EnumerationException, ZarrException { checkOutputPaths(); @@ -1510,38 +1551,7 @@ public void convert() scaleFormatString = "%s/%s/%d/%d"; } - // fileset level metadata - if (!noRootGroup) { - final ZarrGroup root = ZarrGroup.create(getRootPath()); - Map attributes = new HashMap(); - attributes.put("bioformats2raw.layout", LAYOUT); - - root.writeAttributes(attributes); - } - if (!noOMEMeta) { - Path metadataPath = getRootPath().resolve("OME"); - final ZarrGroup root = ZarrGroup.create(metadataPath); - Map attributes = new HashMap(); - - // record the path to each series (multiscales) and the corresponding - // series (OME-XML Image) index - // using the index as the key would mean that the index is stored - // as a string instead of an integer - List groups = new ArrayList(); - for (Integer index : seriesList) { - String resolutionString = String.format( - scaleFormatString, getScaleFormatStringArgs(index, 0)); - String seriesString = ""; - if (resolutionString.indexOf('/') >= 0) { - seriesString = resolutionString.substring(0, - resolutionString.lastIndexOf('/')); - } - groups.add(seriesString); - } - attributes.put("series", groups); - - root.writeAttributes(attributes); - } + writeZarrMetadata(); // pre-calculate resolution and tile counts long totalTiles = 0; @@ -1592,6 +1602,57 @@ public void convert() } } + private void writeZarrMetadata() throws IOException, ZarrException { + if (getV3() && v3Store == null) { + v3Store = new FilesystemStore(getRootPath()); + } + + // fileset level metadata + if (!noRootGroup) { + Map attributes = new HashMap(); + attributes.put("bioformats2raw.layout", LAYOUT); + + if (getV3()) { + Group v3Root = Group.create(v3Store.resolve()); + v3Root.setAttributes(attributes); + } + else { + final ZarrGroup root = ZarrGroup.create(getRootPath()); + root.writeAttributes(attributes); + } + } + if (!noOMEMeta) { + Map attributes = new HashMap(); + + // record the path to each series (multiscales) and the corresponding + // series (OME-XML Image) index + // using the index as the key would mean that the index is stored + // as a string instead of an integer + List groups = new ArrayList(); + for (Integer index : seriesList) { + String resolutionString = String.format( + scaleFormatString, getScaleFormatStringArgs(index, 0)); + String seriesString = ""; + if (resolutionString.indexOf('/') >= 0) { + seriesString = resolutionString.substring(0, + resolutionString.lastIndexOf('/')); + } + groups.add(seriesString); + } + attributes.put("series", groups); + + if (getV3()) { + Group v3OME = Group.create(v3Store.resolve("OME")); + v3OME.setAttributes(attributes); + } + else { + Path metadataPath = getRootPath().resolve("OME"); + final ZarrGroup root = ZarrGroup.create(metadataPath); + root.writeAttributes(attributes); + } + } + } + /** * Pre-calculate the number of resolutions and tiles for the * given series index. This is useful for accurate progress reporting @@ -1703,7 +1764,7 @@ private int[] calculateTileCounts(int series) */ public void write(int series) throws FormatException, IOException, InterruptedException, - EnumerationException + EnumerationException, ZarrException { readers.forEach((reader) -> { reader.setSeries(series); @@ -1789,28 +1850,23 @@ private Object[] getScaleFormatStringArgs( } /** - * Return the number of bytes per pixel for a JZarr data type. - * @param dataType type to return number of bytes per pixel for - * @return See above. + * Read tile as bytes from typed Zarr v3 array. + * @param array Zarr array to read from + * @param shape array describing the number of elements in each dimension to + * be read + * @param offset array describing the offset in each dimension at which to + * begin reading + * @return tile data as bytes of size shape * bytesPerPixel + * read from offset. */ - public static int bytesPerPixel(DataType dataType) { - switch (dataType) { - case i1: - case u1: - return 1; - case i2: - case u2: - return 2; - case i4: - case u4: - case f4: - return 4; - case f8: - return 8; - default: - throw new IllegalArgumentException( - "Unsupported data type: " + dataType); - } + public static byte[] readAsBytesV3(Array array, int[] shape, int[] offset) + throws ZarrException + { + ucar.ma2.Array tile = array.read(Utils.toLongArray(offset), shape); + ByteBuffer buf = tile.getDataAsByteBuffer(ByteOrder.BIG_ENDIAN); + byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + return bytes; } /** @@ -1829,7 +1885,7 @@ public static byte[] readAsBytes(ZarrArray zArray, int[] shape, int[] offset) throws IOException, InvalidRangeException { DataType dataType = zArray.getDataType(); - int bytesPerPixel = bytesPerPixel(dataType); + int bytesPerPixel = ZarrTypes.bytesPerPixel(dataType); int size = IntStream.of(shape).reduce((a, b) -> a * b).orElse(0); byte[] tileAsBytes = new byte[size * bytesPerPixel]; ByteBuffer tileAsByteBuffer = ByteBuffer.wrap(tileAsBytes); @@ -1874,7 +1930,7 @@ public static byte[] readAsBytes(ZarrArray zArray, int[] shape, int[] offset) /** * Write tile as bytes to typed Zarr array. - * @param zArray Zarr array to write to + * @param pathName path to Zarr array * @param shape array describing the number of elements in each dimension to * be written * @param offset array describing the offset in each dimension at which to @@ -1884,9 +1940,31 @@ public static byte[] readAsBytes(ZarrArray zArray, int[] shape, int[] offset) * @throws IOException * @throws InvalidRangeException */ - private static void writeBytes( + private void writeBytes( + String pathName, int[] shape, int[] offset, ByteBuffer tile) + throws IOException, InvalidRangeException, ZarrException + { + if (getV3()) { + writeBytesV3(Array.open(v3Store.resolve(pathName)), shape, offset, tile); + } + else { + writeBytesV2(ZarrArray.open(getRootPath().resolve(pathName)), + shape, offset, tile); + } + } + + private static void writeBytesV3( + Array array, int[] shape, int[] offset, ByteBuffer tile) + throws ZarrException + { + final ucar.ma2.Array pixels = ucar.ma2.Array.factory( + array.metadata.dataType.getMA2DataType(), shape, tile); + array.write(Utils.toLongArray(offset), pixels); + } + + private static void writeBytesV2( ZarrArray zArray, int[] shape, int[] offset, ByteBuffer tile) - throws IOException, InvalidRangeException + throws IOException, InvalidRangeException { int size = IntStream.of(shape).reduce((a, b) -> a * b).orElse(0); DataType dataType = zArray.getDataType(); @@ -1938,14 +2016,43 @@ private byte[] getTileDownsampled( int series, int resolution, int plane, Region boundingBox, List axes) throws FormatException, IOException, InterruptedException, - EnumerationException, InvalidRangeException + EnumerationException, InvalidRangeException, + ZarrException { final String pathName = String.format(scaleFormatString, getScaleFormatStringArgs(series, resolution - 1)); - final ZarrArray zarr = ZarrArray.open(getRootPath().resolve(pathName)); - int[] dimensions = zarr.getShape(); - int[] blockSizes = zarr.getChunks(); + + // Upscale our base X and Y offsets to the previous resolution + // based on the pyramid scaling factor + int xx = boundingBox.x * PYRAMID_SCALE; + int yy = boundingBox.y * PYRAMID_SCALE; + IFormatReader reader = readers.take(); + int[] offset; + try { + offset = getOffset(reader, xx, yy, plane, axes); + } + finally { + readers.put(reader); + } + + int[] dimensions = null; + int[] blockSizes = null; + + ZarrArray v2Array = null; + Array v3Array = null; + + if (getV3()) { + v3Array = Array.open(v3Store.resolve(pathName)); + dimensions = Utils.toIntArray(v3Array.metadata.shape); + blockSizes = v3Array.metadata.chunkShape(); + } + else { + v2Array = ZarrArray.open(getRootPath().resolve(pathName)); + dimensions = v2Array.getShape(); + blockSizes = v2Array.getChunks(); + } + int xDim = 1; int yDim = 1; int activeTileWidth = 1; @@ -1965,25 +2072,18 @@ private byte[] getTileDownsampled( } } - // Upscale our base X and Y offsets, and sizes to the previous resolution - // based on the pyramid scaling factor - int xx = boundingBox.x * PYRAMID_SCALE; - int yy = boundingBox.y * PYRAMID_SCALE; int width = (int) Math.min(activeTileWidth * PYRAMID_SCALE, xDim - xx); int height = (int) Math.min(activeTileHeight * PYRAMID_SCALE, yDim - yy); - IFormatReader reader = readers.take(); - int[] offset; - try { - offset = getOffset(reader, xx, yy, plane, axes); + int[] shape = createShape(axes, width, height, 1); + byte[] tileAsBytes = null; + if (getV3()) { + tileAsBytes = readAsBytesV3(v3Array, shape, offset); } - finally { - readers.put(reader); + else { + tileAsBytes = readAsBytes(v2Array, shape, offset); } - int bytesPerPixel = FormatTools.getBytesPerPixel(pixelType); - int[] shape = createShape(axes, width, height, 1); - byte[] tileAsBytes = readAsBytes(zarr, shape, offset); if (downsampling == Downsampling.SIMPLE) { return scaler.downsample(tileAsBytes, width, height, @@ -2000,7 +2100,8 @@ private byte[] getTile( int series, int resolution, int plane, Region boundingBox, List axes) throws FormatException, IOException, InterruptedException, - EnumerationException, InvalidRangeException + EnumerationException, InvalidRangeException, + ZarrException { IFormatReader reader = readers.take(); try { @@ -2194,12 +2295,11 @@ private int[] getOffset( private void processChunk(int series, int resolution, int plane, int[] offset, int[] shape, List axes) throws EnumerationException, FormatException, IOException, - InterruptedException, InvalidRangeException + InterruptedException, InvalidRangeException, ZarrException { String pathName = String.format(scaleFormatString, getScaleFormatStringArgs(series, resolution)); - final ZarrArray zarr = ZarrArray.open(getRootPath().resolve(pathName)); IFormatReader reader = readers.take(); boolean littleEndian = reader.isLittleEndian(); int bpp = FormatTools.getBytesPerPixel(reader.getPixelType()); @@ -2270,7 +2370,7 @@ private void processChunk(int series, int resolution, int plane, tileBuffer.order( littleEndian ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN); } - writeBytes(zarr, shape, offset, tileBuffer); + writeBytes(pathName, shape, offset, tileBuffer); getProgressListener().notifyChunkEnd(plane, xOffset, yOffset, zOffset); } @@ -2286,7 +2386,7 @@ private void processChunk(int series, int resolution, int plane, */ public void saveResolutions(int series) throws FormatException, IOException, InterruptedException, - EnumerationException + EnumerationException, ZarrException { int[] resTileCounts = tileCounts.get(series); int resolutions = resTileCounts.length; @@ -2374,17 +2474,31 @@ public void saveResolutions(int series) getDimensions(workingReader, scaledWidth, scaledHeight, scaledDepth, activeTileWidth, activeTileHeight, activeChunkDepth); - DataType dataType = getZarrType(pixelType); String resolutionString = String.format( scaleFormatString, getScaleFormatStringArgs(series, resolution)); - ArrayParams arrayParams = new ArrayParams() - .shape(getShapeArray(activeAxes)) - .chunks(getChunkSizeArray(activeAxes)) - .dataType(dataType) - .dimensionSeparator(getDimensionSeparator()) - .compressor(CompressorFactory.create( - compressionType.toString(), compressionProperties)); - ZarrArray.create(getRootPath().resolve(resolutionString), arrayParams); + + if (getV3()) { + StoreHandle v3Handle = v3Store.resolve(resolutionString); + Array v3Array = Array.create(v3Handle, + Array.metadataBuilder() + .withShape(Utils.toLongArray(getShapeArray(activeAxes))) + .withDataType(ZarrTypes.getV3ZarrType(pixelType)) + .withChunkShape(getChunkSizeArray(activeAxes)) + //.withCodecs(c -> builder) + .build() + ); + } + else { + DataType dataType = ZarrTypes.getZarrType(pixelType); + ArrayParams arrayParams = new ArrayParams() + .shape(getShapeArray(activeAxes)) + .chunks(getChunkSizeArray(activeAxes)) + .dataType(dataType) + .dimensionSeparator(getDimensionSeparator()) + .compressor(CompressorFactory.create( + compressionType.toString(), compressionProperties)); + ZarrArray.create(getRootPath().resolve(resolutionString), arrayParams); + } if (!writeImageData) { continue; @@ -2635,7 +2749,8 @@ private void saveHCSMetadata(IMetadata meta) throws IOException { * @throws InterruptedException */ private void setSeriesLevelMetadata(int series, int resolutions) - throws IOException, InterruptedException, EnumerationException + throws IOException, InterruptedException, EnumerationException, + ZarrException { LOGGER.debug("setSeriesLevelMetadata({}, {})", series, resolutions); String resolutionString = String.format( @@ -2789,9 +2904,6 @@ else if (scale instanceof Time) { } multiscale.put("name", name); - Path subGroupPath = getRootPath().resolve(seriesString); - LOGGER.debug(" creating subgroup {}", subGroupPath); - ZarrGroup subGroup = ZarrGroup.create(subGroupPath); Map attributes = new HashMap(); attributes.put("multiscales", multiscales); @@ -2808,7 +2920,7 @@ else if (scale instanceof Time) { omero.put("rdefs", rdefs); double[] defaultMinMax = - getRange(FormatTools.pixelTypeFromString( + ZarrTypes.getRange(FormatTools.pixelTypeFromString( meta.getPixelsType(seriesIndex).toString())); OMEXMLMetadata omexml = (OMEXMLMetadata) meta; @@ -2882,7 +2994,16 @@ else if (scale instanceof Time) { attributes.put("omero", omero); } - subGroup.writeAttributes(attributes); + if (getV3()) { + Group v3Group = Group.create(v3Store.resolve(seriesString)); + v3Group.setAttributes(attributes); + } + else { + Path subGroupPath = getRootPath().resolve(seriesString); + LOGGER.debug(" creating subgroup {}", subGroupPath); + ZarrGroup subGroup = ZarrGroup.create(subGroupPath); + subGroup.writeAttributes(attributes); + } LOGGER.debug(" finished writing subgroup attributes"); } @@ -2972,70 +3093,6 @@ private IMetadata createMetadata() throws FormatException { } } - /** - * Get the minimum and maximum pixel values for the given pixel type. - * - * @param bfPixelType pixel type as defined in FormatTools - * @return array of length 2 representing the minimum and maximum - * pixel values, or null if converting to the given type is - * not supported - */ - private double[] getRange(int bfPixelType) { - double[] range = new double[2]; - switch (bfPixelType) { - case FormatTools.INT8: - range[0] = -128.0; - range[1] = 127.0; - break; - case FormatTools.UINT8: - range[0] = 0.0; - range[1] = 255.0; - break; - case FormatTools.INT16: - range[0] = -32768.0; - range[1] = 32767.0; - break; - case FormatTools.UINT16: - range[0] = 0.0; - range[1] = 65535.0; - break; - default: - return null; - } - - return range; - } - - /** - * Convert Bio-Formats pixel type to Zarr data type. - * - * @param type Bio-Formats pixel type - * @return corresponding Zarr data type - */ - public static DataType getZarrType(int type) { - switch (type) { - case FormatTools.INT8: - return DataType.i1; - case FormatTools.UINT8: - return DataType.u1; - case FormatTools.INT16: - return DataType.i2; - case FormatTools.UINT16: - return DataType.u2; - case FormatTools.INT32: - return DataType.i4; - case FormatTools.UINT32: - return DataType.u4; - case FormatTools.FLOAT: - return DataType.f4; - case FormatTools.DOUBLE: - return DataType.f8; - default: - throw new IllegalArgumentException("Unsupported pixel type: " - + FormatTools.getPixelTypeString(type)); - } - } - private DimensionSeparator getDimensionSeparator() { return nested ? DimensionSeparator.SLASH : DimensionSeparator.DOT; } diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/ZarrTypes.java b/src/main/java/com/glencoesoftware/bioformats2raw/ZarrTypes.java new file mode 100644 index 0000000..5054b90 --- /dev/null +++ b/src/main/java/com/glencoesoftware/bioformats2raw/ZarrTypes.java @@ -0,0 +1,164 @@ +/** + * Copyright (c) 2025 Glencoe Software, Inc. All rights reserved. + * + * This software is distributed under the terms described by the LICENSE.txt + * file you can find at the root of the distribution bundle. If the file is + * missing please request a copy by contacting info@glencoesoftware.com + */ +package com.glencoesoftware.bioformats2raw; + +import com.bc.zarr.DataType; +import loci.formats.FormatTools; + +public final class ZarrTypes { + + /** + * Convert Bio-Formats pixel type to UCAR array type. + * + * @param type Bio-Formats pixel type + * @return UCAR array type + */ + public static ucar.ma2.DataType getDataType(int type) { + switch (type) { + case FormatTools.INT8: + return ucar.ma2.DataType.BYTE; + case FormatTools.UINT8: + return ucar.ma2.DataType.BYTE; + case FormatTools.INT16: + return ucar.ma2.DataType.SHORT; + case FormatTools.UINT16: + return ucar.ma2.DataType.SHORT; + case FormatTools.INT32: + return ucar.ma2.DataType.INT; + case FormatTools.UINT32: + return ucar.ma2.DataType.INT; + case FormatTools.FLOAT: + return ucar.ma2.DataType.FLOAT; + case FormatTools.DOUBLE: + return ucar.ma2.DataType.DOUBLE; + default: + throw new IllegalArgumentException("Unsupported pixel type: " + + FormatTools.getPixelTypeString(type)); + } + } + + /** + * Convert Bio-Formats pixel type to Zarr v3 data type. + * + * @param type Bio-Formats pixel type + * @return corresponding Zarr v3 data type + */ + public static dev.zarr.zarrjava.v3.DataType getV3ZarrType(int type) { + switch (type) { + case FormatTools.INT8: + return dev.zarr.zarrjava.v3.DataType.INT8; + case FormatTools.UINT8: + return dev.zarr.zarrjava.v3.DataType.UINT8; + case FormatTools.INT16: + return dev.zarr.zarrjava.v3.DataType.INT16; + case FormatTools.UINT16: + return dev.zarr.zarrjava.v3.DataType.UINT16; + case FormatTools.INT32: + return dev.zarr.zarrjava.v3.DataType.INT32; + case FormatTools.UINT32: + return dev.zarr.zarrjava.v3.DataType.UINT32; + case FormatTools.FLOAT: + return dev.zarr.zarrjava.v3.DataType.FLOAT32; + case FormatTools.DOUBLE: + return dev.zarr.zarrjava.v3.DataType.FLOAT64; + default: + throw new IllegalArgumentException("Unsupported pixel type: " + + FormatTools.getPixelTypeString(type)); + } + } + + /** + * Convert Bio-Formats pixel type to Zarr data type. + * + * @param type Bio-Formats pixel type + * @return corresponding Zarr data type + */ + public static DataType getZarrType(int type) { + switch (type) { + case FormatTools.INT8: + return DataType.i1; + case FormatTools.UINT8: + return DataType.u1; + case FormatTools.INT16: + return DataType.i2; + case FormatTools.UINT16: + return DataType.u2; + case FormatTools.INT32: + return DataType.i4; + case FormatTools.UINT32: + return DataType.u4; + case FormatTools.FLOAT: + return DataType.f4; + case FormatTools.DOUBLE: + return DataType.f8; + default: + throw new IllegalArgumentException("Unsupported pixel type: " + + FormatTools.getPixelTypeString(type)); + } + } + + /** + * Return the number of bytes per pixel for a JZarr data type. + * @param dataType type to return number of bytes per pixel for + * @return See above. + */ + public static int bytesPerPixel(DataType dataType) { + switch (dataType) { + case i1: + case u1: + return 1; + case i2: + case u2: + return 2; + case i4: + case u4: + case f4: + return 4; + case f8: + return 8; + default: + throw new IllegalArgumentException( + "Unsupported data type: " + dataType); + } + } + + /** + * Get the minimum and maximum pixel values for the given pixel type. + * + * @param bfPixelType pixel type as defined in FormatTools + * @return array of length 2 representing the minimum and maximum + * pixel values, or null if converting to the given type is + * not supported + */ + public static double[] getRange(int bfPixelType) { + double[] range = new double[2]; + switch (bfPixelType) { + case FormatTools.INT8: + range[0] = -128.0; + range[1] = 127.0; + break; + case FormatTools.UINT8: + range[0] = 0.0; + range[1] = 255.0; + break; + case FormatTools.INT16: + range[0] = -32768.0; + range[1] = 32767.0; + break; + case FormatTools.UINT16: + range[0] = 0.0; + range[1] = 65535.0; + break; + default: + return null; + } + + return range; + } + +} From 9ac541b4358bbadf4345dcc047e9b15d9b72ef69 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 17 Sep 2025 20:34:50 -0500 Subject: [PATCH 2/7] Save HCs metadata when writing v3 --- .../bioformats2raw/Converter.java | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index 15aaae9..4bb830d 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -2585,7 +2585,9 @@ public void saveResolutions(int series) getProgressListener().notifySeriesEnd(series); } - private void saveHCSMetadata(IMetadata meta) throws IOException { + private void saveHCSMetadata(IMetadata meta) + throws IOException, ZarrException + { if (noHCS) { LOGGER.debug("skipping HCS metadata"); return; @@ -2593,8 +2595,6 @@ private void saveHCSMetadata(IMetadata meta) throws IOException { LOGGER.debug("saving HCS metadata"); // assumes only one plate defined - Path rootPath = getRootPath(); - ZarrGroup root = ZarrGroup.open(rootPath); int plate = 0; Map plateMap = new HashMap(); @@ -2659,10 +2659,6 @@ private void saveHCSMetadata(IMetadata meta) throws IOException { List> imageList = new ArrayList>(); - String rowPath = index.getRowPath(); - ZarrGroup rowGroup = root.createSubGroup(rowPath); - String columnPath = index.getColumnPath(); - ZarrGroup columnGroup = rowGroup.createSubGroup(columnPath); for (HCSIndex field : hcsIndexes) { if (field.getPlateIndex() == index.getPlateIndex() && field.getWellRowIndex() == index.getWellRowIndex() && @@ -2680,9 +2676,24 @@ private void saveHCSMetadata(IMetadata meta) throws IOException { Map wellMap = new HashMap(); wellMap.put("images", imageList); - Map attributes = columnGroup.getAttributes(); - attributes.put("well", wellMap); - columnGroup.writeAttributes(attributes); + Map columnAttrs = new HashMap(); + columnAttrs.put("well", wellMap); + + String rowPath = index.getRowPath(); + String columnPath = index.getColumnPath(); + if (getV3()) { + Group rowGroup = Group.create(v3Store.resolve(rowPath)); + Group columnGroup = + Group.create(v3Store.resolve(rowPath, columnPath)); + columnGroup.setAttributes(columnAttrs); + } + else { + Path rootPath = getRootPath(); + ZarrGroup root = ZarrGroup.open(rootPath); + ZarrGroup rowGroup = root.createSubGroup(rowPath); + ZarrGroup columnGroup = rowGroup.createSubGroup(columnPath); + columnGroup.writeAttributes(columnAttrs); + } // make sure the row/column indexes are added to the plate attributes // this is necessary when Plate.Rows or Plate.Columns is not set @@ -2732,9 +2743,19 @@ private void saveHCSMetadata(IMetadata meta) throws IOException { plateMap.put("field_count", maxField + 1); plateMap.put("version", NGFF_VERSION); - Map attributes = root.getAttributes(); - attributes.put("plate", plateMap); - root.writeAttributes(attributes); + if (getV3()) { + Group v3Group = Group.open(v3Store.resolve()); + Map attributes = v3Group.metadata.attributes; + attributes.put("plate", plateMap); + v3Group.setAttributes(attributes); + } + else { + Path rootPath = getRootPath(); + ZarrGroup root = ZarrGroup.open(rootPath); + Map attributes = root.getAttributes(); + attributes.put("plate", plateMap); + root.writeAttributes(attributes); + } } /** From 080734df4c6b04437c12d280af323f7af3e93abd Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 18 Sep 2025 10:16:38 -0500 Subject: [PATCH 3/7] Add basic sharding and codec support for v3 --- .../bioformats2raw/Converter.java | 176 +++++++++++++++++- 1 file changed, 172 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index 4bb830d..2e9f620 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -113,7 +113,7 @@ import dev.zarr.zarrjava.v3.Array; import dev.zarr.zarrjava.v3.Group; //import dev.zarr.zarrjava.v3.Node; -//import dev.zarr.zarrjava.v3.codec.CodecBuilder; +import dev.zarr.zarrjava.v3.codec.CodecBuilder; /** * Command line tool for converting whole slide imaging files to Zarr. @@ -155,6 +155,11 @@ public class Converter implements Callable { private volatile int tileWidth; private volatile int tileHeight; private volatile int chunkDepth; + + private volatile int shardWidth; + private volatile int shardHeight; + private volatile int shardDepth; + private volatile String logLevel; private volatile boolean progressBars = false; private volatile boolean printVersion = false; @@ -399,6 +404,65 @@ public void setChunkDepth(int depth) { } } + /** + * Set the maximum shard width (X shard size) when writing v3. + * + * @param width shard width + */ + @Option( + names = {"--shard-width", "--shard_width"}, + description = "Maximum shard width (default: ${DEFAULT-VALUE}). " + + "Changing this may have performance implications.", + defaultValue = "1024" + ) + public void setShardWidth(int width) { + if (width > 0) { + shardWidth = width; + } + else { + LOGGER.warn("Ignoring invalid shard width: {}", width); + } + } + + /** + * Set the maximum shard height (Y shard size) when writing v3. + * + * @param height shard height + */ + @Option( + names = {"--shard-height", "--shard_height"}, + description = "Maximum shard height (default: ${DEFAULT-VALUE}). " + + "Changing this may have performance implications.", + defaultValue = "1024" + ) + public void setShardHeight(int height) { + if (height > 0) { + shardHeight = height; + } + else { + LOGGER.warn("Ignoring invalid shard height: {}", height); + } + } + + /** + * Set the maximum shard depth (Z shard size) when writing v3. + * + * @param depth Z shard size + */ + @Option( + names = {"--shard-depth", "--shard_depth"}, + description = "Maximum shard depth to read (default: ${DEFAULT-VALUE}) ", + defaultValue = "1" + ) + public void setShardDepth(int depth) { + if (depth > 0) { + shardDepth = depth; + } + else { + LOGGER.warn("Ignoring invalid shard depth: {}", depth); + } + } + /** * Set whether or not tiles should actually be converted. * @@ -1024,6 +1088,27 @@ public int getChunkDepth() { return chunkDepth; } + /** + * @return shard width (X shard size) + */ + public int getShardWidth() { + return shardWidth; + } + + /** + * @return shard height (Y shard size) + */ + public int getShardHeight() { + return shardHeight; + } + + /** + * @return shard depth (Z shard size) + */ + public int getShardDepth() { + return shardDepth; + } + /** * @return true if image data will not be converted */ @@ -2228,6 +2313,27 @@ private int[] getChunkSizeArray(List axes) { return chunk; } + private int[] getShardSizeArray(List axes) { + int[] shard = new int[axes.size()]; + for (int i=0; i axes, int width, int height, int depth) { int[] shape = new int[axes.size()]; Arrays.fill(shape, 1); @@ -2478,13 +2584,43 @@ public void saveResolutions(int series) scaleFormatString, getScaleFormatStringArgs(series, resolution)); if (getV3()) { + CodecBuilder codecBuilder = + new CodecBuilder(ZarrTypes.getV3ZarrType(pixelType)); + + int[] chunkSizes = getChunkSizeArray(activeAxes); + int[] shardSizes = getShardSizeArray(activeAxes); + int[] shape = getShapeArray(activeAxes); + boolean useSharding = false; + + for (int axis=0; axis builder) + .withChunkShape(useSharding ? shardSizes : chunkSizes) + .withCodecs(c -> builder) .build() ); } @@ -3298,6 +3434,38 @@ private int calculateResolutions(int width, int height) { return resolutions; } + /** + * Check that the desired chunk and shard sizes are compatible. + * In each dimension, the chunk size must evenly divide into the shard size. + * + * @param chunkSize expected chunk size + * @param shardSize expected shard size + * @param shape array shape + * @return true if the chunk and shard can be used together + */ + private boolean chunkAndShardCompatible( + int[] chunkSize, int[] shardSize, int[] shape) + { + if (chunkSize.length != shardSize.length || + shape.length != chunkSize.length) + { + return false; + } + for (int d=0; d shape[d]) { + LOGGER.warn("Shard={} must be smaller than shape={} (axis {})", + shardSize[d], shape[d], d); + return false; + } + } + return true; + } + private static Slf4JStopWatch stopWatch() { return new Slf4JStopWatch(LOGGER, Slf4JStopWatch.DEBUG_LEVEL); } From 1faf72cf8bb761c4240ceacc01de26c691627aee Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 18 Sep 2025 20:34:52 -0500 Subject: [PATCH 4/7] Fix specification version numbers --- .../glencoesoftware/bioformats2raw/Converter.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index 2e9f620..faf6067 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -1360,7 +1360,7 @@ public Integer call() throws Exception { ).orElse("development"); System.out.println("Version = " + version); System.out.println("Bio-Formats version = " + FormatTools.VERSION); - System.out.println("NGFF specification version = " + NGFF_VERSION); + System.out.println("NGFF specification version = " + getNGFFVersion()); return -1; } @@ -2877,7 +2877,7 @@ private void saveHCSMetadata(IMetadata meta) plateMap.put("rows", rows); plateMap.put("field_count", maxField + 1); - plateMap.put("version", NGFF_VERSION); + plateMap.put("version", getNGFFVersion()); if (getV3()) { Group v3Group = Group.open(v3Store.resolve()); @@ -2940,7 +2940,7 @@ private void setSeriesLevelMetadata(int series, int resolutions) multiscale.put("type", downsampling.getName()); } multiscale.put("metadata", metadata); - multiscale.put("version", nested ? NGFF_VERSION : "0.1"); + multiscale.put("version", getNGFFVersion()); multiscales.add(multiscale); IFormatReader v = null; @@ -3434,6 +3434,13 @@ private int calculateResolutions(int width, int height) { return resolutions; } + private String getNGFFVersion() { + if (getV3()) { + return NGFF_VERSION_V3; + } + return getNested() ? NGFF_VERSION : "0.1"; + } + /** * Check that the desired chunk and shard sizes are compatible. * In each dimension, the chunk size must evenly divide into the shard size. From 2e363ad3b9ac087150e1e313e4d28877f06856ca Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 18 Sep 2025 20:35:24 -0500 Subject: [PATCH 5/7] Refactor v2 tests, and add a few v3 tests (more are needed) --- .../bioformats2raw/test/AbstractZarrTest.java | 195 ++++++++++++++++++ .../bioformats2raw/test/ZarrTest.java | 180 +--------------- .../bioformats2raw/test/ZarrV3Test.java | 125 +++++++++++ 3 files changed, 330 insertions(+), 170 deletions(-) create mode 100644 src/test/java/com/glencoesoftware/bioformats2raw/test/AbstractZarrTest.java create mode 100644 src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrV3Test.java diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/AbstractZarrTest.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/AbstractZarrTest.java new file mode 100644 index 0000000..0af2f9f --- /dev/null +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/AbstractZarrTest.java @@ -0,0 +1,195 @@ +/** + * Copyright (c) 2025 Glencoe Software, Inc. All rights reserved. + * + * This software is distributed under the terms described by the LICENSE.txt + * file you can find at the root of the distribution bundle. If the file is + * missing please request a copy by contacting info@glencoesoftware.com + */ +package com.glencoesoftware.bioformats2raw.test; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import loci.common.LogbackTools; +import loci.common.services.ServiceFactory; +import loci.formats.services.OMEXMLService; +import ome.xml.model.OME; + +import com.glencoesoftware.bioformats2raw.Converter; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import picocli.CommandLine; + +public abstract class AbstractZarrTest { + Path input; + Path output; + Converter converter; + + /** + * Set logging to warn before all methods. + * + * @param tmp temporary directory for output file + */ + @BeforeEach + public void setup(@TempDir Path tmp) throws Exception { + output = tmp.resolve("test"); + LogbackTools.setRootLevel("warn"); + } + + /** + * Run the Converter main method and check for success or failure. + * + * @param additionalArgs CLI arguments as needed beyond "-o output input" + */ + void assertTool(String...additionalArgs) throws IOException { + List args = new ArrayList(); + for (String arg : additionalArgs) { + args.add(arg); + } + args.add(input.toString()); + args.add(output.toString()); + try { + converter = new Converter(); + CommandLine.call(converter, args.toArray(new String[]{})); + } + catch (RuntimeException rt) { + throw rt; + } + catch (Throwable t) { + throw new RuntimeException(t); + } + } + + static Path fake(String...args) { + assertTrue(args.length %2 == 0); + Map options = new HashMap(); + for (int i = 0; i < args.length; i += 2) { + options.put(args[i], args[i+1]); + } + return fake(options); + } + + static Path fake(Map options) { + return fake(options, null); + } + + /** + * Create a Bio-Formats fake INI file to use for testing. + * @param options map of the options to assign as part of the fake filename + * from the allowed keys + * @param series map of the integer series index and options map (same format + * as options to add to the fake INI content + * @see https://docs.openmicroscopy.org/bio-formats/6.4.0/developers/ + * generating-test-images.html#key-value-pairs + * @return path to the fake INI file that has been created + */ + static Path fake(Map options, + Map> series) + { + return fake(options, series, null); + } + + static Path fake(Map options, + Map> series, + Map originalMetadata) + { + StringBuilder sb = new StringBuilder(); + sb.append("image"); + if (options != null) { + for (Map.Entry kv : options.entrySet()) { + sb.append("&"); + sb.append(kv.getKey()); + sb.append("="); + sb.append(kv.getValue()); + } + } + sb.append("&"); + try { + List lines = new ArrayList(); + if (originalMetadata != null) { + lines.add("[GlobalMetadata]"); + for (String key : originalMetadata.keySet()) { + lines.add(String.format("%s=%s", key, originalMetadata.get(key))); + } + } + if (series != null) { + for (int s : series.keySet()) { + Map seriesOptions = series.get(s); + lines.add(String.format("[series_%d]", s)); + for (String key : seriesOptions.keySet()) { + lines.add(String.format("%s=%s", key, seriesOptions.get(key))); + } + } + } + Path ini = Files.createTempFile(sb.toString(), ".fake.ini"); + File iniAsFile = ini.toFile(); + String iniPath = iniAsFile.getAbsolutePath(); + String fakePath = iniPath.substring(0, iniPath.length() - 4); + Path fake = Paths.get(fakePath); + File fakeAsFile = fake.toFile(); + Files.write(fake, new byte[]{}); + Files.write(ini, lines); + iniAsFile.deleteOnExit(); + fakeAsFile.deleteOnExit(); + return ini; + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + void checkAxes(List> axes, String order, + String[] units) + { + assertEquals(axes.size(), order.length()); + for (int i=0; i multiscale, String name) { + assertEquals(getNGFFVersion(), multiscale.get("version")); + assertEquals(name, multiscale.get("name")); + } + + abstract String getNGFFVersion(); +} diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java index eead274..3888533 100644 --- a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java @@ -10,11 +10,8 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -32,7 +29,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.glencoesoftware.bioformats2raw.Converter; import com.glencoesoftware.bioformats2raw.Downsampling; -import loci.common.LogbackTools; import loci.common.services.ServiceFactory; import loci.formats.FormatTools; import loci.formats.Memoizer; @@ -53,9 +49,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; @@ -63,125 +57,11 @@ import org.junit.jupiter.params.provider.ValueSource; import org.opencv.core.Core; -public class ZarrTest { +public class ZarrTest extends AbstractZarrTest { - Path input; - - Path output; - - Converter converter; - - /** - * Set logging to warn before all methods. - * - * @param tmp temporary directory for output file - */ - @BeforeEach - public void setup(@TempDir Path tmp) throws Exception { - output = tmp.resolve("test"); - LogbackTools.setRootLevel("warn"); - } - - /** - * Run the Converter main method and check for success or failure. - * - * @param additionalArgs CLI arguments as needed beyond "-o output input" - */ - void assertTool(String...additionalArgs) throws IOException { - List args = new ArrayList(); - for (String arg : additionalArgs) { - args.add(arg); - } - args.add(input.toString()); - args.add(output.toString()); - try { - converter = new Converter(); - CommandLine.call(converter, args.toArray(new String[]{})); - } - catch (RuntimeException rt) { - throw rt; - } - catch (Throwable t) { - throw new RuntimeException(t); - } - } - - static Path fake(String...args) { - assertTrue(args.length %2 == 0); - Map options = new HashMap(); - for (int i = 0; i < args.length; i += 2) { - options.put(args[i], args[i+1]); - } - return fake(options); - } - - static Path fake(Map options) { - return fake(options, null); - } - - /** - * Create a Bio-Formats fake INI file to use for testing. - * @param options map of the options to assign as part of the fake filename - * from the allowed keys - * @param series map of the integer series index and options map (same format - * as options to add to the fake INI content - * @see https://docs.openmicroscopy.org/bio-formats/6.4.0/developers/ - * generating-test-images.html#key-value-pairs - * @return path to the fake INI file that has been created - */ - static Path fake(Map options, - Map> series) - { - return fake(options, series, null); - } - - static Path fake(Map options, - Map> series, - Map originalMetadata) - { - StringBuilder sb = new StringBuilder(); - sb.append("image"); - if (options != null) { - for (Map.Entry kv : options.entrySet()) { - sb.append("&"); - sb.append(kv.getKey()); - sb.append("="); - sb.append(kv.getValue()); - } - } - sb.append("&"); - try { - List lines = new ArrayList(); - if (originalMetadata != null) { - lines.add("[GlobalMetadata]"); - for (String key : originalMetadata.keySet()) { - lines.add(String.format("%s=%s", key, originalMetadata.get(key))); - } - } - if (series != null) { - for (int s : series.keySet()) { - Map seriesOptions = series.get(s); - lines.add(String.format("[series_%d]", s)); - for (String key : seriesOptions.keySet()) { - lines.add(String.format("%s=%s", key, seriesOptions.get(key))); - } - } - } - Path ini = Files.createTempFile(sb.toString(), ".fake.ini"); - File iniAsFile = ini.toFile(); - String iniPath = iniAsFile.getAbsolutePath(); - String fakePath = iniPath.substring(0, iniPath.length() - 4); - Path fake = Paths.get(fakePath); - File fakeAsFile = fake.toFile(); - Files.write(fake, new byte[]{}); - Files.write(ini, lines); - iniAsFile.deleteOnExit(); - fakeAsFile.deleteOnExit(); - return ini; - } - catch (IOException e) { - throw new RuntimeException(e); - } + @Override + String getNGFFVersion() { + return "0.4"; } /** @@ -1382,7 +1262,7 @@ public void testHCSMetadata() throws Exception { Map plate = (Map) z.getAttributes().get("plate"); assertEquals(fieldCount, ((Number) plate.get("field_count")).intValue()); - assertEquals("0.4", plate.get("version")); + assertEquals(getNGFFVersion(), plate.get("version")); List> acquisitions = (List>) plate.get("acquisitions"); @@ -1602,7 +1482,7 @@ public void testHCSMetadataNoAcquisitions() throws Exception { Map plate = (Map) z.getAttributes().get("plate"); assertEquals(fieldCount, ((Number) plate.get("field_count")).intValue()); - assertEquals("0.4", plate.get("version")); + assertEquals(getNGFFVersion(), plate.get("version")); List> rows = (List>) plate.get("rows"); @@ -1681,7 +1561,7 @@ public void testSingleWell() throws IOException { Map plate = (Map) z.getAttributes().get("plate"); assertEquals(fieldCount, ((Number) plate.get("field_count")).intValue()); - assertEquals("0.4", plate.get("version")); + assertEquals(getNGFFVersion(), plate.get("version")); List> acquisitions = (List>) plate.get("acquisitions"); @@ -1744,7 +1624,7 @@ public void testTwoWells(String resourceName) throws IOException { Map plate = (Map) z.getAttributes().get("plate"); assertEquals(fieldCount, ((Number) plate.get("field_count")).intValue()); - assertEquals("0.4", plate.get("version")); + assertEquals(getNGFFVersion(), plate.get("version")); List> acquisitions = (List>) plate.get("acquisitions"); @@ -1805,7 +1685,7 @@ public void testOnePlateRow() throws IOException { Map plate = (Map) z.getAttributes().get("plate"); assertEquals(fieldCount, ((Number) plate.get("field_count")).intValue()); - assertEquals("0.4", plate.get("version")); + assertEquals(getNGFFVersion(), plate.get("version")); List> acquisitions = (List>) plate.get("acquisitions"); @@ -1861,7 +1741,7 @@ public void testSingleWellTwoAcquisitions() throws IOException { Map plate = (Map) z.getAttributes().get("plate"); assertEquals(fieldCount, ((Number) plate.get("field_count")).intValue()); - assertEquals("0.4", plate.get("version")); + assertEquals(getNGFFVersion(), plate.get("version")); List> acquisitions = (List>) plate.get("acquisitions"); @@ -2450,41 +2330,6 @@ private void checkWell( } } - private void checkAxes(List> axes, String order, - String[] units) - { - assertEquals(axes.size(), order.length()); - for (int i=0; i> getMultiscales(String group) throws IOException { @@ -2493,9 +2338,4 @@ private List> getMultiscales(String group) "multiscales"); } - private void checkMultiscale(Map multiscale, String name) { - assertEquals("0.4", multiscale.get("version")); - assertEquals(name, multiscale.get("name")); - } - } diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrV3Test.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrV3Test.java new file mode 100644 index 0000000..2a3f8d7 --- /dev/null +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrV3Test.java @@ -0,0 +1,125 @@ +/** + * Copyright (c) 2025 Glencoe Software, Inc. All rights reserved. + * + * This software is distributed under the terms described by the LICENSE.txt + * file you can find at the root of the distribution bundle. If the file is + * missing please request a copy by contacting info@glencoesoftware.com + */ +package com.glencoesoftware.bioformats2raw.test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.Group; + +import loci.formats.in.FakeReader; + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ZarrV3Test extends AbstractZarrTest { + + @Override + String getNGFFVersion() { + return "0.5"; + } + + /** + * Test basic v3 conversion. + */ + @Test + public void testDefault() throws Exception { + input = fake(); + assertTool("--v3"); + + FilesystemStore store = new FilesystemStore(output); + + Array array = Array.open(store.resolve("0", "0")); + assertArrayEquals(new long[] {1, 1, 1, 512, 512}, array.metadata.shape); + + Group rootGroup = Group.open(store.resolve("0")); + Map attrs = rootGroup.metadata.attributes; + List> multiscales = + (List>) attrs.get("multiscales"); + assertEquals(1, multiscales.size()); + Map multiscale = multiscales.get(0); + checkMultiscale(multiscale, "image"); + + List> datasets = + (List>) multiscale.get("datasets"); + assertTrue(datasets.size() > 0); + assertEquals("0", datasets.get(0).get("path")); + + List> axes = + (List>) multiscale.get("axes"); + checkAxes(axes, "TCZYX", null); + + for (int r=0; r dataset = datasets.get(r); + List> transforms = + (List>) dataset.get("coordinateTransformations"); + assertEquals(1, transforms.size()); + Map scale = transforms.get(0); + assertEquals("scale", scale.get("type")); + List axisValues = (List) scale.get("scale"); + + assertEquals(5, axisValues.size()); + double factor = Math.pow(2, r); + // X and Y are the only dimensions that are downsampled, + // so the TCZ physical scales remain the same across all resolutions + assertEquals(axisValues, Arrays.asList(new Double[] { + 1.0, 1.0, 1.0, factor, factor})); + } + } + + /** + * Test special pixels across several series and planes. + */ + @Test + public void testSpecialPixels() throws Exception { + int seriesCount = 3; + int sizeC = 2; + int sizeZ = 4; + int sizeT = 5; + input = fake("series", String.valueOf(seriesCount), + "sizeC", String.valueOf(sizeC), + "sizeZ", String.valueOf(sizeZ), + "sizeT", String.valueOf(sizeT)); + assertTool("--v3"); + + FilesystemStore store = new FilesystemStore(output); + + int[] shape = new int[] {1, 1, 1, 512, 512}; + long[] arrayShape = new long[] {sizeT, sizeC, sizeZ, 512, 512}; + for (int s=0; s Date: Mon, 29 Sep 2025 17:49:11 -0500 Subject: [PATCH 6/7] Fix difference in v2/v3 version and multiscales attributes --- .../glencoesoftware/bioformats2raw/Converter.java | 14 ++++++++++++-- .../bioformats2raw/test/AbstractZarrTest.java | 5 +---- .../bioformats2raw/test/ZarrTest.java | 6 ++++++ .../bioformats2raw/test/ZarrV3Test.java | 10 +++++++++- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index faf6067..45e632a 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -2940,7 +2940,9 @@ private void setSeriesLevelMetadata(int series, int resolutions) multiscale.put("type", downsampling.getName()); } multiscale.put("metadata", metadata); - multiscale.put("version", getNGFFVersion()); + if (!getV3()) { + multiscale.put("version", getNGFFVersion()); + } multiscales.add(multiscale); IFormatReader v = null; @@ -3062,7 +3064,15 @@ else if (scale instanceof Time) { multiscale.put("name", name); Map attributes = new HashMap(); - attributes.put("multiscales", multiscales); + if (getV3()) { + Map omeAttributes = new HashMap(); + omeAttributes.put("version", getNGFFVersion()); + omeAttributes.put("multiscales", multiscales); + attributes.put("ome", omeAttributes); + } + else { + attributes.put("multiscales", multiscales); + } if (omeroMetadata) { Map omero = new HashMap(); diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/AbstractZarrTest.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/AbstractZarrTest.java index 0af2f9f..7e9fe54 100644 --- a/src/test/java/com/glencoesoftware/bioformats2raw/test/AbstractZarrTest.java +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/AbstractZarrTest.java @@ -186,10 +186,7 @@ OME getOMEMetadata() throws Exception { return (OME) xmlService.createOMEXMLRoot(omexml); } - void checkMultiscale(Map multiscale, String name) { - assertEquals(getNGFFVersion(), multiscale.get("version")); - assertEquals(name, multiscale.get("name")); - } + abstract void checkMultiscale(Map multiscale, String name); abstract String getNGFFVersion(); } diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java index 3888533..ac06e6e 100644 --- a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java @@ -64,6 +64,12 @@ String getNGFFVersion() { return "0.4"; } + @Override + void checkMultiscale(Map multiscale, String name) { + assertEquals(getNGFFVersion(), multiscale.get("version")); + assertEquals(name, multiscale.get("name")); + } + /** * Test a fake file with default values smaller than * the default tile size (512 vs 1024). diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrV3Test.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrV3Test.java index 2a3f8d7..02dbb05 100644 --- a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrV3Test.java +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrV3Test.java @@ -31,6 +31,11 @@ String getNGFFVersion() { return "0.5"; } + @Override + void checkMultiscale(Map multiscale, String name) { + assertEquals(name, multiscale.get("name")); + } + /** * Test basic v3 conversion. */ @@ -46,8 +51,11 @@ public void testDefault() throws Exception { Group rootGroup = Group.open(store.resolve("0")); Map attrs = rootGroup.metadata.attributes; + Map omeAttrs = (Map) attrs.get("ome"); + assertEquals("0.5", omeAttrs.get("version")); + List> multiscales = - (List>) attrs.get("multiscales"); + (List>) omeAttrs.get("multiscales"); assertEquals(1, multiscales.size()); Map multiscale = multiscales.get(0); checkMultiscale(multiscale, "image"); From ad1b09832b335623138a138ab4300e7ce527cfab Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 1 Oct 2025 12:09:29 -0500 Subject: [PATCH 7/7] Add a few more v3 tests, including sharding options --- .../bioformats2raw/Converter.java | 15 +- .../bioformats2raw/test/AbstractZarrTest.java | 47 ++++ .../bioformats2raw/test/ZarrTest.java | 37 +-- .../bioformats2raw/test/ZarrV3Test.java | 234 ++++++++++++++++-- 4 files changed, 282 insertions(+), 51 deletions(-) diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index 45e632a..a0c2d21 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -107,13 +107,14 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.FilesystemStore; -//import dev.zarr.zarrjava.store.Store; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.ArrayMetadata; import dev.zarr.zarrjava.v3.Group; -//import dev.zarr.zarrjava.v3.Node; +import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; +import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; /** * Command line tool for converting whole slide imaging files to Zarr. @@ -2131,6 +2132,16 @@ private byte[] getTileDownsampled( v3Array = Array.open(v3Store.resolve(pathName)); dimensions = Utils.toIntArray(v3Array.metadata.shape); blockSizes = v3Array.metadata.chunkShape(); + + Optional shardingCodec = + ArrayMetadata.getShardingIndexedCodec(v3Array.metadata.codecs); + if (shardingCodec.isPresent() && + shardingCodec.get() instanceof ShardingIndexedCodec) + { + ShardingIndexedCodec shardIndex = + (ShardingIndexedCodec) shardingCodec.get(); + blockSizes = shardIndex.configuration.chunkShape; + } } else { v2Array = ZarrArray.open(getRootPath().resolve(pathName)); diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/AbstractZarrTest.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/AbstractZarrTest.java index 7e9fe54..04533f1 100644 --- a/src/test/java/com/glencoesoftware/bioformats2raw/test/AbstractZarrTest.java +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/AbstractZarrTest.java @@ -186,6 +186,53 @@ OME getOMEMetadata() throws Exception { return (OME) xmlService.createOMEXMLRoot(omexml); } + void checkPlateSeriesMetadata(List groupMap, + int rowCount, int colCount, int fieldCount) + { + assertEquals(groupMap.size(), rowCount * colCount * fieldCount); + int index = 0; + for (int r=0; r plate, + int rowCount, int colCount, int fieldCount) + { + assertEquals(fieldCount, ((Number) plate.get("field_count")).intValue()); + assertEquals(getNGFFVersion(), plate.get("version")); + + List> acquisitions = + (List>) plate.get("acquisitions"); + List> rows = + (List>) plate.get("rows"); + List> columns = + (List>) plate.get("columns"); + List> wells = + (List>) plate.get("wells"); + + assertEquals(1, acquisitions.size()); + assertEquals(0, acquisitions.get(0).get("id")); + + assertEquals(rows.size(), rowCount); + assertEquals(columns.size(), colCount); + + assertEquals(rows.size() * columns.size(), wells.size()); + for (int row=0; row multiscale, String name); abstract String getNGFFVersion(); diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java index ac06e6e..fc0ed1b 100644 --- a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java @@ -1247,16 +1247,7 @@ public void testHCSMetadata() throws Exception { ZarrGroup omeGroup = ZarrGroup.open(omePath.toString()); List groupMap = (List) omeGroup.getAttributes().get("series"); - assertEquals(groupMap.size(), 12); - int index = 0; - for (int r=0; r> plateMap = new HashMap>(); plateMap.put("A", Arrays.asList("1", "2", "3")); @@ -1267,35 +1258,13 @@ public void testHCSMetadata() throws Exception { // check plate/well level metadata Map plate = (Map) z.getAttributes().get("plate"); - assertEquals(fieldCount, ((Number) plate.get("field_count")).intValue()); - assertEquals(getNGFFVersion(), plate.get("version")); + checkPlateDimensions(plate, rowCount, colCount, fieldCount); - List> acquisitions = - (List>) plate.get("acquisitions"); + // check well metadata List> rows = (List>) plate.get("rows"); List> columns = (List>) plate.get("columns"); - List> wells = - (List>) plate.get("wells"); - - assertEquals(1, acquisitions.size()); - assertEquals(0, acquisitions.get(0).get("id")); - - assertEquals(rows.size(), rowCount); - assertEquals(columns.size(), colCount); - - assertEquals(rows.size() * columns.size(), wells.size()); - for (int row=0; row row : rows) { String rowName = (String) row.get("name"); for (Map column : columns) { diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrV3Test.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrV3Test.java index 02dbb05..9b947ff 100644 --- a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrV3Test.java +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrV3Test.java @@ -9,19 +9,31 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.file.Files; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; +import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.ArrayMetadata; import dev.zarr.zarrjava.v3.Group; +import dev.zarr.zarrjava.v3.codec.Codec; +import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; import loci.formats.in.FakeReader; +import ome.xml.model.OME; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; public class ZarrV3Test extends AbstractZarrTest { @@ -87,6 +99,110 @@ public void testDefault() throws Exception { } } + /** + * Test HCS v3 conversion. + */ + @Test + public void testHCS() throws Exception { + int rowCount = 2; + int colCount = 3; + int fieldCount = 2; + input = fake( + "plates", "1", "plateAcqs", "1", + "plateRows", String.valueOf(rowCount), + "plateCols", String.valueOf(colCount), + "fields", String.valueOf(fieldCount)); + assertTool("--v3"); + + FilesystemStore store = new FilesystemStore(output); + + Group rootGroup = Group.open(store.resolve("")); + Group omeGroup = Group.open(store.resolve("OME")); + List groupMap = + (List) omeGroup.metadata.attributes.get("series"); + + checkPlateSeriesMetadata(groupMap, rowCount, colCount, fieldCount); + + Map plate = + (Map) rootGroup.metadata.attributes.get("plate"); + checkPlateDimensions(plate, rowCount, colCount, fieldCount); + + long[] arrayShape = new long[] {1, 1, 1, 512, 512}; + for (int r=0; r omero = + (Map) z.metadata.attributes.get("omero"); + + Map rdefs = (Map) omero.get("rdefs"); + assertEquals( + names[i].length == 1 ? "greyscale" : "color", rdefs.get("model")); + + List> channels = + (List>) omero.get("channels"); + assertEquals(names[i].length, channels.size()); + + for (int c=0; c channel = channels.get(c); + assertEquals(names[i][c], channel.get("label")); + assertEquals(colors[i][c], channel.get("color")); + assertEquals(true, channel.get("active")); + } + } + } + + /** + * Convert with the --no-root-group option and make sure + * no root group is present. + */ + @Test + public void testNoRootGroupOption() throws Exception { + input = fake(); + assertTool("--no-root-group", "--v3"); + + assertFalse(Files.exists(output.resolve("zarr.json"))); + } + + /** + * Convert with the --no-ome-meta-export option and make sure + * no OME metadata is present. + */ + @Test + public void testNoOMEOption() throws Exception { + input = fake(); + assertTool("--no-ome-meta-export", "--v3"); + + assertTrue( + !Files.exists(output.resolve("OME").resolve("METADATA.ome.xml"))); + } + /** * Test special pixels across several series and planes. */ @@ -110,24 +226,112 @@ public void testSpecialPixels() throws Exception { Array array = Array.open(store.resolve(String.valueOf(s), "0")); assertArrayEquals(arrayShape, array.metadata.shape); - int plane = 0; - for (int t=0; t getShardSizes() { + return Stream.of( + Arguments.of(512, 512, 1), + Arguments.of(1024, 1024, 2), + Arguments.of(1024, 512, 4), + Arguments.of(2048, 3072, 4) + ); + } + + /** + * Test a few different shard sizes. + * + * @param x shard width + * @param y shard height + * @param z shard depth + */ + @ParameterizedTest + @MethodSource("getShardSizes") + public void testShardSizes(int x, int y, int z) throws Exception { + int sizeX = 2048; + int sizeY = 3072; + int sizeC = 2; + int sizeZ = 4; + int sizeT = 5; + input = fake("sizeX", String.valueOf(sizeX), + "sizeY", String.valueOf(sizeY), + "sizeC", String.valueOf(sizeC), + "sizeZ", String.valueOf(sizeZ), + "sizeT", String.valueOf(sizeT)); + assertTool("--v3", + "--tile-width", "512", + "--tile-height", "512", + "--shard-width", String.valueOf(x), + "--shard-height", String.valueOf(y), + "--shard-depth", String.valueOf(z)); + + FilesystemStore store = new FilesystemStore(output); + + long[] arrayShape = new long[] {sizeT, sizeC, sizeZ, sizeY, sizeX}; + Array array = Array.open(store.resolve("0", "0")); + assertArrayEquals(arrayShape, array.metadata.shape); + + int[] shardShape = new int[] {1, 1, z, y, x}; + int[] chunkShape = new int[] {1, 1, 1, 512, 512}; + assertArrayEquals(shardShape, array.metadata.chunkShape()); + Optional shardingCodec = + ArrayMetadata.getShardingIndexedCodec(array.metadata.codecs); + assertTrue(shardingCodec.isPresent()); + assertTrue(shardingCodec.get() instanceof ShardingIndexedCodec); + ShardingIndexedCodec shardIndex = + (ShardingIndexedCodec) shardingCodec.get(); + assertArrayEquals(chunkShape, shardIndex.configuration.chunkShape); + + int[] shape = new int[] {1, 1, 1, sizeY, sizeX}; + checkSpecialPixels(0, sizeZ, sizeC, sizeT, shape, array); + } + + /** + * Test invalid shard size. + */ + @Test + public void testInvalidShardSizes() throws Exception { + int sizeX = 2048; + int sizeY = 3192; + input = fake("sizeX", String.valueOf(sizeX), + "sizeY", String.valueOf(sizeY)); + assertTool("--v3", + "--tile-width", "512", + "--tile-height", "512", + "--shard-width", String.valueOf(sizeX), + "--shard-height", String.valueOf(sizeY)); + + FilesystemStore store = new FilesystemStore(output); + + long[] arrayShape = new long[] {1, 1, 1, sizeY, sizeX}; + Array array = Array.open(store.resolve("0", "0")); + assertArrayEquals(arrayShape, array.metadata.shape); + Optional shardingCodec = + ArrayMetadata.getShardingIndexedCodec(array.metadata.codecs); + assertFalse(shardingCodec.isPresent()); + } + + void checkSpecialPixels(int s, int sizeZ, int sizeC, int sizeT, + int[] shape, Array array) + throws ZarrException + { + int plane = 0; + for (int t=0; t