diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index d7b55d1..7be611d 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -209,6 +209,7 @@ public class Converter implements Callable { private volatile Path outputPath; private IProgressListener progressListener; + private Map tileCounts = new HashMap(); // Option setters @@ -1337,6 +1338,17 @@ public void convert() root.writeAttributes(attributes); } + // pre-calculate resolution and tile counts + long totalTiles = 0; + for (Integer index : seriesList) { + int[] counts = calculateTileCounts(index); + tileCounts.put(index, counts); + for (int count : counts) { + totalTiles += count; + } + } + getProgressListener().notifyStart(seriesList.size(), totalTiles); + for (Integer index : seriesList) { try { write(index); @@ -1375,6 +1387,105 @@ public void convert() } } + /** + * Pre-calculate the number of resolutions and tiles for the + * given series index. This is useful for accurate progress reporting + * and failing faster for incompatible pixel/downsampling combinations. + * + * @param series series index + * @return array of tile counts; the array length is the number of resolutions + */ + private int[] calculateTileCounts(int series) + throws FormatException, IOException, InterruptedException, + EnumerationException + { + readers.forEach((reader) -> { + reader.setSeries(series); + }); + + IFormatReader workingReader = readers.take(); + int resolutions = 1; + int sizeX; + int sizeY; + int sizeZ; + int imageCount; + try { + // calculate a reasonable pyramid depth if not specified as an argument + sizeX = workingReader.getSizeX(); + sizeY = workingReader.getSizeY(); + if (pyramidResolutions == null) { + if (workingReader.getResolutionCount() > 1 + && reuseExistingResolutions) + { + resolutions = workingReader.getResolutionCount(); + } + else { + resolutions = calculateResolutions(sizeX, sizeY); + } + } + else { + resolutions = pyramidResolutions; + + // check to make sure too many resolutions aren't being used + if ((int) (sizeX / Math.pow(PYRAMID_SCALE, resolutions)) == 0 || + (int) (sizeY / Math.pow(PYRAMID_SCALE, resolutions)) == 0) + { + resolutions = calculateResolutions(sizeX, sizeY); + LOGGER.warn("Too many resolutions specified; reducing to {}", + resolutions); + } + } + LOGGER.info("Using {} pyramid resolutions", resolutions); + sizeZ = workingReader.getSizeZ(); + imageCount = workingReader.getImageCount(); + pixelType = workingReader.getPixelType(); + } + finally { + readers.put(workingReader); + } + + int[] resTileCounts = new int[resolutions]; + + if ((pixelType == FormatTools.INT8 || pixelType == FormatTools.INT32) && + getDownsampling() != Downsampling.SIMPLE && resolutions > 0) + { + String type = FormatTools.getPixelTypeString(pixelType); + throw new UnsupportedOperationException( + "OpenCV does not support downsampling " + type + " data. " + + "See https://github.com/opencv/opencv/issues/7862"); + } + + for (int resCounter=0; resCounter 1 + && reuseExistingResolutions) + { + workingReader.setResolution(resCounter); + scaledWidth = workingReader.getSizeX(); + scaledHeight = workingReader.getSizeY(); + scaledDepth = workingReader.getSizeZ(); + } + } + finally { + readers.put(workingReader); + } + + resTileCounts[resCounter] = + (int) Math.ceil((double) scaledWidth / tileWidth) + * (int) Math.ceil((double) scaledHeight / tileHeight) + * (int) Math.ceil((double) scaledDepth / chunkDepth) + * (imageCount / sizeZ); + } + return resTileCounts; + } + /** * Convert the data specified by the given initialized reader to * an intermediate form. @@ -1835,9 +1946,15 @@ public void saveResolutions(int series) throws FormatException, IOException, InterruptedException, EnumerationException { - getProgressListener().notifySeriesStart(series); + int[] resTileCounts = tileCounts.get(series); + int resolutions = resTileCounts.length; + int seriesTiles = 0; + for (int t : resTileCounts) { + seriesTiles += t; + } + getProgressListener().notifySeriesStart(series, resolutions, seriesTiles); + IFormatReader workingReader = readers.take(); - int resolutions = 1; int sizeX; int sizeY; int sizeZ; @@ -1849,29 +1966,6 @@ public void saveResolutions(int series) // calculate a reasonable pyramid depth if not specified as an argument sizeX = workingReader.getSizeX(); sizeY = workingReader.getSizeY(); - if (pyramidResolutions == null) { - if (workingReader.getResolutionCount() > 1 - && reuseExistingResolutions) - { - resolutions = workingReader.getResolutionCount(); - } - else { - resolutions = calculateResolutions(sizeX, sizeY); - } - } - else { - resolutions = pyramidResolutions; - - // check to make sure too many resolutions aren't being used - if ((int) (sizeX / Math.pow(PYRAMID_SCALE, resolutions)) == 0 || - (int) (sizeY / Math.pow(PYRAMID_SCALE, resolutions)) == 0) - { - resolutions = calculateResolutions(sizeX, sizeY); - LOGGER.warn("Too many resolutions specified; reducing to {}", - resolutions); - } - } - LOGGER.info("Using {} pyramid resolutions", resolutions); sizeZ = workingReader.getSizeZ(); sizeT = workingReader.getSizeT(); sizeC = workingReader.getSizeC(); @@ -1883,15 +1977,6 @@ public void saveResolutions(int series) readers.put(workingReader); } - if ((pixelType == FormatTools.INT8 || pixelType == FormatTools.INT32) && - getDownsampling() != Downsampling.SIMPLE && resolutions > 0) - { - String type = FormatTools.getPixelTypeString(pixelType); - throw new UnsupportedOperationException( - "OpenCV does not support downsampling " + type + " data. " + - "See https://github.com/opencv/opencv/issues/7862"); - } - LOGGER.info( "Preparing to write pyramid sizeX {} (tileWidth: {}) " + "sizeY {} (tileWidth: {}) sizeZ {} (tileDepth: {}) imageCount {}", @@ -1958,10 +2043,7 @@ public void saveResolutions(int series) ZarrArray.create(getRootPath().resolve(resolutionString), arrayParams); nTile = new AtomicInteger(0); - tileCount = (int) Math.ceil((double) scaledWidth / tileWidth) - * (int) Math.ceil((double) scaledHeight / tileHeight) - * (int) Math.ceil((double) scaledDepth / chunkDepth) - * (imageCount / sizeZ); + tileCount = resTileCounts[resolution]; List> futures = new ArrayList>(); diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/IProgressListener.java b/src/main/java/com/glencoesoftware/bioformats2raw/IProgressListener.java index bda2e7e..f69821d 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/IProgressListener.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/IProgressListener.java @@ -11,12 +11,23 @@ public interface IProgressListener extends EventListener { + /** + * Indicates the total number of chunks in this conversion operation. + * Includes all resolutions in all series. + * + * @param seriesCount total number of series + * @param chunkCount total number of chunks + */ + void notifyStart(int seriesCount, long chunkCount); + /** * Indicates the beginning of processing a particular series. * * @param series the series index being processed + * @param resolutionCount total number of resolutions in this series + * @param chunkCount total number of chunks for all resolutions in this series */ - void notifySeriesStart(int series); + void notifySeriesStart(int series, int resolutionCount, int chunkCount); /** * Indicates the end of processing a particular series. @@ -29,9 +40,9 @@ public interface IProgressListener extends EventListener { * Indicates the beginning of processing a particular resolution. * * @param resolution the resolution index being processed - * @param tileCount the total number of tiles in this resolution + * @param chunkCount the total number of chunks in this resolution */ - void notifyResolutionStart(int resolution, int tileCount); + void notifyResolutionStart(int resolution, int chunkCount); /** * Indicates the end of processing a particular resolution. diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/NoOpProgressListener.java b/src/main/java/com/glencoesoftware/bioformats2raw/NoOpProgressListener.java index 914f537..f0de146 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/NoOpProgressListener.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/NoOpProgressListener.java @@ -10,7 +10,13 @@ public class NoOpProgressListener implements IProgressListener { @Override - public void notifySeriesStart(int series) { + public void notifyStart(int seriesCount, long chunkCount) { + } + + @Override + public void notifySeriesStart(int series, int resolutionCount, + int chunkCount) + { } @Override @@ -18,7 +24,7 @@ public void notifySeriesEnd(int series) { } @Override - public void notifyResolutionStart(int resolution, int tileCount) { + public void notifyResolutionStart(int resolution, int chunkCount) { } @Override diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/ProgressBarListener.java b/src/main/java/com/glencoesoftware/bioformats2raw/ProgressBarListener.java index fc61e36..c9dc5fe 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/ProgressBarListener.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/ProgressBarListener.java @@ -32,20 +32,27 @@ public ProgressBarListener(String level) { logLevel = level; } + @Override + public void notifyStart(int seriesCount, long chunkCount) { + // intentional no-op + } @Override - public void notifySeriesStart(int series) { + public void notifySeriesStart(int series, int resolutionCount, + int chunkCount) + { currentSeries = series; } @Override public void notifySeriesEnd(int series) { + // intentional no-op } @Override - public void notifyResolutionStart(int resolution, int tileCount) { + public void notifyResolutionStart(int resolution, int chunkCount) { ProgressBarBuilder builder = new ProgressBarBuilder() - .setInitialMax(tileCount) + .setInitialMax(chunkCount) .setTaskName(String.format("[%d/%d]", currentSeries, resolution)); if (!(logLevel.equals("OFF") || diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/TestProgressListener.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/TestProgressListener.java index 46d8179..41a4b1c 100644 --- a/src/test/java/com/glencoesoftware/bioformats2raw/test/TestProgressListener.java +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/TestProgressListener.java @@ -18,9 +18,17 @@ public class TestProgressListener implements IProgressListener { private int startedTiles = 0; private int completedTiles = 0; private int expectedTileCount = 0; + private int seriesTiles = 0; + private long totalTiles = 0; @Override - public void notifySeriesStart(int series) { + public void notifyStart(int seriesCount, long tileCount) { + totalTiles = tileCount; + } + + @Override + public void notifySeriesStart(int series, int res, int tiles) { + seriesTiles = tiles; } @Override @@ -60,8 +68,22 @@ public void notifyResolutionEnd(int resolution) { * * @return an array with one element per resolution */ - public Integer[] getTileCounts() { + public Integer[] getChunkCounts() { return finishedResolutions.toArray(new Integer[finishedResolutions.size()]); } + /** + * @return the reported number of tiles for the most recent series + */ + public int getSeriesChunkCount() { + return seriesTiles; + } + + /** + * @return the reported number of total tiles for the conversion + */ + public long getTotalChunkCount() { + return totalTiles; + } + } diff --git a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java index 3acdb19..59a125a 100644 --- a/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java +++ b/src/test/java/com/glencoesoftware/bioformats2raw/test/ZarrTest.java @@ -708,9 +708,15 @@ public void testProgressListener() throws Exception { throw new RuntimeException(t); } - Integer[] expectedTileCounts = new Integer[] {320, 80, 20, 5, 5, 5}; - Integer[] tileCounts = listener.getTileCounts(); - assertArrayEquals(expectedTileCounts, tileCounts); + Integer[] expectedChunkCounts = new Integer[] {320, 80, 20, 5, 5, 5}; + Integer[] chunkCounts = listener.getChunkCounts(); + assertArrayEquals(expectedChunkCounts, chunkCounts); + long totalChunkCount = 0; + for (Integer t : expectedChunkCounts) { + totalChunkCount += t; + } + assertEquals(totalChunkCount, listener.getTotalChunkCount()); + assertEquals(totalChunkCount, listener.getSeriesChunkCount()); } private int bytesPerPixel(DataType dataType) {