sis-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From desruisse...@apache.org
Subject [sis] branch geoapi-4.0 updated: Read all tiles that are inside the requested domain at `GridCoverageResource.read(…)` invocation time by default, but provide a `RasterLoadingStrategy` enumeration for allowing developers to keep the previous mode if desired. This enumeration is not in public API for now; we will see how it works in practice first.
Date Fri, 30 Jul 2021 16:29:08 GMT
This is an automated email from the ASF dual-hosted git repository.

desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git


The following commit(s) were added to refs/heads/geoapi-4.0 by this push:
     new e16859b  Read all tiles that are inside the requested domain at `GridCoverageResource.read(…)`
invocation time by default, but provide a `RasterLoadingStrategy` enumeration for allowing
developers to keep the previous mode if desired. This enumeration is not in public API for
now; we will see how it works in practice first.
e16859b is described below

commit e16859b2c048227c88a16a045f63aba23d663d0d
Author: Martin Desruisseaux <martin.desruisseaux@geomatys.com>
AuthorDate: Fri Jul 30 18:26:49 2021 +0200

    Read all tiles that are inside the requested domain at `GridCoverageResource.read(…)`
invocation time by default,
    but provide a `RasterLoadingStrategy` enumeration for allowing developers to keep the
previous mode if desired.
    This enumeration is not in public API for now; we will see how it works in practice first.
---
 .../apache/sis/coverage/grid/GridCoverage2D.java   |   1 -
 .../org/apache/sis/storage/geotiff/DataCube.java   |  14 ++-
 .../org/apache/sis/storage/geotiff/DataSubset.java |   2 +-
 .../sis/storage/geotiff/ImageFileDirectory.java    |   4 +-
 .../sis/internal/storage/AbstractGridResource.java |  28 +++++
 .../internal/storage/RasterLoadingStrategy.java    | 112 +++++++++++++++++++
 .../sis/internal/storage/TiledGridResource.java    | 121 ++++++++++++++++++++-
 .../sis/test/storage/CoverageReadConsistency.java  |  13 +++
 8 files changed, 287 insertions(+), 8 deletions(-)

diff --git a/core/sis-feature/src/main/java/org/apache/sis/coverage/grid/GridCoverage2D.java
b/core/sis-feature/src/main/java/org/apache/sis/coverage/grid/GridCoverage2D.java
index e1bc00f..837a475 100644
--- a/core/sis-feature/src/main/java/org/apache/sis/coverage/grid/GridCoverage2D.java
+++ b/core/sis-feature/src/main/java/org/apache/sis/coverage/grid/GridCoverage2D.java
@@ -608,7 +608,6 @@ public class GridCoverage2D extends GridCoverage {
              * may force data loading earlier than desired.
              */
             final ReshapedImage result = new ReshapedImage(data, xmin, ymin, xmax, ymax);
-            String error; assert (error = result.verify()) == null : error;
             return result.isIdentity() ? data : result;
         } catch (ArithmeticException e) {
             throw new CannotEvaluateException(e.getMessage(), e);
diff --git a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataCube.java
b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataCube.java
index 23f2e66..1ecb856 100644
--- a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataCube.java
+++ b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataCube.java
@@ -19,6 +19,7 @@ package org.apache.sis.storage.geotiff;
 import java.nio.file.Path;
 import java.awt.image.SampleModel;
 import java.awt.image.BandedSampleModel;
+import org.apache.sis.storage.DataStore;
 import org.apache.sis.storage.DataStoreException;
 import org.apache.sis.storage.DataStoreContentException;
 import org.apache.sis.coverage.grid.GridCoverage;
@@ -59,6 +60,14 @@ abstract class DataCube extends TiledGridResource implements ResourceOnFileSyste
     }
 
     /**
+     * Returns the object on which to perform all synchronizations for thread-safety.
+     */
+    @Override
+    protected final DataStore getSynchronizationLock() {
+        return reader.store;
+    }
+
+    /**
      * Shortcut for a frequently requested information.
      */
     final String filename() {
@@ -122,9 +131,9 @@ abstract class DataCube extends TiledGridResource implements ResourceOnFileSyste
     @Override
     public final GridCoverage read(final GridGeometry domain, final int... range) throws
DataStoreException {
         final long startTime = System.nanoTime();
-        final GridCoverage coverage;
+        GridCoverage coverage;
         try {
-            synchronized (reader.store) {
+            synchronized (getSynchronizationLock()) {
                 final Subset subset = new Subset(domain, range, false);
                 final Compression compression = getCompression();
                 if (compression == null) {
@@ -145,6 +154,7 @@ abstract class DataCube extends TiledGridResource implements ResourceOnFileSyste
                                 Resources.Keys.UnsupportedCompressionMethod_1, compression));
                     }
                 }
+                coverage = preload(coverage);
             }
         } catch (RuntimeException e) {
             throw new DataStoreException(reader.errors().getString(Errors.Keys.CanNotRead_1,
filename()), e);
diff --git a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataSubset.java
b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataSubset.java
index 313b262..d0a7574 100644
--- a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataSubset.java
+++ b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/DataSubset.java
@@ -283,7 +283,7 @@ class DataSubset extends TiledGridCoverage implements Localized {
         final Tile[] missings = new Tile[iterator.tileCountInQuery];
         int numMissings = 0;
         boolean needsCompaction = false;
-        synchronized (reader().store) {
+        synchronized (source.getSynchronizationLock()) {
             do {
                 final WritableRaster tile = iterator.getCachedTile();
                 if (tile != null) {
diff --git a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/ImageFileDirectory.java
b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/ImageFileDirectory.java
index 778c3f5..a14997b 100644
--- a/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/ImageFileDirectory.java
+++ b/storage/sis-geotiff/src/main/java/org/apache/sis/storage/geotiff/ImageFileDirectory.java
@@ -1297,7 +1297,7 @@ final class ImageFileDirectory extends DataCube {
      */
     @Override
     public GridGeometry getGridGeometry() throws DataStoreContentException {
-        synchronized (reader.store) {
+        synchronized (getSynchronizationLock()) {
             if (referencing != null) {
                 GridGeometry gridGeometry = referencing.gridGeometry;
                 if (gridGeometry == null) try {
@@ -1321,7 +1321,7 @@ final class ImageFileDirectory extends DataCube {
     @Override
     @SuppressWarnings("ReturnOfCollectionOrArrayField")
     public List<SampleDimension> getSampleDimensions() throws DataStoreContentException
{
-        synchronized (reader.store) {
+        synchronized (getSynchronizationLock()) {
             if (sampleDimensions == null) {
                 final SampleDimension[] dimensions = new SampleDimension[samplesPerPixel];
                 final SampleDimension.Builder builder = new SampleDimension.Builder();
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/AbstractGridResource.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/AbstractGridResource.java
index d502e8c..fe5b1ce 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/AbstractGridResource.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/AbstractGridResource.java
@@ -482,6 +482,34 @@ public abstract class AbstractGridResource extends AbstractResource implements
G
     }
 
     /**
+     * Returns an indication about when the "physical" loading of raster data will happen.
+     * This is the strategy actually applied by this resource implementation, not necessarily
+     * the strategy given in the last call to {@link #setLoadingStrategy(RasterLoadingStrategy)
+     * setLoadingStrategy(…)}.
+     *
+     * <p>The default strategy is to load raster data at {@link #read read(…)} method
invocation time.</p>
+     *
+     * @return current raster data loading strategy for this resource.
+     */
+    public RasterLoadingStrategy getLoadingStrategy() {
+        return RasterLoadingStrategy.AT_READ_TIME;
+    }
+
+    /**
+     * Sets the preferred strategy about when to do the "physical" loading of raster data.
+     * Implementations are free to ignore this parameter or to replace the given strategy
+     * by the closest alternative that this resource can support.
+     *
+     * @param  strategy  the desired strategy for loading raster data.
+     * @return {@code true} if the given strategy has been accepted, or {@code false}
+     *         if this implementation replaced the given strategy by an alternative.
+     */
+    public boolean setLoadingStrategy(final RasterLoadingStrategy strategy) {
+        ArgumentChecks.ensureNonNull("strategy", strategy);
+        return strategy == getLoadingStrategy();
+    }
+
+    /**
      * Logs the execution of a {@link #read(GridGeometry, int...)} operation.
      * The log level will be {@link Level#FINE} if the operation was quick enough,
      * or {@link PerformanceLevel#SLOW} or higher level otherwise.
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/RasterLoadingStrategy.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/RasterLoadingStrategy.java
new file mode 100644
index 0000000..7d674c2
--- /dev/null
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/RasterLoadingStrategy.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sis.internal.storage;
+
+import java.lang.ref.SoftReference;
+import java.awt.image.RenderedImage;
+import java.awt.image.ImagingOpException;
+import org.opengis.coverage.CannotEvaluateException;
+import org.apache.sis.coverage.grid.GridExtent;
+import org.apache.sis.coverage.grid.GridGeometry;
+import org.apache.sis.coverage.grid.GridCoverage;
+import org.apache.sis.storage.GridCoverageResource;
+import org.apache.sis.storage.DataStoreException;
+
+
+/**
+ * Time when the "physical" loading of raster data should happen. Some resource implementations
may
+ * not load data immediately when the {@linkplain GridCoverageResource#read read method}
is invoked,
+ * but instead defer the actual loading until the {@linkplain GridCoverage#render image is
rendered}.
+ * This enumeration gives some control over the time when data loading happens.
+ * The different strategies are compromises between memory consumption,
+ * redundant loading of same data and early error detection.
+ *
+ * <p>Enumeration values are ordered from the most eager strategy to the laziest strategy.
+ * The eager strategy is fastest when all pixels are used, at the cost of largest memory
consumption.
+ * The lazy strategy is more efficient when only a few tiles will be used and those tiles
are not known in advance.
+ * The lazy strategy is also the only applicable one if the image is too large for holding
in memory.</p>
+ *
+ * @todo This enumeration is internal for now, but may move to public API in a future version.
+ *
+ * @author  Martin Desruisseaux (Geomatys)
+ * @version 1.1
+ * @since   1.1
+ * @module
+ */
+public enum RasterLoadingStrategy {
+    /**
+     * Raster data are loaded at {@link GridCoverageResource#read(GridGeometry, int...)}
invocation time.
+     * This is the most eager loading strategy.
+     *
+     * <p><b>Advantages:</b></p>
+     * <ul>
+     *   <li>Fastest loading strategy if all pixels in the returned {@link GridCoverage}
will be used.</li>
+     *   <li>No redundant data loading: all data stay in memory until the resource
is garbage-collected.</li>
+     *   <li>Immediate error notification with a checked {@link DataStoreException}.</li>
+     * </ul>
+     *
+     * <p><b>Disadvantages:</b></p>
+     * <ul>
+     *   <li>Slower than other strategies if only a subset of the returned {@link GridCoverage}
will be used.</li>
+     *   <li>Consume memory for the full {@link GridCoverage} as long as the resource
is referenced.</li>
+     * </ul>
+     */
+    AT_READ_TIME,
+
+    /**
+     * Raster data are loaded at {@link GridCoverage#render(GridExtent)} invocation time.
+     * Speed and memory usage are at an intermediate level between {@link #AT_READ_TIME}
and {@link #AT_GET_TILE_TIME}.
+     *
+     * <p><b>Advantages:</b></p>
+     * <ul>
+     *   <li>Faster than {@link #AT_GET_TILE_TIME} if all pixels in the returned {@link
RenderedImage} will be used.</li>
+     *   <li>Load less data than {@link #AT_READ_TIME} if the {@link GridExtent} is
a sub-region of the coverage.</li>
+     *   <li>Memory is released sooner (when the {@link RenderedImage} is garbage-collected)
than above mode.</li>
+     * </ul>
+     *
+     * <p><b>Disadvantages:</b></p>
+     * <ul>
+     *   <li>Slower than {@link #AT_GET_TILE_TIME} if only a few tiles will be used.</li>
+     *   <li>Consume memory for the full {@link RenderedImage} as long as the image
is referenced.</li>
+     *   <li>May reload the same data many times if images are discarded and recreated.</li>
+     *   <li>Unchecked {@link CannotEvaluateException} can happen relatively late
+     *       (at {@code render(…)} invocation time) in a chain of coverage operations.</li>
+     * </ul>
+     */
+    AT_RENDER_TIME,
+
+    /**
+     * Raster data are loaded at {@link RenderedImage#getTile(int, int)} invocation time.
+     * This is the laziest loading strategy.
+     * This is also the only strategy that can handle very large {@link RenderedImage}s.
+     *
+     * <p><b>Advantages:</b></p>
+     * <ul>
+     *   <li>Only the tiles that are actually used are loaded.</li>
+     *   <li>Memory can be released at any time (tiles are kept by {@linkplain SoftReference
soft references}).</li>
+     * </ul>
+     *
+     * <p><b>Disadvantages:</b></p>
+     * <ul>
+     *   <li>Slower read operations (numerous seeks when tiles are read in random order).</li>
+     *   <li>May reload the same data many times if tiles are discarded and recreated.</li>
+     *   <li>Unchecked {@link ImagingOpException} can happen late
+     *       (at {@code getTile(…)} invocation time) in a chain of image operations.</li>
+     * </ul>
+     */
+    AT_GET_TILE_TIME
+}
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridResource.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridResource.java
index e21c73e..212c1b4 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridResource.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/TiledGridResource.java
@@ -20,13 +20,18 @@ import java.util.List;
 import java.util.Arrays;
 import java.awt.image.ColorModel;
 import java.awt.image.SampleModel;
+import java.awt.image.RenderedImage;
 import java.awt.image.WritableRaster;
 import java.awt.image.RasterFormatException;
+import org.opengis.coverage.CannotEvaluateException;
 import org.apache.sis.coverage.SampleDimension;
+import org.apache.sis.coverage.grid.GridCoverage;
+import org.apache.sis.coverage.grid.GridCoverage2D;
 import org.apache.sis.coverage.grid.GridDerivation;
 import org.apache.sis.coverage.grid.GridExtent;
 import org.apache.sis.coverage.grid.GridGeometry;
 import org.apache.sis.coverage.grid.GridRoundingMode;
+import org.apache.sis.storage.DataStore;
 import org.apache.sis.storage.DataStoreException;
 import org.apache.sis.storage.event.StoreListeners;
 import org.apache.sis.util.collection.WeakValueHashMap;
@@ -96,6 +101,15 @@ public abstract class TiledGridResource extends AbstractGridResource {
     private final WeakValueHashMap<CacheKey, WritableRaster> rasters;
 
     /**
+     * Whether all tiles should be loaded at {@code read(…)} method call or deferred to
a later time.
+     * This field is initially {@code null} and is initialized to its default value only
when needed.
+     *
+     * @see #getLoadingStrategy()
+     * @see #setLoadingStrategy(RasterLoadingStrategy)
+     */
+    private RasterLoadingStrategy loadingStrategy;
+
+    /**
      * Creates a new resource.
      *
      * @param  parent  listeners of the parent resource, or {@code null} if none.
@@ -106,13 +120,21 @@ public abstract class TiledGridResource extends AbstractGridResource
{
     }
 
     /**
+     * Returns the object on which to perform all synchronizations for thread-safety.
+     * In current implementation, this is the {@link DataStore} that created this resource.
+     * However this restriction may be relaxed in any future version.
+     *
+     * @return the synchronization lock.
+     */
+    protected abstract DataStore getSynchronizationLock();
+
+    /**
      * Returns the size of tiles in this resource.
      * The length of the returned array is the number of dimensions.
      *
      * @return the size of tiles in this resource.
-     * @throws DataStoreException if an error occurred while fetching tile size information.
      */
-    protected abstract int[] getTileSize() throws DataStoreException;
+    protected abstract int[] getTileSize();
 
     /**
      * Returns the Java2D sample model describing pixel type and layout for all bands.
@@ -329,4 +351,99 @@ public abstract class TiledGridResource extends AbstractGridResource
{
             return selectedBands != null;
         }
     }
+
+    /**
+     * If the loading strategy is to load all tiles at {@code read(…)} time, replaces the
given coverage
+     * by a coverage will all data in memory. This method should be invoked by subclasses
at the end of
+     * their {@link #read(GridGeometry, int...)} method implementation.
+     *
+     * @param  coverage  the {@link TiledGridCoverage} to potentially replace by a coverage
with preloaded data.
+     * @return a coverage with preloaded data, or the given coverage if preloading is not
enabled.
+     * @throws DataStoreException if an error occurred while preloading data.
+     */
+    protected final GridCoverage preload(final GridCoverage coverage) throws DataStoreException
{
+        assert Thread.holdsLock(getSynchronizationLock());
+        // Note: `loadingStrategy` may still be null if unitialized.
+        if (loadingStrategy != RasterLoadingStrategy.AT_RENDER_TIME) {
+            /*
+             * In theory the following condition is redundant with `supportImmediateLoading()`.
+             * We apply it anyway in case the coverage geometry is not what was announced.
+             * This condition is also necessary if `loadingStrategy` has not been initialized.
+             */
+            if (coverage.getGridGeometry().getDimension() == TiledGridCoverage.BIDIMENSIONAL)
try {
+                final RenderedImage image = coverage.render(null);
+                return new GridCoverage2D(coverage.getGridGeometry(), coverage.getSampleDimensions(),
image);
+            } catch (RuntimeException e) {
+                /*
+                 * The `coverage.render(…)` implementation may have wrapped the checked
`DataStoreException`
+                 * because of API restriction. In that case we can unwrap the exception here
since this API
+                 * allows it. This is one of the reasons for preferring the `AT_READ_TIME`
loading mode.
+                 */
+                Throwable cause = e.getCause();
+                if (cause instanceof DataStoreException) {
+                    throw (DataStoreException) cause;
+                }
+                /*
+                 * The `CannotEvaluateException` wrapper is created by `TiledGridCoverage.render(…)`,
+                 * in which case we avoid that level of indirection for making stack trace
simpler.
+                 * But if the exception is another kind, keep it.
+                 */
+                if (cause == null || !(e instanceof CannotEvaluateException)) {
+                    cause = e;
+                }
+                throw new DataStoreException(e.getLocalizedMessage(), cause);
+            }
+        }
+        return coverage;
+    }
+
+    /**
+     * Whether this resource supports immediate loading of raster data.
+     */
+    private boolean supportImmediateLoading() {
+        return getTileSize().length == TiledGridCoverage.BIDIMENSIONAL;
+    }
+
+    /**
+     * Returns an indication about when the "physical" loading of raster data will happen.
+     *
+     * @return current raster data loading strategy for this resource.
+     */
+    @Override
+    public final RasterLoadingStrategy getLoadingStrategy() {
+        synchronized (getSynchronizationLock()) {
+            if (loadingStrategy == null) {
+                setLoadingStrategy(supportImmediateLoading());
+            }
+            return loadingStrategy;
+        }
+    }
+
+    /**
+     * Sets the preferred strategy about when to do the "physical" loading of raster data.
+     *
+     * @param  strategy  the desired strategy for loading raster data.
+     * @return {@code true} if the given strategy has been accepted, or {@code false}
+     *         if this implementation replaced the given strategy by an alternative.
+     */
+    @Override
+    public final boolean setLoadingStrategy(final RasterLoadingStrategy strategy) {
+        synchronized (getSynchronizationLock()) {
+            if (strategy != null) {
+                setLoadingStrategy(strategy == RasterLoadingStrategy.AT_READ_TIME &&
supportImmediateLoading());
+            }
+            return super.setLoadingStrategy(strategy);
+        }
+    }
+
+    /**
+     * Sets the strategy for the given flag.
+     *
+     * @param  loadAtReadTime  whether all tiles should be read immediately
+     *         at {@code read(…)} method call or deferred at a later time.
+     */
+    private void setLoadingStrategy(final boolean loadAtReadTime) {
+        loadingStrategy = loadAtReadTime ? RasterLoadingStrategy.AT_READ_TIME
+                                         : RasterLoadingStrategy.AT_RENDER_TIME;
+    }
 }
diff --git a/storage/sis-storage/src/test/java/org/apache/sis/test/storage/CoverageReadConsistency.java
b/storage/sis-storage/src/test/java/org/apache/sis/test/storage/CoverageReadConsistency.java
index b9665e2..105b80e 100644
--- a/storage/sis-storage/src/test/java/org/apache/sis/test/storage/CoverageReadConsistency.java
+++ b/storage/sis-storage/src/test/java/org/apache/sis/test/storage/CoverageReadConsistency.java
@@ -27,6 +27,8 @@ import org.apache.sis.coverage.grid.GridGeometry;
 import org.apache.sis.coverage.grid.GridExtent;
 import org.apache.sis.storage.DataStoreException;
 import org.apache.sis.storage.GridCoverageResource;
+import org.apache.sis.internal.storage.AbstractGridResource;
+import org.apache.sis.internal.storage.RasterLoadingStrategy;
 import org.apache.sis.internal.util.StandardDateFormat;
 import org.apache.sis.internal.util.Numerics;
 import org.apache.sis.image.PixelIterator;
@@ -236,6 +238,16 @@ public strictfp class CoverageReadConsistency extends TestCase {
     }
 
     /**
+     * Applies a random configuration on the resource.
+     */
+    private void randomConfigureResource() {
+        if (resource instanceof AbstractGridResource) {
+            final RasterLoadingStrategy[] choices = RasterLoadingStrategy.values();
+            ((AbstractGridResource) resource).setLoadingStrategy(choices[random.nextInt(choices.length)]);
+        }
+    }
+
+    /**
      * Creates a random domain to be used as a query on the {@link #resource} to test.
      * All arrays given to this method will have their values overwritten.
      *
@@ -291,6 +303,7 @@ public strictfp class CoverageReadConsistency extends TestCase {
      * @throws DataStoreException if an error occurred while using the resource.
      */
     private void readAndCompareRandomRegions() throws DataStoreException {
+        randomConfigureResource();
         final GridGeometry gg = resource.getGridGeometry();
         final int    dimension   = gg.getDimension();
         final long[] low         = new long[dimension];

Mime
View raw message