sis-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From desruisse...@apache.org
Subject [sis] branch geoapi-4.0 updated: More development (but still incomplete) for the support of a netCDF dimension as bands. https://issues.apache.org/jira/browse/SIS-449
Date Thu, 21 Mar 2019 02:46:48 GMT
This is an automated email from the ASF dual-hosted git repository.

desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git


The following commit(s) were added to refs/heads/geoapi-4.0 by this push:
     new c9f4e82  More development (but still incomplete) for the support of a netCDF dimension
as bands. https://issues.apache.org/jira/browse/SIS-449
c9f4e82 is described below

commit c9f4e825aa0edcef7edd0c745af116c6307e2945
Author: Martin Desruisseaux <martin.desruisseaux@geomatys.com>
AuthorDate: Thu Mar 21 03:38:18 2019 +0100

    More development (but still incomplete) for the support of a netCDF dimension as bands.
    https://issues.apache.org/jira/browse/SIS-449
---
 .../apache/sis/internal/raster/RasterFactory.java  |   2 +-
 .../apache/sis/internal/netcdf/RasterResource.java | 117 +++++++++------
 .../org/apache/sis/internal/netcdf/Variable.java   |  14 ++
 .../sis/internal/netcdf/impl/VariableInfo.java     |   8 +
 .../sis/internal/netcdf/ucar/VariableWrapper.java  |   8 +
 .../sis/internal/storage/AbstractGridResource.java | 163 +++++++++++++++++----
 6 files changed, 238 insertions(+), 74 deletions(-)

diff --git a/core/sis-raster/src/main/java/org/apache/sis/internal/raster/RasterFactory.java
b/core/sis-raster/src/main/java/org/apache/sis/internal/raster/RasterFactory.java
index a276788..a16822b 100644
--- a/core/sis-raster/src/main/java/org/apache/sis/internal/raster/RasterFactory.java
+++ b/core/sis-raster/src/main/java/org/apache/sis/internal/raster/RasterFactory.java
@@ -168,7 +168,7 @@ public final class RasterFactory extends Static {
      * Wraps the backing arrays of given NIO buffers into Java2D buffers.
      * This method wraps the underlying array of primitive types; data are not copied.
      * For each buffer, the data starts at {@linkplain Buffer#position() buffer position}
-     * and ends at the position + {@linkplain Buffer#remaining() remaining}.
+     * and ends at {@linkplain Buffer#limit() limit}.
      *
      * @param  dataType  type of buffer to create as one of {@link DataBuffer} constants.
      * @param  data      the data, one for each band.
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/RasterResource.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/RasterResource.java
index dfeadf5..b176279 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/RasterResource.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/RasterResource.java
@@ -28,8 +28,10 @@ import org.opengis.referencing.operation.MathTransform1D;
 import org.opengis.referencing.operation.TransformException;
 import org.apache.sis.internal.storage.AbstractGridResource;
 import org.apache.sis.internal.storage.ResourceOnFileSystem;
+import org.apache.sis.internal.util.UnmodifiableArrayList;
 import org.apache.sis.internal.raster.RasterFactory;
 import org.apache.sis.coverage.SampleDimension;
+import org.apache.sis.coverage.grid.GridExtent;
 import org.apache.sis.coverage.grid.GridCoverage;
 import org.apache.sis.coverage.grid.GridGeometry;
 import org.apache.sis.coverage.grid.GridDerivation;
@@ -41,9 +43,9 @@ import org.apache.sis.math.MathFunctions;
 import org.apache.sis.measure.MeasurementRange;
 import org.apache.sis.measure.NumberRange;
 import org.apache.sis.util.Numbers;
+import org.apache.sis.util.ArraysExt;
 import org.apache.sis.util.resources.Errors;
 import org.apache.sis.util.resources.Vocabulary;
-import org.apache.sis.internal.util.UnmodifiableArrayList;
 
 
 /**
@@ -108,8 +110,8 @@ public final class RasterResource extends AbstractGridResource implements
Resour
     private final SampleDimension[] ranges;
 
     /**
-     * The netCDF variable wrapped by this resource.
-     * The same variable may be repeated if one of its dimension shall be interpreted as
bands.
+     * The netCDF variable wrapped by this resource. The length of this array shall be equal
to {@code ranges.length},
+     * except if a variable dimension represents bands. In the later case, this array should
contain only one element.
      */
     private final Variable[] data;
 
@@ -143,7 +145,22 @@ public final class RasterResource extends AbstractGridResource implements
Resour
         identifier   = decoder.nameFactory.createLocalName(decoder.namespace, name);
         location     = decoder.location;
         gridGeometry = grid;
-        bandDimension = -1;
+        switch (data[0].getDimension() - grid.getDimension()) {
+            case 0: {
+                // All dimensions are in the CRS. This is the usual case.
+                bandDimension = -1;
+                break;
+            }
+            case 1: {
+                // One dimension is interpreted as bands.
+                bandDimension = data[0].bandDimension;
+                break;
+            }
+            default: {
+                // Too many missing dimensions.
+                throw new DataStoreException();
+            }
+        }
     }
 
     /**
@@ -379,60 +396,78 @@ public final class RasterResource extends AbstractGridResource implements
Resour
      * @throws DataStoreException if an error occurred while reading the grid coverage data.
      */
     @Override
-    public GridCoverage read(GridGeometry domain, int... range) throws DataStoreException
{
-        range = validateRangeArgument(ranges.length, range);
+    public GridCoverage read(GridGeometry domain, final int... range) throws DataStoreException
{
+        final RangeArgument rangeIndices = validateRangeArgument(ranges.length, range);
         if (domain == null) {
             domain = gridGeometry;
         }
-        final Variable first = data[range[0]];
+        final Variable first = data[bandDimension >= 0 ? 0 : rangeIndices.first()];  
  // Only one variable if bandDimension ≧ 0.
         final DataType dataType = first.getDataType();
-        for (int i=1; i<data.length; i++) {
-            final Variable variable = data[range[i]];
-            if (!dataType.equals(variable.getDataType())) {
-                throw new DataStoreContentException(Resources.forLocale(getLocale()).getString(
-                        Resources.Keys.MismatchedVariableType_3, getFilename(), first.getName(),
variable.getName()));
+        if (bandDimension < 0) {
+            for (int i=0; i<rangeIndices.getNumBands(); i++) {
+                final Variable variable = data[rangeIndices.getSourceIndex(i)];
+                if (!dataType.equals(variable.getDataType())) {
+                    throw new DataStoreContentException(Resources.forLocale(getLocale()).getString(
+                            Resources.Keys.MismatchedVariableType_3, getFilename(), first.getName(),
variable.getName()));
+                }
             }
         }
+        /*
+         * At this point the arguments and the state of this resource have been validated.
+         * There is three ways to read the data, determined by 'bandDimension' value:
+         *
+         *   • (bandDimension < 0): one variable per band (usual case).
+         *   • (bandDimension = 0): one variable containing all bands, with bands in the
first dimension.
+         *   • (bandDimension > 0): one variable containing all bands, with bands in
the last dimension.
+         */
         final DataBuffer imageBuffer;
-        final SampleDimension[] selected = new SampleDimension[range.length];
+        final SampleDimension[] bands = new SampleDimension[rangeIndices.getNumBands()];
         try {
-            final Buffer[] samples = new Buffer[range.length];
-            final GridDerivation change = gridGeometry.derive().subgrid(domain);
-            final int[] subsamplings = change.getSubsamplings();
-            SampleDimension.Builder builder = null;
+            final Buffer[] sampleValues = new Buffer[bands.length];
+            final GridDerivation targetGeometry = gridGeometry.derive().subgrid(domain);
+            GridExtent areaOfInterest = targetGeometry.getIntersection();
+            final int[] scales = targetGeometry.getSubsamplings();
+            int[] subsamplings = scales;
+            if (bandDimension >= 0) {
+                areaOfInterest = rangeIndices.insertBandDimension(areaOfInterest, bandDimension);
+                subsamplings   = ArraysExt.insert(subsamplings, bandDimension, 1);
+                subsamplings[bandDimension] = 1;
+            }
             /*
              * Iterate over netCDF variables in the order they appear in the file, not in
the order requested
              * by the 'range' argument.  The intent is to perform sequential I/O as much
as possible, without
-             * seeking backward. A side effect is that even if the same sample dimension
were requested many
-             * times, the data would still be loaded at most once.
+             * seeking backward.
              */
-            for (int i=0; i<data.length; i++) {
-                final Variable variable = data[i];
-                SampleDimension def = ranges[i];
-                Buffer values = null;
-                for (int j=0; j<range.length; j++) {
-                    if (range[j] == i) {                // Search sample dimensions specified
in the 'range' argument.
-                        if (def == null) {
-                            if (builder == null) builder = new SampleDimension.Builder();
-                            ranges[i] = def = createSampleDimension(builder, variable);
-                            builder.clear();
-                        }
-                        if (values == null) {
-                            // Optional.orElseThrow() below should never fail since Variable.read(…)
wraps primitive array.
-                            values = variable.read(change.getIntersection(), subsamplings).buffer().get();
-                        }
-                        selected[j] = def;
-                        samples[j] = values;
-                    }
+            Buffer values = null;
+            for (int i=0; i<bands.length; i++) {
+                final int r = rangeIndices.getSourceIndex(i);                   // In strictly
increasing order.
+                final Variable variable = data[bandDimension >= 0 ? 0 : r];     // Only
one variable if bandDimension ≧ 0.
+                SampleDimension sd = ranges[r];
+                if (sd == null) {
+                    ranges[r] = sd = createSampleDimension(rangeIndices.builder(), variable);
+                }
+                if (bandDimension > 0) {
+                    // TODO: adjust 'areaOfInterest'.
+                    throw new UnsupportedOperationException();
+                }
+                if (bandDimension != 0 || values == null) {
+                    // Optional.orElseThrow() below should never fail since Variable.read(…)
wraps primitive array.
+                    values = variable.read(areaOfInterest, subsamplings).buffer().get();
+                }
+                if (bandDimension == 0) {
+                    values.position(r);
                 }
+                final int p = rangeIndices.getTargetIndex(i);
+                sampleValues[p] = values;
+                bands[p] = sd;
             }
-            domain = change.subsample(subsamplings).build();
-            imageBuffer = RasterFactory.wrap(dataType.rasterDataType, samples);
+            domain = targetGeometry.subsample(scales).build();
+            imageBuffer = RasterFactory.wrap(dataType.rasterDataType, sampleValues);
         } catch (IOException e) {
             throw new DataStoreException(e);
         } catch (TransformException e) {
             throw new DataStoreReferencingException(e);
-        } catch (RuntimeException e) {                  // Many exceptions thrown by RasterFactory.wrap(…).
+        } catch (RuntimeException e) {                          // Many exceptions thrown
by RasterFactory.wrap(…).
             final Throwable cause = e.getCause();
             if (cause instanceof TransformException) {
                 throw new DataStoreReferencingException(cause);
@@ -442,7 +477,7 @@ public final class RasterResource extends AbstractGridResource implements
Resour
         if (imageBuffer == null) {
             throw new DataStoreContentException(Errors.format(Errors.Keys.UnsupportedType_1,
dataType.name()));
         }
-        return new Raster(domain, UnmodifiableArrayList.wrap(selected), imageBuffer, first.getStandardName());
+        return new Raster(domain, UnmodifiableArrayList.wrap(bands), imageBuffer, first.getStandardName());
     }
 
     /**
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
index e28037b..68d44ba 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
@@ -131,6 +131,12 @@ public abstract class Variable extends NamedElement {
     private boolean gridDetermined;
 
     /**
+     * If {@link #gridGeometry} has less dimensions than this variable, index of a grid dimension
to take as raster bands.
+     * Otherwise this field is left uninitialized. If set, the index is relative to "natural"
order (reverse of netCDF order).
+     */
+    int bandDimension;
+
+    /**
      * Creates a new variable.
      *
      * @param decoder  the netCDF file where this variable is stored.
@@ -634,6 +640,13 @@ public abstract class Variable extends NamedElement {
     }
 
     /**
+     * Returns the number of grid dimension. This is the length of the array returned by
{@link #getGridDimensions()}.
+     *
+     * @return number of grid dimensions.
+     */
+    public abstract int getDimension();
+
+    /**
      * Returns the grid geometry for this variable, or {@code null} if this variable is not
a data cube.
      * Not all variables have a grid geometry. For example collections of features do not
have such grid.
      * The same grid geometry may be shared by many variables.
@@ -674,6 +687,7 @@ public abstract class Variable extends NamedElement {
                                 copied = true;
                                 dimensions = new ArrayList<>(dimensions);
                             }
+                            bandDimension = getDimension() - i - 1;         // Convert netCDF
order to "natural" order.
                             dimensions.remove(i);
                             if (dimensions.size() < numToKeep) {
                                 throw new InternalDataStoreException();     // Should not
happen (see above comment).
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
index 1d58f8a..7d82287 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
@@ -500,6 +500,14 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo>
{
     }
 
     /**
+     * Returns the number of grid dimension. This is the length of the array returned by
{@link #getGridDimensions()}.
+     */
+    @Override
+    public int getDimension() {
+        return dimensions.length;
+    }
+
+    /**
      * Returns the names of all attributes associated to this variable.
      *
      * @return names of all attributes associated to this variable.
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
index 0505744..0d20e46 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
@@ -287,6 +287,14 @@ final class VariableWrapper extends Variable {
     }
 
     /**
+     * Returns the number of grid dimension. This is the length of the array returned by
{@link #getGridDimensions()}.
+     */
+    @Override
+    public int getDimension() {
+        return variable.getRank();
+    }
+
+    /**
      * Returns the names of all attributes associated to this variable.
      *
      * @return names of all attributes associated to this variable.
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/AbstractGridResource.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/AbstractGridResource.java
index 1355db9..bd596ae 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/AbstractGridResource.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/AbstractGridResource.java
@@ -16,17 +16,18 @@
  */
 package org.apache.sis.internal.storage;
 
-import java.util.BitSet;
+import java.util.Arrays;
 import org.opengis.geometry.Envelope;
 import org.apache.sis.storage.DataStore;
 import org.apache.sis.storage.DataStoreException;
 import org.apache.sis.storage.GridCoverageResource;
 import org.apache.sis.coverage.grid.GridGeometry;
+import org.apache.sis.coverage.grid.GridExtent;
 import org.apache.sis.coverage.SampleDimension;
 import org.apache.sis.storage.Resource;
 import org.apache.sis.util.logging.WarningListeners;
 import org.apache.sis.util.ArgumentChecks;
-import org.apache.sis.util.ArraysExt;
+import org.opengis.metadata.spatial.DimensionNameType;
 
 
 /**
@@ -95,45 +96,143 @@ public abstract class AbstractGridResource extends AbstractResource implements
G
      * This method verifies that all indices are between 0 and {@code numSampleDimensions}
      * and that there is no duplicated index.
      *
-     * <p>On success, this method always returns a non-null array with the same content
than the user argument.
-     * The user specified array is not returned directly in order to allow the caller to
perform modifications,
-     * and also as a paranoiac safety against concurrent changes.</p>
-     *
      * @param  numSampleDimensions  number of sample dimensions.
      * @param  range  the {@code range} argument given by the user. May be null or empty.
-     * @return the 0-based indices of ranges to use. May be a copy of the given {@code range}
argument or a sequence
-     *         from 0 to {@code numSampleDimensions} exclusive if {@code range} was {@code
null} or empty.
+     * @return the {@code range} argument encapsulated with a set of convenience tools.
+     * @throws IllegalArgumentException if a range index is invalid.
      */
-    protected final int[] validateRangeArgument(final int numSampleDimensions, int[] range)
{
+    protected final RangeArgument validateRangeArgument(final int numSampleDimensions, final
int[] range) {
         ArgumentChecks.ensureStrictlyPositive("numSampleDimensions", numSampleDimensions);
+        final long[] packed;
         if (range == null || range.length == 0) {
-            return ArraysExt.range(0, numSampleDimensions);
-        }
-        range = range.clone();
-        long previous = 0;                          // Cheap way to remember previous dimensions.
-        BitSet extra = null;                        // Used only if there is more than 64
dimensions.
-        for (int i=0; i<range.length; i++) {
-            final int r = range[i];
-            if (r < 0 || r >= numSampleDimensions) {
-                throw new IllegalArgumentException(Resources.forLocale(getLocale()).getString(
-                        Resources.Keys.InvalidSampleDimensionIndex_2, i, numSampleDimensions
- 1));
+            packed = new long[numSampleDimensions];
+            for (int i=1; i<numSampleDimensions; i++) {
+                packed[i] = (((long) i) << Integer.SIZE) | i;
             }
-            final boolean duplicated;
-            if (r < Long.SIZE) {
-                duplicated = (previous == (previous |= (1L << r)));
-            } else {
-                if (extra == null) {
-                    extra = new BitSet();
+            return new RangeArgument(packed);
+        } else {
+            packed = new long[range.length];
+            for (int i=0; i<range.length; i++) {
+                final int r = range[i];
+                if (r < 0 || r >= numSampleDimensions) {
+                    throw new IllegalArgumentException(Resources.forLocale(getLocale()).getString(
+                            Resources.Keys.InvalidSampleDimensionIndex_2, r, numSampleDimensions
- 1));
+                }
+                packed[i] = (((long) r) << Integer.SIZE) | i;
+            }
+            Arrays.sort(packed);
+            int previous = -1;
+            for (int i=0; i<packed.length; i++) {
+                final int r = (int) (packed[i] >>> Integer.SIZE);
+                if (r == previous) {
+                    throw new IllegalArgumentException(Resources.forLocale(getLocale()).getString(
+                            Resources.Keys.DuplicatedSampleDimensionIndex_1, r));
+                }
+                previous = r;
+            }
+        }
+        return new RangeArgument(packed);
+    }
+
+    /**
+     * The user-provided {@code range} argument, together with a set of convenience tools.
+     */
+    protected static final class RangeArgument {
+        /**
+         * Name of the extent dimension for bands.
+         */
+        private static DimensionNameType BAND = DimensionNameType.valueOf("BAND");
+
+        /**
+         * The user-specified range indices in high bits, together with indices order in
the low bits.
+         * This array is sorted.
+         */
+        private final long[] packed;
+
+        /**
+         * A builder for sample dimensions, created when first needed.
+         */
+        private SampleDimension.Builder builder;
+
+        /**
+         * Encapsulates the given {@code range} argument packed in high bits.
+         */
+        RangeArgument(final long[] packed) {
+            this.packed = packed;
+        }
+
+        /**
+         * Returns the number of sample dimensions. This is the length of the range array
supplied by user.
+         *
+         * @return the number of sample dimensions.
+         */
+        public int getNumBands() {
+            return packed.length;
+        }
+
+        /**
+         * Returns the i<sup>th</sup> index of the band to read from the resource.
+         * Indices are returned in strictly increasing order.
+         *
+         * @param  i  index of the range index to get, from 0 inclusive to {@link #getNumBands()}
exclusive.
+         * @return index of the i<sup>th</sup> band to read from the resource.
+         */
+        public int getSourceIndex(final int i) {
+            return (int) (packed[i] >>> Integer.SIZE);
+        }
+
+        /**
+         * Returns the i<sup>th</sup> band position. This is the index in the
user-supplied {@code range} array
+         * where was specified the {@code getBandIndex(i)} value.
+         *
+         * @param  i  index of the range index to get, from 0 inclusive to {@link #getNumBands()}
exclusive.
+         * @return index in user-supplied {@code range} array where was specified the {@code
getBandIndex(i)} value.
+         */
+        public int getTargetIndex(final int i) {
+            return (int) packed[i];
+        }
+
+        /**
+         * Returns the value of the first index specified by the user. This is not necessarily
equal to
+         * {@code getBandIndex(0)} if the user specified bands out of order.
+         *
+         * @return index of the first value in the user-specified {@code range} array.
+         */
+        public int first() {
+            for (final long p : packed) {
+                if (((int) p) == 0) {
+                    return (int) (p >>> Integer.SIZE);
                 }
-                final int j = r - Long.SIZE;
-                duplicated = extra.get(j);
-                extra.set(j);
             }
-            if (duplicated) {
-                throw new IllegalArgumentException(Resources.forLocale(getLocale()).getString(
-                        Resources.Keys.DuplicatedSampleDimensionIndex_1, i));
+            throw new IllegalStateException();              // Should never happen.
+        }
+
+        /**
+         * Returns the given extent with a new dimension added for the bands. The extent
in the new dimension
+         * will range from the minimum {@code range} value to the maximum {@code range} value
inclusive.
+         *
+         * @param  areaOfInterest  the extent to which to add a new dimension for bands.
+         * @param  bandDimension   index of the band dimension.
+         * @return a new extent with the same value than the given extent plus one dimension
for bands.
+         */
+        public GridExtent insertBandDimension(final GridExtent areaOfInterest, final int
bandDimension) {
+            return areaOfInterest.insert(bandDimension, BAND, getSourceIndex(0), getSourceIndex(packed.length
- 1), true);
+        }
+
+        /**
+         * Returns a builder for sample dimensions. This method recycles the same builder
on every calls.
+         * If the builder has been returned by a previous call to this method,
+         * then it is {@linkplain SampleDimension.Builder#clear() cleared} before to be returned
again.
+         *
+         * @return a recycled builder for sample dimensions.
+         */
+        public SampleDimension.Builder builder() {
+            if (builder == null) {
+                builder = new SampleDimension.Builder();
+            } else {
+                builder.clear();
             }
+            return builder;
         }
-        return range;
     }
 }


Mime
View raw message