sis-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From desruisse...@apache.org
Subject [sis] branch geoapi-4.0 updated: When an additional offset needs to be added to the pixel stride, scanline stride, plane stride, etc., express that offset in byte units instead than sample units. This change fixes https://issues.apache.org/jira/browse/SIS-476
Date Mon, 07 Oct 2019 18:32:35 GMT
This is an automated email from the ASF dual-hosted git repository.

desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git


The following commit(s) were added to refs/heads/geoapi-4.0 by this push:
     new 47cbce0  When an additional offset needs to be added to the pixel stride, scanline
stride, plane stride, etc., express that offset in byte units instead than sample units. This
change fixes https://issues.apache.org/jira/browse/SIS-476
47cbce0 is described below

commit 47cbce0aa7d2080c5303e63803e04801dbe5c61e
Author: Martin Desruisseaux <martin.desruisseaux@geomatys.com>
AuthorDate: Mon Oct 7 20:25:56 2019 +0200

    When an additional offset needs to be added to the pixel stride, scanline stride,
    plane stride, etc., express that offset in byte units instead than sample units.
    This change fixes https://issues.apache.org/jira/browse/SIS-476
---
 .../apache/sis/internal/netcdf/RasterResource.java |  4 +-
 .../org/apache/sis/internal/netcdf/Variable.java   | 14 +++-
 .../sis/internal/netcdf/impl/VariableInfo.java     | 10 ++-
 .../apache/sis/internal/netcdf/package-info.java   |  2 +-
 .../internal/storage/io/HyperRectangleReader.java  | 13 ++--
 .../org/apache/sis/internal/storage/io/Region.java | 78 +++++++++++++++++-----
 .../sis/internal/storage/io/package-info.java      |  2 +-
 7 files changed, 89 insertions(+), 34 deletions(-)

diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/RasterResource.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/RasterResource.java
index 6a7c949..20a8eea 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/RasterResource.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/RasterResource.java
@@ -600,9 +600,11 @@ public final class RasterResource extends AbstractGridResource implements
Resour
                         bandOffsets[indexInRaster] = i;
                         indexInRaster = 0;                  // Pixels interleaved in one
bank: sampleValues.length = 1.
                     }
-                    if (i < numBuffers) {
+                    if (i < numBuffers) try {
                         // Optional.orElseThrow() below should never fail since Variable.read(…)
wraps primitive array.
                         sampleValues[indexInRaster] = variable.read(areaOfInterest, subsamplings).buffer().get();
+                    } catch (ArithmeticException e) {
+                        throw variable.canNotComputePosition(e);
                     }
                 }
             }
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
index fcf6b98..308f47d 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
@@ -30,6 +30,7 @@ import javax.measure.Unit;
 import org.opengis.referencing.operation.Matrix;
 import org.apache.sis.referencing.operation.transform.TransferFunction;
 import org.apache.sis.storage.DataStoreException;
+import org.apache.sis.storage.DataStoreContentException;
 import org.apache.sis.storage.InternalDataStoreException;
 import org.apache.sis.coverage.grid.GridGeometry;
 import org.apache.sis.coverage.grid.GridExtent;
@@ -51,7 +52,7 @@ import ucar.nc2.constants.CF;
  *
  * @author  Martin Desruisseaux (Geomatys)
  * @author  Johann Sorel (Geomatys)
- * @version 1.0
+ * @version 1.1
  * @since   0.3
  * @module
  */
@@ -1041,6 +1042,17 @@ public abstract class Variable extends Node {
     }
 
     /**
+     * Constructs the exception to thrown when the variable position can not be computed.
+     *
+     * @param  cause  the reason why we can not compute the position, or {@code null}.
+     * @return the exception to thrown.
+     */
+    protected final DataStoreContentException canNotComputePosition(final ArithmeticException
cause) {
+        return new DataStoreContentException(resources().getString(
+                Resources.Keys.CanNotComputeVariablePosition_2, getFilename(), getName()),
cause);
+    }
+
+    /**
      * Appends the name of the variable data type as the name of the primitive type
      * followed by the span of each dimension (in unit of grid cells) between brackets.
      * Dimensions are listed in "natural" order (reverse of netCDF order).
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
index 72e73e2..0c0b3bd 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
@@ -316,7 +316,7 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo>
{
         } else if (count == 1) {
             unlimited[0].offsetToNextRecord = 0;        // Special case cited in method javadoc.
         } else for (int i=0; i<count; i++) {
-            unlimited[i].offsetToNextRecord = recordStride - unlimited[i].offsetToNextRecord;
+            unlimited[i].offsetToNextRecord = Math.subtractExact(recordStride, unlimited[i].offsetToNextRecord);
         }
         /*
          * If some variables have a "coordinates" attribute listing names of variables used
as axes,
@@ -630,12 +630,10 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo>
{
      */
     private void applyUnlimitedDimensionStride(final Region region) throws DataStoreContentException
{
         if (isUnlimited()) {
-            final int dataSize = reader.dataSize();
-            if (offsetToNextRecord < 0 || (offsetToNextRecord % dataSize) != 0) {
-                throw new DataStoreContentException(resources()
-                        .getString(Resources.Keys.CanNotComputeVariablePosition_2, getFilename(),
name));
+            if (offsetToNextRecord < 0) {
+                throw canNotComputePosition(null);
             }
-            region.increaseStride(dimensions.length - 1, offsetToNextRecord / dataSize);
+            region.setAdditionalByteOffset(dimensions.length - 1, offsetToNextRecord);
         }
     }
 
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/package-info.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/package-info.java
index 3a164ec..350d15f 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/package-info.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/package-info.java
@@ -38,7 +38,7 @@
  * consistent vocabulary (more similar to the rest of Apache SIS).</p>
  *
  * @author  Martin Desruisseaux (IRD, Geomatys)
- * @version 1.0
+ * @version 1.1
  * @since   0.3
  * @module
  */
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/HyperRectangleReader.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/HyperRectangleReader.java
index 84245e1..021c0f9 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/HyperRectangleReader.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/HyperRectangleReader.java
@@ -109,7 +109,7 @@ public final class HyperRectangleReader {
      *
      * @return number of bytes per value.
      */
-    public int dataSize() {
+    public final int sampleSize() {
         return 1 << reader.dataSizeShift();
     }
 
@@ -120,17 +120,18 @@ public final class HyperRectangleReader {
      * @param  region  the sub-area to read and the subsampling to use.
      * @return the data in an array of primitive type.
      * @throws IOException if an error occurred while transferring data from the channel.
+     * @throws ArithmeticException if the region to read is too large or too far from origin.
      */
     public Object read(final Region region) throws IOException {
         final int contiguousDataDimension = region.contiguousDataDimension();
         final int contiguousDataLength = region.targetLength(contiguousDataDimension);
         final long[] strides = new long[region.getDimension() - contiguousDataDimension];
         final int[]   cursor = new int[strides.length];
-        final int  sizeShift = reader.dataSizeShift();
-        long  streamPosition = origin + (region.startAt << sizeShift);
+        final int sampleSize = sampleSize();
+        long  streamPosition = Math.addExact(origin, Math.multiplyExact(region.startAt, sampleSize));
         int    arrayPosition = 0;
         for (int i=0; i<strides.length; i++) {
-            strides[i] = (region.skips[i + contiguousDataDimension] + contiguousDataLength)
<< sizeShift;
+            strides[i] = region.stride(i + contiguousDataDimension, contiguousDataLength,
sampleSize);
             assert (strides[i] > 0) : i;
         }
         try {
@@ -149,8 +150,8 @@ loop:       do {
                      * skip.
                      */
                     if (++cursor[i] < region.targetSize[contiguousDataDimension + i])
{
-                        streamPosition += strides[i];
-                        arrayPosition  += contiguousDataLength;
+                        streamPosition = Math.addExact(streamPosition, strides[i]);
+                        arrayPosition  = Math.addExact(arrayPosition, contiguousDataLength);
                         continue loop;
                     }
                     cursor[i] = 0;
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/Region.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/Region.java
index 0d51113..25d0be3 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/Region.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/Region.java
@@ -17,7 +17,11 @@
 package org.apache.sis.internal.storage.io;
 
 import org.apache.sis.io.TableAppender;
-import org.apache.sis.internal.util.Numerics;
+
+import static java.lang.Math.addExact;
+import static java.lang.Math.multiplyExact;
+import static java.lang.Math.toIntExact;
+import static org.apache.sis.internal.util.Numerics.ceilDiv;
 
 
 /**
@@ -30,9 +34,12 @@ import org.apache.sis.internal.util.Numerics;
  * <p>This class assumes that the values are stored in a sequence (array or uncompressed
file)
  * where index at dimension 0 varies fastest, followed by index at dimension 1, <i>etc</i>.</p>
  *
+ * <p>This class has no knowledge of data size. The same {@code Region} instance can
be used
+ * for reading {@code byte} and {@code float} arrays for instance.</p>
+ *
  * @author  Johann Sorel (Geomatys)
  * @author  Martin Desruisseaux (Geomatys)
- * @version 1.0
+ * @version 1.1
  * @since   0.7
  * @module
  */
@@ -64,7 +71,14 @@ public final class Region {
      *
      * The length of this array is the hyper-rectangle dimension plus one.
      */
-    final long[] skips;
+    private final long[] skips;
+
+    /**
+     * Additional values to add to {@link #skips}, but in bytes instead than as a number
of values.
+     * This is the only field in this {@link Region} class to be expressed in byte units.
+     * This offset is rarely provided.
+     */
+    private long[] skipBytes;
 
     /**
      * Creates a new region. It is caller's responsibility to ensure that:
@@ -94,36 +108,44 @@ public final class Region {
         for (int i=0; i<dimension;) {
             final int  step  = subsamplings[i];
             final long lower =  regionLower[i];
-            final long count = Numerics.ceilDiv(regionUpper[i] - lower, step);
+            final long count = ceilDiv(regionUpper[i] - lower, step);
             final long upper = lower + ((count-1) * step + 1);
             final long span  = size[i];
             assert (count > 0) && (lower >= 0) && (upper > lower)
&& (upper <= span) : i;
-            targetSize[i] = Math.toIntExact(count);
 
-            position = Math.addExact(position, Math.multiplyExact(stride, lower));
-            skip     = Math.addExact(skip,     Math.multiplyExact(stride, span - (upper -
lower)));
-            skips[i] = Math.addExact(skips[i], Math.multiplyExact(stride, step - 1));
-            stride   = Math.multiplyExact(stride, span);
-            skips[++i] = skip;
+            targetSize[i] = toIntExact(count);
+            position      = addExact(position, multiplyExact(stride, lower));
+            skip          = addExact(skip,     multiplyExact(stride, span - (upper - lower)));
+            skips[i]      = addExact(skips[i], multiplyExact(stride, step - 1));
+            stride        = multiplyExact(stride, span);
+            skips[++i]    = skip;
         }
         startAt = position;
     }
 
     /**
-     * Increases the number of values between two consecutive index values in the given dimension
of the hyper-cube.
+     * Sets an additional offset between values at two consecutive index in the given dimension
of the hyper-cube.
      * The strides are computed automatically at construction time, but this method can be
invoked in some rare cases
      * where those values need to be modified (example: for adapting to the layout of netCDF
"unlimited" variable).
      *
      * <div class="note"><b>Example:</b> in a cube of dimension 10×10×10,
the number of values between indices
-     * (0,0,1) and (0,0,2) is 100. Invoking {@code increaseStride(1, 4)} will increase this
value to 104.
-     * {@link HyperRectangleReader} will still read only the requested 100 values, but will
skip 4 more values
-     * when moving from plane 1 to plane 2.</div>
+     * (0,0,1) and (0,0,2) is 100. If the values type is {@code float}, invoking {@code increaseStride(1,
12)}
+     * will increase this value to 103 (computed as 100 + 12/{@value Float#SIZE}). {@link
HyperRectangleReader}
+     * will still read only the requested 100 values, but will skip 3 more values when moving
from plane 1 to
+     * plane 2.</div>
+     *
+     * This method is the only one in this {@link Region} class to use a count of bytes instead
than a count
+     * of sample values.
      *
      * @param  dimension  dimension for which to increase the stride.
-     * @param  skip       additional number of values to skip after we finished reading a
block of data in the specified dimension.
+     * @param  skip       additional number of <strong>bytes</strong> to skip
after we finished reading
+     *                    a block of data in the specified dimension.
      */
-    public void increaseStride(final int dimension, final long skip) {
-        skips[dimension] = Math.addExact(skips[dimension], skip);
+    public void setAdditionalByteOffset(final int dimension, final long skip) {
+        if (skipBytes == null) {
+            skipBytes = new long[getDimension()];
+        }
+        skipBytes[dimension] = skip;
     }
 
     /**
@@ -149,6 +171,26 @@ public final class Region {
     }
 
     /**
+     * Computes the number of stride for the given dimension. Caller must provide a base
stride value,
+     * which is the stride that we would have in absence of subregion and additional bytes
to skip.
+     * This method adds the given base the number of bytes to skip.
+     *
+     * @param  dimension    the dimension for which to compute the stride.
+     * @param  base         stride that we would have in absence of subregion and bytes to
skip.
+     * @param  sampleSize   size of sample values, in bytes.
+     * @return stride in bytes.
+     * @throws ArithmeticException if the number of bytes exceed the {@code long} capacity.
+     */
+    final long stride(final int dimension, final int base, final int sampleSize) {
+        long stride = multiplyExact(addExact(skips[dimension], base), sampleSize);
+        if (skipBytes != null) {
+            // This additional offset is in bytes.
+            stride = addExact(stride, skipBytes[dimension]);
+        }
+        return stride;
+    }
+
+    /**
      * Returns the total number of values to be read from the sub-region while applying the
subsampling.
      * This method takes in account only the given number of dimensions.
      */
@@ -157,7 +199,7 @@ public final class Region {
         for (int i=0; i<dimension; i++) {
             length *= targetSize[i];
         }
-        return Math.toIntExact(length);
+        return toIntExact(length);
     }
 
     /**
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/package-info.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/package-info.java
index 79e0b07..ea669bd 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/package-info.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/package-info.java
@@ -24,7 +24,7 @@
  * may change in incompatible ways in any future version without notice.
  *
  * @author  Martin Desruisseaux (Geomatys)
- * @version 1.0
+ * @version 1.1
  * @since   0.3
  * @module
  */


Mime
View raw message