sis-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From desruisse...@apache.org
Subject [sis] 02/02: Share the localization grid computed by previous reading of similar netCDF files. A MD5 checksum is used for determining if the coordinate values are the same. This can save a lot of computation since a localization grid may take a few seconds to compute.
Date Sat, 01 Jun 2019 15:20:06 GMT
This is an automated email from the ASF dual-hosted git repository.

desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git

commit a898d3fb0b9b7041bbdb96feb94becf57cce94d6
Author: Martin Desruisseaux <martin.desruisseaux@geomatys.com>
AuthorDate: Sat Jun 1 17:18:51 2019 +0200

    Share the localization grid computed by previous reading of similar netCDF files.
    A MD5 checksum is used for determining if the coordinate values are the same.
    This can save a lot of computation since a localization grid may take a few seconds to compute.
---
 .../java/org/apache/sis/internal/netcdf/Axis.java  | 165 ++++++++++----
 .../org/apache/sis/internal/netcdf/Decoder.java    |  48 +++-
 .../java/org/apache/sis/internal/netcdf/Grid.java  |  29 +--
 .../apache/sis/internal/netcdf/GridCacheKey.java   | 251 +++++++++++++++++++++
 .../org/apache/sis/internal/netcdf/Linearizer.java |   2 +-
 .../org/apache/sis/internal/netcdf/Resources.java  |   5 +
 .../sis/internal/netcdf/Resources.properties       |   1 +
 .../sis/internal/netcdf/Resources_fr.properties    |   1 +
 .../sis/internal/netcdf/impl/VariableInfo.java     |   6 +-
 .../sis/internal/netcdf/ucar/VariableWrapper.java  |   5 +
 .../apache/sis/internal/storage/io/ByteWriter.java | 213 +++++++++++++++++
 .../sis/internal/storage/io/package-info.java      |   2 +-
 12 files changed, 654 insertions(+), 74 deletions(-)

diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java
index 68db514..00f359b 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java
@@ -18,8 +18,10 @@ package org.apache.sis.internal.netcdf;
 
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Set;
 import java.util.Map;
 import java.util.HashMap;
+import java.util.Arrays;
 import java.io.IOException;
 import javax.measure.Unit;
 import javax.measure.UnitConverter;
@@ -31,6 +33,7 @@ import org.opengis.referencing.cs.AxisDirection;
 import org.opengis.referencing.cs.CoordinateSystemAxis;
 import org.opengis.referencing.operation.Matrix;
 import org.opengis.referencing.operation.MathTransform;
+import org.opengis.referencing.operation.MathTransformFactory;
 import org.opengis.metadata.content.TransferFunctionType;
 import org.apache.sis.internal.metadata.AxisDirections;
 import org.apache.sis.internal.util.Numerics;
@@ -41,6 +44,7 @@ import org.apache.sis.referencing.NamedIdentifier;
 import org.apache.sis.metadata.iso.citation.Citations;
 import org.apache.sis.storage.DataStoreException;
 import org.apache.sis.util.resources.Errors;
+import org.apache.sis.util.collection.Cache;
 import org.apache.sis.util.iso.Types;
 import org.apache.sis.util.ArraysExt;
 import org.apache.sis.measure.Longitude;
@@ -691,53 +695,102 @@ public final class Axis extends NamedElement {
      * @throws IOException if an error occurred while reading the data.
      * @throws DataStoreException if a logical error occurred.
      */
-    final LocalizationGridBuilder createLocalizationGrid(final Axis other) throws IOException, DataStoreException {
-        if (getDimension() == 2 && other.getDimension() == 2) {
-            final int xd =  this.sourceDimensions[0];
-            final int yd =  this.sourceDimensions[1];
-            final int xo = other.sourceDimensions[0];
-            final int yo = other.sourceDimensions[1];
-            if ((xo == xd && yo == yd) || (xo == yd && yo == xd)) {
+    final MathTransform createLocalizationGrid(final Axis other) throws IOException, FactoryException, DataStoreException {
+        if (getDimension() != 2 || other.getDimension() != 2) {
+            return null;
+        }
+        final int xd =  this.sourceDimensions[0];
+        final int yd =  this.sourceDimensions[1];
+        final int xo = other.sourceDimensions[0];
+        final int yo = other.sourceDimensions[1];
+        if ((xo != xd | yo != yd) & (xo != yd | yo != xd)) {
+            return null;
+        }
+        /*
+         * Found two axes for the same set of dimensions, which implies that they have the same
+         * shape (width and height) unless the two axes ignored a different amount of NaN values.
+         * Negative width and height means that their actual values overflow the 'int' capacity,
+         * which we can not process here.
+         */
+        final int ri = (xd <= yd) ? 0 : 1;          // Take in account that mainDimensionFirst(…) may have reordered values.
+        final int ro = (xo <= yo) ? 0 : 1;
+        final int width  = getSize(ri ^ 1);         // Fastest varying is right-most dimension (when in netCDF order).
+        final int height = getSize(ri    );         // Slowest varying is left-most dimension (when in netCDF order).
+        if (other.sourceSizes[ro ^ 1] != width ||
+            other.sourceSizes[ro    ] != height)
+        {
+            warning(null, Errors.Keys.MismatchedGridGeometry_2, getName(), other.getName());
+            return null;
+        }
+        /*
+         * First, verify if the localization grid has already been created previously. It happens if the netCDF file
+         * contains data with different number of dimensions. For example a file may have a variable with (longitude,
+         * latitude) and another variable with (longitude, latitude, depth) dimensions, with both variables using the
+         * same localization grid for the (longitude, latitude) part.
+         */
+        final Decoder decoder = coordinates.decoder;
+        final GridCacheKey keyLocal = new GridCacheKey(width, height, this, other);
+        MathTransform tr = keyLocal.cached(decoder);
+        if (tr != null) {
+            return tr;
+        }
+        /*
+         * If there is no localization grid in the cache locale to the netCDF decoder, try again in the global cache.
+         * This check is more expensive since it computes MD5 sum of all coordinate values.
+         */
+        final long time = System.nanoTime();
+        final Vector vx =  this.read();
+        final Vector vy = other.read();
+        final Set<Linearizer> linearizers = decoder.convention().linearizers(decoder);
+        final GridCacheKey.Global keyGlobal = new GridCacheKey.Global(keyLocal, vx, vy, linearizers);
+        final Cache.Handler<MathTransform> handler = keyGlobal.lock();
+        try {
+            tr = handler.peek();
+            if (tr == null) {
+                final LocalizationGridBuilder grid = new LocalizationGridBuilder(width, height);
+                grid.setControlPoints(vx, vy);
                 /*
-                 * Found two axes for the same set of dimensions, which implies that they have the same
-                 * shape (width and height) unless the two axes ignored a different amount of NaN values.
-                 * Negative width and height means that their actual values overflow the 'int' capacity,
-                 * which we can not process here.
+                 * At this point we finished to set values in the localization grid, but did not computed the transform yet.
+                 * Before to use the grid for calculation, we need to repair discontinuities sometime found with longitudes.
+                 * If the grid crosses the anti-meridian, some values may suddenly jump from +180° to -180° or conversely.
+                 * Even when not crossing the anti-meridian, we still observe apparently random 360° jumps in some files,
+                 * especially close to poles. The methods invoked below try to make the longitude grid more continuous.
+                 * The "ri" or "ro" argument specifies which dimension varies slowest, i.e. which dimension have values
+                 * that do not change much when increasing longitudes. This is usually 1 (the rows).
                  */
-                final int ri = (xd <= yd) ? 0 : 1;      // Take in account that mainDimensionFirst(…) may have reordered values.
-                final int ro = (xo <= yo) ? 0 : 1;
-                final int width  = getSize(ri ^ 1);     // Fastest varying is right-most dimension (when in netCDF order).
-                final int height = getSize(ri    );     // Slowest varying is left-most dimension (when in netCDF order).
-                if (other.sourceSizes[ro ^ 1] == width &&
-                    other.sourceSizes[ro    ] == height)
-                {
-                    final LocalizationGridBuilder grid = new LocalizationGridBuilder(width, height);
-                    final Vector vx =  this.read();
-                    final Vector vy = other.read();
-                    grid.setControlPoints(vx, vy);
-                    /*
-                     * At this point we finished to set values in the localization grid, but did not computed the transform yet.
-                     * Before to use the grid for calculation, we need to repair discontinuities sometime found with longitudes.
-                     * If the grid crosses the anti-meridian, some values may suddenly jump from +180° to -180° or conversely.
-                     * Even when not crossing the anti-meridian, we still observe apparently random 360° jumps in some files,
-                     * especially close to poles. The methods invoked below try to make the longitude grid more continuous.
-                     * The "ri" or "ro" argument specifies which dimension varies slowest, i.e. which dimension have values
-                     * that do not change much when increasing longitudes. This is usually 1 (the rows).
-                     */
-                    double period;
-                    if (!Double.isNaN(period = wraparoundRange())) {
-                        grid.resolveWraparoundAxis(0, ri, period);
-                    }
-                    if (!Double.isNaN(period = other.wraparoundRange())) {
-                        grid.resolveWraparoundAxis(1, ro, period);
-                    }
-                    return grid;
-                } else {
-                    warning(null, Errors.Keys.MismatchedGridGeometry_2, getName(), other.getName());
+                double period;
+                if (!Double.isNaN(period = wraparoundRange())) {
+                    grid.resolveWraparoundAxis(0, ri, period);
+                }
+                if (!Double.isNaN(period = other.wraparoundRange())) {
+                    grid.resolveWraparoundAxis(1, ro, period);
+                }
+                /*
+                 * Forward coordinate conversions are straightforward interpolations in the localization grid.
+                 * But inverse conversions are more difficult to perform as they require iterations. They will
+                 * converge better if the grid is close to linear.
+                 */
+                final MathTransformFactory factory = decoder.getMathTransformFactory();
+                if (!linearizers.isEmpty()) {
+                    Linearizer.applyTo(linearizers, factory, grid, this, other);
                 }
+                /*
+                 * There is usually a one-to-one relationship between localization grid cells and image pixels.
+                 * Consequently an accuracy set to a fraction of cell should be enough.
+                 *
+                 * TODO: take in account the case where Variable.Adjustment.dataToGridIndices() returns a value
+                 * smaller than 1. For now we set the desired precision to a value 10 times smaller in order to
+                 * take in account the case where dataToGridIndices() returns 0.1.
+                 */
+                grid.setDesiredPrecision(0.001);
+                tr = grid.create(factory);
+                tr = keyLocal.cache(decoder, tr);
             }
+        } finally {
+            handler.putAndUnlock(tr);
         }
-        return null;
+        decoder.performance(Grid.class, "getGridGeometry", Resources.Keys.ComputeLocalizationGrid_2, time);
+        return tr;
     }
 
     /**
@@ -774,4 +827,32 @@ public final class Axis extends NamedElement {
             throw new DataStoreException(coordinates.resources().getString(Resources.Keys.CanNotUseAxis_1, getName()));
         }
     }
+
+    /**
+     * Compares this axis with the given object for equality.
+     *
+     * @param  other  the other object to compare with this axis.
+     * @return whether the other object describes the same axis than this object.
+     */
+    @Override
+    public boolean equals(final Object other) {
+        if (other instanceof Axis) {
+            final Axis that = (Axis) other;
+            return that.abbreviation == abbreviation && that.direction == direction
+                    && Arrays.equals(that.sourceDimensions, sourceDimensions)
+                    && Arrays.equals(that.sourceSizes, sourceSizes)
+                    && coordinates.equals(that.coordinates);
+        }
+        return false;
+    }
+
+    /**
+     * Returns a hash code value for this axis. This method uses only properties that are quick to compute.
+     *
+     * @return a hash code value.
+     */
+    @Override
+    public int hashCode() {
+        return abbreviation + Arrays.hashCode(sourceDimensions) + Arrays.hashCode(sourceSizes);
+    }
 }
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Decoder.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Decoder.java
index 724ea61..63c55ca 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Decoder.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Decoder.java
@@ -24,6 +24,8 @@ import java.util.Collection;
 import java.util.Objects;
 import java.util.Date;
 import java.util.TimeZone;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.LogRecord;
 import java.io.Closeable;
 import java.io.IOException;
 import java.nio.file.Path;
@@ -31,14 +33,18 @@ import org.opengis.util.NameSpace;
 import org.opengis.util.NameFactory;
 import org.opengis.referencing.datum.Datum;
 import org.opengis.referencing.crs.CoordinateReferenceSystem;
+import org.opengis.referencing.operation.MathTransform;
 import org.apache.sis.setup.GeometryLibrary;
 import org.apache.sis.storage.DataStore;
 import org.apache.sis.storage.DataStoreException;
 import org.apache.sis.util.Utilities;
 import org.apache.sis.util.ComparisonMode;
+import org.apache.sis.util.logging.Logging;
+import org.apache.sis.util.logging.PerformanceLevel;
 import org.apache.sis.util.logging.WarningListeners;
 import org.apache.sis.internal.util.StandardDateFormat;
 import org.apache.sis.internal.system.DefaultFactories;
+import org.apache.sis.internal.system.Modules;
 import org.apache.sis.internal.referencing.ReferencingFactoryContainer;
 
 
@@ -115,6 +121,19 @@ public abstract class Decoder extends ReferencingFactoryContainer implements Clo
     final Map<Object,GridMapping> gridMapping;
 
     /**
+     * Cache of localization grids created for a given pair of (<var>x</var>,<var>y</var>) axes. Localization grids
+     * are expensive to compute and consume a significant amount of memory.  The {@link Grid} instances returned by
+     * {@link #getGrids()} allow to share localization grids only between variables using the exact same list of dimensions.
+     * This {@code localizationGrids} cache allows to cover other cases.
+     * For example a netCDF file may have a variable with (<var>longitude</var>, <var>latitude</var>) dimensions
+     * and another variable with (<var>longitude</var>, <var>latitude</var>, <var>depth</var>) dimensions,
+     * with both variables using the same localization grid for the (<var>longitude</var>, <var>latitude</var>) part.
+     *
+     * @see GridCacheKey#cached(Decoder)
+     */
+    final Map<GridCacheKey,MathTransform> localizationGrids;
+
+    /**
      * Where to send the warnings.
      */
     public final WarningListeners<DataStore> listeners;
@@ -133,11 +152,12 @@ public abstract class Decoder extends ReferencingFactoryContainer implements Clo
      */
     protected Decoder(final GeometryLibrary geomlib, final WarningListeners<DataStore> listeners) {
         Objects.requireNonNull(listeners);
-        this.geomlib     = geomlib;
-        this.listeners   = listeners;
-        this.nameFactory = DefaultFactories.forBuildin(NameFactory.class);
-        this.datumCache  = new Datum[CRSBuilder.DATUM_CACHE_SIZE];
-        this.gridMapping = new HashMap<>();
+        this.geomlib      = geomlib;
+        this.listeners    = listeners;
+        this.nameFactory  = DefaultFactories.forBuildin(NameFactory.class);
+        this.datumCache   = new Datum[CRSBuilder.DATUM_CACHE_SIZE];
+        this.gridMapping  = new HashMap<>();
+        localizationGrids = new HashMap<>();
     }
 
     /**
@@ -409,4 +429,22 @@ public abstract class Decoder extends ReferencingFactoryContainer implements Clo
      * @return the variable or group of the given name, or {@code null} if none.
      */
     protected abstract Node findNode(String name);
+
+    /**
+     * Logs a message about a potentially slow operation. This method does use the listeners registered to the netCDF reader
+     * because this is not a warning.
+     *
+     * @param  caller       the class to report as the source.
+     * @param  method       the method to report as the source.
+     * @param  resourceKey  a {@link Resources} key expecting filename as first argument and elapsed time as second argument.
+     * @param  time         value of {@link System#nanoTime()} when the operation started.
+     */
+    final void performance(final Class<?> caller, final String method, final short resourceKey, long time) {
+        time = System.nanoTime() - time;
+        final LogRecord record = Resources.forLocale(listeners.getLocale()).getLogRecord(
+                PerformanceLevel.forDuration(time, TimeUnit.NANOSECONDS), resourceKey,
+                getFilename(), time / (double) StandardDateFormat.NANOS_PER_SECOND);
+        record.setLoggerName(Modules.NETCDF);
+        Logging.log(caller, method, record);
+    }
 }
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Grid.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Grid.java
index fdaedb5..c539fa1 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Grid.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Grid.java
@@ -16,7 +16,6 @@
  */
 package org.apache.sis.internal.netcdf;
 
-import java.util.Set;
 import java.util.List;
 import java.util.Arrays;
 import java.util.ArrayList;
@@ -34,7 +33,6 @@ import org.opengis.referencing.crs.CoordinateReferenceSystem;
 import org.opengis.metadata.spatial.DimensionNameType;
 import org.apache.sis.internal.metadata.AxisDirections;
 import org.apache.sis.referencing.operation.matrix.Matrices;
-import org.apache.sis.referencing.operation.builder.LocalizationGridBuilder;
 import org.apache.sis.referencing.CRS;
 import org.apache.sis.coverage.grid.GridExtent;
 import org.apache.sis.coverage.grid.GridGeometry;
@@ -398,7 +396,7 @@ public abstract class Grid extends NamedElement {
              * dimension which is not already taken by another row. If we have choice, we give preference to the dimension
              * which seems most closely oriented toward axis direction (i.e. the first element in axis.sourceDimensions).
              *
-             * Example: if the 'axes' array contains (longitude, latitude) in that order, and if the longitude axis said
+             * Example: if the `axes` array contains (longitude, latitude) in that order, and if the longitude axis said
              * that its preferred dimension is 1 (after conversion to "natural" order) while the latitude axis said that
              * its preferred dimension is 0, then we build the following matrix:
              *
@@ -408,7 +406,7 @@ public abstract class Grid extends NamedElement {
              *    │ 0  0  1 │
              *    └         ┘
              *
-             * The preferred grid dimensions are stored in the 'sourceDimensions' array. In above example this is {1, 0}.
+             * The preferred grid dimensions are stored in the `sourceDimensions` array. In above example this is {1, 0}.
              */
             final int[] sourceDimensions = new int[nonLinears.size()];
             Arrays.fill(sourceDimensions, -1);
@@ -432,15 +430,14 @@ findFree:       for (int srcDim : axis.sourceDimensions) {
              * two-dimensional localization grid. Those transforms require two variables, i.e. "two-dimensional"
              * axes come in pairs.
              */
-            final MathTransformFactory factory = decoder.getMathTransformFactory();
             for (int i=0; i<nonLinears.size(); i++) {         // Length of 'nonLinears' may change in this loop.
                 if (nonLinears.get(i) == null) {
                     for (int j=i; ++j < nonLinears.size();) {
                         if (nonLinears.get(j) == null) {
                             /*
                              * Found a pair of axes.  Prepare an array of length 2, to be reordered later in the
-                             * axis order declared in 'sourceDimensions'. This is not necessarily the same order
-                             * than iteration order because it depends on values of 'axis.sourceDimensions[0]'.
+                             * axis order declared in `sourceDimensions`. This is not necessarily the same order
+                             * than iteration order because it depends on values of `axis.sourceDimensions[0]`.
                              * Those values take in account what is the "main" dimension of each axis.
                              */
                             final Axis[] gridAxes = new Axis[] {
@@ -454,26 +451,13 @@ findFree:       for (int srcDim : axis.sourceDimensions) {
                                 case +1: ArraysExt.swap(gridAxes, 0, 1); break;
                                 default: continue;            // Needs axes at consecutive source dimensions.
                             }
-                            final LocalizationGridBuilder grid = gridAxes[0].createLocalizationGrid(gridAxes[1]);
+                            final MathTransform grid = gridAxes[0].createLocalizationGrid(gridAxes[1]);
                             if (grid != null) {
-                                final Set<Linearizer> linearizers = decoder.convention().linearizers(decoder);
-                                if (!linearizers.isEmpty()) {
-                                    Linearizer.applyTo(linearizers, factory, grid, gridAxes);
-                                }
-                                /*
-                                 * There is usually a one-to-one relationship between localization grid cells and image pixels.
-                                 * Consequently an accuracy set to a fraction of cell should be enough.
-                                 *
-                                 * TODO: take in account the case where Variable.Adjustment.dataToGridIndices() returns a value
-                                 * smaller than 1. For now we set the desired precision to a value 10 times smaller in order to
-                                 * take in account the case where dataToGridIndices() returns 0.1.
-                                 */
-                                grid.setDesiredPrecision(0.001);
                                 /*
                                  * Replace the first transform by the two-dimensional localization grid and
                                  * remove the other transform. Removals need to be done in arrays too.
                                  */
-                                nonLinears.set(i, grid.create(factory));
+                                nonLinears.set(i, grid);
                                 nonLinears.remove(j);
                                 final int n = nonLinears.size() - j;
                                 System.arraycopy(deferred,         j+1, deferred,         j, n);
@@ -493,6 +477,7 @@ findFree:       for (int srcDim : axis.sourceDimensions) {
              */
             MathTransform gridToCRS = null;
             final int nonLinearCount = nonLinears.size();
+            final MathTransformFactory factory = decoder.getMathTransformFactory();
             nonLinears.add(factory.createAffineTransform(affine));
             for (int i=0; i <= nonLinearCount; i++) {
                 MathTransform tr = nonLinears.get(i);
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/GridCacheKey.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/GridCacheKey.java
new file mode 100644
index 0000000..5d14b2a
--- /dev/null
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/GridCacheKey.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sis.internal.netcdf;
+
+import java.util.Set;
+import java.util.Arrays;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import org.opengis.referencing.operation.MathTransform;
+import org.apache.sis.util.collection.Cache;
+import org.apache.sis.internal.util.Strings;
+import org.apache.sis.internal.storage.io.ByteWriter;
+import org.apache.sis.math.Vector;
+
+
+/**
+ * Cache management of localization grids. {@code GridCache} are used as keys in {@code HashMap}.
+ * There is two level of caches:
+ *
+ * <ul>
+ *   <li>Local to the {@link Decoder}. This avoid the need to compute MD5 sum of coordinate vectors.</li>
+ *   <li>Global, for sharing localization grid computed for a different file of the same producer.</li>
+ * </ul>
+ *
+ * The base class if for local cache. The inner class is for the global cache.
+ *
+ * @author  Martin Desruisseaux (Geomatys)
+ * @version 1.0
+ * @since   1.0
+ * @module
+ */
+class GridCacheKey {
+    /**
+     * Size of cached localization grid, in number of cells.
+     */
+    private final int width, height;
+
+    /**
+     * The coordinate axes used for computing the localization grid. For local cache, it shall be {@link Axis} instances.
+     * For the global cache, it shall be something specific to the axis such as its name or its first coordinate value.
+     * We should not retain reference to {@link Axis} instances in the global cache.
+     */
+    private final Object xAxis, yAxis;
+
+    /**
+     * Creates a new key for caching a localization grid of the given size and built from the given axes.
+     */
+    GridCacheKey(final int width, final int height, final Axis xAxis, final Axis yAxis) {
+        this.width  = width;
+        this.height = height;
+        this.xAxis  = xAxis;
+        this.yAxis  = yAxis;
+    }
+
+    /**
+     * Creates a global key from the given local key.
+     */
+    private GridCacheKey(final GridCacheKey keyLocal) {
+        width  = keyLocal.width;
+        height = keyLocal.height;
+        xAxis  = id(keyLocal.xAxis);
+        yAxis  = id(keyLocal.yAxis);
+    }
+
+    /**
+     * Returns an identifier for the given axis. Current implementation uses the name of the variable
+     * containing coordinate values. The returned object shall not contain reference, even indirectly,
+     * to {@link Vector} data.
+     */
+    private static Object id(final Object axis) {
+        return ((Axis) axis).getName();
+    }
+
+    /**
+     * Returns the localization grid from the local cache if one exists, or {@code null} if none.
+     * This method looks only in the local cache. For the global cache, see {@link Global#lock()}.
+     */
+    final MathTransform cached(final Decoder decoder) {
+        return decoder.localizationGrids.get(this);
+    }
+
+    /**
+     * Caches the given localization grid in the local caches.
+     * This method is invoked after a new grid has been created.
+     *
+     * @param  decoder  the decoder with local cache.
+     * @param  grid     the grid to cache.
+     * @return the cached grid. Should be the given {@code grid} instance, unless another grid has been cached concurrently.
+     */
+    final MathTransform cache(final Decoder decoder, final MathTransform grid) {
+        final MathTransform tr = decoder.localizationGrids.putIfAbsent(this, grid);
+        return (tr != null) ? tr : grid;
+    }
+
+    /**
+     * Key for localization grids in the global cache. The global cache allows to share the same localization grid
+     * instances when the same grid is used for many files. This may happen for files originating from the same producer.
+     * Callers should check in the local cache before to try the global cache.
+     *
+     * <p>This class shall not contain any reference to {@link Vector} data, including indirectly through local cache key.
+     * This class tests vector equality with checksum.</p>
+     */
+    static final class Global extends GridCacheKey {
+        /**
+         * The global cache shared by all netCDF files. All grids are retained by soft references.
+         */
+        private static final Cache<GridCacheKey,MathTransform> CACHE = new Cache<>(12, 0, true);
+
+        /**
+         * The algorithms tried for making the localization grids more linear.
+         * May be empty but shall not be null.
+         */
+        private final Set<Linearizer> linearizers;
+
+        /**
+         * Concatenation of the digests of the two vectors.
+         */
+        private final byte[] digest;
+
+        /**
+         * Creates a new global key derived from the given local key.
+         * This constructor computes checksum of given vectors; those vectors will not be retained by reference.
+         *
+         * @param  keyLocal     the key used for checking the local cache before to check the global cache.
+         * @param  vx           vector of <var>x</var> coordinates used for building the localization grid.
+         * @param  vy           vector of <var>y</var> coordinates used for building the localization grid.
+         * @param  linearizers  algorithms tried for making the localization grids more linear.
+         */
+        Global(final GridCacheKey keyLocal, final Vector vx, final Vector vy, final Set<Linearizer> linearizers) {
+            super(keyLocal);
+            this.linearizers = linearizers;
+            final MessageDigest md;
+            try {
+                md = MessageDigest.getInstance("MD5");
+            } catch (NoSuchAlgorithmException e) {
+                // Should not happen since every Java implementation shall support MD5, SHA-1 and SHA-256.
+                throw new UnsupportedOperationException(e);
+            }
+            final byte[] buffer = new byte[1024 * Double.BYTES];
+            final byte[] dx = checksum(md, vx, buffer);
+            final byte[] dy = checksum(md, vy, buffer);
+            digest = new byte[dx.length + dy.length];
+            System.arraycopy(dx, 0, digest, 0, dx.length);
+            System.arraycopy(dy, 0, digest, dx.length, dy.length);
+        }
+
+        /**
+         * Computes the checksum for the given vector.
+         *
+         * @param  md      the digest algorithm to use.
+         * @param  vector  the vector for which to compute a digest.
+         * @param  buffer  temporary buffer used by this method.
+         * @return the digest.
+         */
+        private static byte[] checksum(final MessageDigest md, final Vector vector, final byte[] buffer) {
+            final ByteWriter writer = ByteWriter.create(vector, buffer);
+            int n;
+            while ((n = writer.write()) > 0) {
+                md.update(buffer, 0, n);
+            }
+            return md.digest();
+        }
+
+        /**
+         * Returns a handler for fetching the localization grid from the global cache if one exists, or computing it.
+         * This method must be used with a {@code try … finally} block as below:
+         *
+         * {@preformat java
+         *     MathTransform tr;
+         *     final Cache.Handler<MathTransform> handler = key.lock();
+         *     try {
+         *         tr = handler.peek();
+         *         if (tr == null) {
+         *             // compute the localization grid.
+         *         }
+         *     } finally {
+         *         handler.putAndUnlock(tr);
+         *     }
+         * }
+         */
+        final Cache.Handler<MathTransform> lock() {
+            return CACHE.lock(this);
+        }
+
+        /**
+         * Computes a hash code for this global key.
+         * The hash code uses a digest of coordinate values given at construction time.
+         */
+        @Override public int hashCode() {
+            return super.hashCode() + linearizers.hashCode() + Arrays.hashCode(digest);
+        }
+
+        /**
+         * Computes the equality test done by parent class. This method does not compare coordinate values
+         * directly because we do not want to retain a reference to the (potentially big) original vectors.
+         * Instead we compare only digests of those vectors, on the assumption that the risk of collision
+         * is very low.
+         */
+        @Override public boolean equals(final Object other) {
+            if (super.equals(other)) {
+                final Global that = (Global) other;
+                if (linearizers.equals(that.linearizers)) {
+                    return Arrays.equals(digest, that.digest);
+                }
+            }
+            return false;
+        }
+    }
+
+    /**
+     * Returns a hash code value for this key.
+     */
+    @Override
+    public int hashCode() {
+        return 31*width + 37*height + 7*xAxis.hashCode() + yAxis.hashCode();
+    }
+
+    /**
+     * Compares the given object with this key of equality.
+     */
+    @Override
+    public boolean equals(final Object other) {
+        if (other != null && other.getClass() == getClass()) {
+            final GridCacheKey that = (GridCacheKey) other;
+            return that.width == width && that.height == height && xAxis.equals(that.xAxis) && yAxis.equals(that.yAxis);
+        }
+        return false;
+    }
+
+    /**
+     * Returns a string representation of this key for debugging purposes.
+     */
+    @Override
+    public String toString() {
+        return Strings.toString(getClass(), "width", width, "height", height);
+    }
+}
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Linearizer.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Linearizer.java
index 3917825..aff8c9e 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Linearizer.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Linearizer.java
@@ -119,7 +119,7 @@ public enum Linearizer {
      * @param  axes         coordinate system axes in CRS order.
      */
     static void applyTo(final Set<Linearizer> linearizers, final MathTransformFactory factory,
-                        final LocalizationGridBuilder grid, final Axis[] axes)
+                        final LocalizationGridBuilder grid, final Axis... axes)
     {
         int xdim = -1, ydim = -1;
         for (int i=axes.length; --i >= 0;) {
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Resources.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Resources.java
index be7780a..c0ba89c 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Resources.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Resources.java
@@ -108,6 +108,11 @@ public final class Resources extends IndexedResourceBundle {
         public static final short CanNotUseUCAR = 4;
 
         /**
+         * Computed localization grid for “{0}” in {1} seconds.
+         */
+        public static final short ComputeLocalizationGrid_2 = 22;
+
+        /**
          * Dimension “{2}” declared by attribute “{1}” is not found in the “{0}” file.
          */
         public static final short DimensionNotFound_3 = 1;
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Resources.properties b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Resources.properties
index 839956f..b84929e 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Resources.properties
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Resources.properties
@@ -28,6 +28,7 @@ CanNotRender_2                    = Can not render an image for \u201c{0}\u201d.
 CanNotSetProjectionParameter_5    = Can not set map projection parameter \u201c{1}\u200b:{2}\u201d = {3} in the \u201c{0}\u201d netCDF file. The reason is: {4}
 CanNotUseAxis_1                   = Can not use axis \u201c{0}\u201d in a grid geometry.
 CanNotUseUCAR                     = Can not use UCAR library for netCDF format. Fallback on Apache SIS implementation.
+ComputeLocalizationGrid_2         = Computed localization grid for \u201c{0}\u201d in {1} seconds.
 DimensionNotFound_3               = Dimension \u201c{2}\u201d declared by attribute \u201c{1}\u201d is not found in the \u201c{0}\u201d file.
 DuplicatedAxis_2                  = Duplicated axis \u201c{1}\u201d in a grid of netCDF file \u201c{0}\u201d.
 IllegalAttributeValue_3           = Illegal value \u201c{2}\u201d for attribute \u201c{1}\u201d in netCDF file \u201c{0}\u201d.
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Resources_fr.properties b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Resources_fr.properties
index bf8df73..f6ba835 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Resources_fr.properties
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Resources_fr.properties
@@ -33,6 +33,7 @@ CanNotRender_2                    = Ne peut pas produire une image pour \u00ab\u
 CanNotSetProjectionParameter_5    = Ne peut pas d\u00e9finir le param\u00e8tre de projection \u00ab\u202f{1}\u200b:{2}\u202f\u00bb = {3} dans le fichier netCDF \u00ab\u202f{0}\u202f\u00bb. La raison est\u2008: {4}
 CanNotUseAxis_1                   = Ne peut pas utiliser l\u2019axe \u00ab\u202f{0}\u202f\u00bb pour une g\u00e9om\u00e9trie de grille.
 CanNotUseUCAR                     = Ne peut pas utiliser la biblioth\u00e8que de l\u2019UCAR pour le format netCDF. L\u2019impl\u00e9mentation de Apache SIS sera utilis\u00e9e \u00e0 la place.
+ComputeLocalizationGrid_2         = Grille de localisation de \u00ab\u202f{0}\u202f\u00bb calcul\u00e9e en {1} secondes.
 DimensionNotFound_3               = La dimension \u00ab\u202f{2}\u202f\u00bb d\u00e9clar\u00e9e par l\u2019attribut \u00ab\u202f{1}\u202f\u00bb n\u2019a pas \u00e9t\u00e9 trouv\u00e9e dans le fichier \u00ab\u202f{0}\u202f\u00bb.
 DuplicatedAxis_2                  = Axe \u00ab\u202f{1}\u202f\u00bb dupliqu\u00e9 dans une grille du fichier netCDF \u00ab\u202f{0}\u202f\u00bb.
 IllegalAttributeValue_3           = La valeur \u00ab\u202f{2}\u202f\u00bb est ill\u00e9gale pour l\u2019attribut \u00ab\u202f{1}\u202f\u00bb dans le fichier netCDF \u00ab\u202f{0}\u202f\u00bb.
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
index be6f2e7..72e73e2 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
@@ -555,10 +555,10 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo> {
     final void setValues(final Object array) {
         Vector data = createDecimalVector(array, dataType.isUnsigned);
         /*
-         * This method is usually invoked with vector of increasing or decreasing values.  Set a tolerance threshold to the
-         * precision of gratest (in magnitude) number, provided that this precision is not larger than increment. If values
+         * This method is usually invoked with vector of increasing or decreasing values. Set a tolerance threshold to the
+         * precision of greatest (in magnitude) number, provided that this precision is not larger than increment. If values
          * are not sorted in increasing or decreasing order, the tolerance computed below will be smaller than it could be.
-         * This is okay it will cause more conservative compression (i.e. it does not increase the risk of data loss).
+         * This is okay since it will cause more conservative compression (i.e. it does not increase the risk of data loss).
          */
         double tolerance = 0;
         if (Numbers.isFloat(data.getElementType())) {
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
index e0a3b60..1a892a6 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
@@ -419,6 +419,11 @@ final class VariableWrapper extends Variable {
             final Array array = variable.read();                // May be already cached by the UCAR library.
             values = createDecimalVector(get1DJavaArray(array), variable.isUnsigned());
             values = SHARED_VECTORS.unique(values);
+            /*
+             * Do not invoke Vector.compress(…). Compressing vectors is useful only if the original array
+             * is discarded. But the UCAR library has its own cache mechanism which may keep references to
+             * the original arrays. Consequently compressing vectors may result in data being duplicated.
+             */
         }
         return values;
     }
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/ByteWriter.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/ByteWriter.java
new file mode 100644
index 0000000..d8a7ea3
--- /dev/null
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/ByteWriter.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sis.internal.storage.io;
+
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+import java.nio.DoubleBuffer;
+import java.nio.FloatBuffer;
+import java.nio.IntBuffer;
+import java.nio.LongBuffer;
+import java.nio.ShortBuffer;
+import org.apache.sis.math.Vector;
+import org.apache.sis.internal.jdk9.JDK9;
+
+
+/**
+ * Copies bytes from a source {@link Buffer} of arbitrary kind to a target {@link ByteBuffer}.
+ * This class can be used when the source {@link Buffer} subclass is unknown at compile-time.
+ * If the source buffer has a greater capacity than the target buffer, then {@link #write()}
+ * can be invoked in a loop until all data have been transferred.
+ *
+ * @author  Martin Desruisseaux (Geomatys)
+ * @version 1.0
+ * @since   1.0
+ * @module
+ */
+public abstract class ByteWriter {
+    /**
+     * For subclass constructors.
+     */
+    private ByteWriter() {
+    }
+
+    /**
+     * Creates a new writer for copying bytes from the given source vector to the given target array.
+     * This is a convenience method delegating to {@link #create(Buffer, ByteBuffer)}.
+     *
+     * @param  source  the vector from which to copy data.
+     * @param  target  the array where to copy data.
+     * @return a writer from given source to target.
+     */
+    public static ByteWriter create(final Vector source, final byte[] target) {
+        return create(source.buffer().orElseGet(() -> DoubleBuffer.wrap(source.doubleValues())), ByteBuffer.wrap(target));
+    }
+
+    /**
+     * Creates a new writer for copying bytes from the given source to the given target buffers.
+     * Data will be read from the current position of source buffer up to that buffer <em>limit</em>.
+     * Data will be written starting at position 0 of target buffer up to that buffer <em>capacity</em>.
+     * The position and limit of target buffer are ignored.
+     * The position and limit of both buffers may be modified by this method.
+     *
+     * @param  source  the buffer from which to copy data.
+     * @param  target  the buffer where to copy data.
+     * @return a writer from given source to target.
+     */
+    public static ByteWriter create(Buffer source, final ByteBuffer target) {
+        if (source.limit() != source.capacity()) {
+            source = JDK9.slice(source);
+        }
+        if (source instanceof DoubleBuffer) return new Doubles ((DoubleBuffer) source, target);
+        if (source instanceof  FloatBuffer) return new Floats  ( (FloatBuffer) source, target);
+        if (source instanceof   LongBuffer) return new Longs   (  (LongBuffer) source, target);
+        if (source instanceof    IntBuffer) return new Integers(   (IntBuffer) source, target);
+        if (source instanceof  ShortBuffer) return new Shorts  ( (ShortBuffer) source, target);
+        if (source instanceof   ByteBuffer) return new Bytes   (  (ByteBuffer) source, target);
+        throw new IllegalArgumentException();
+    }
+
+    /**
+     * Copies bytes from the source buffer to the target buffer. The target buffer position is not overwritten;
+     * it is caller responsibility to update it (if desired) from the value returned by this method.
+     *
+     * @return the number of bytes copied, or 0 if done.
+     */
+    public abstract int write();
+
+    /**
+     * Prepares the given source and target buffers to a new transfer.
+     */
+    private static void reset(final Buffer source, final Buffer target) {
+        target.clear();
+        source.limit(Math.min(source.capacity(), source.position() + target.capacity()));
+    }
+
+    /** Writer for {@code double} values. */
+    private static final class Doubles extends ByteWriter {
+        /** The buffers between which to transfer data. */
+        private final DoubleBuffer source, target;
+
+        /** Creates a new writer for the given source. */
+        Doubles(final DoubleBuffer source, final ByteBuffer target) {
+            this.source = source;
+            this.target = target.asDoubleBuffer();
+        }
+
+        /** Write bytes. */
+        @Override public int write() {
+            reset(source, target);
+            target.put(source);
+            return target.position() * Double.BYTES;
+        }
+    }
+
+    /** Writer for {@code float} values. */
+    private static final class Floats extends ByteWriter {
+        /** The buffers between which to transfer data. */
+        private final FloatBuffer source, target;
+
+        /** Creates a new writer for the given source. */
+        Floats(final FloatBuffer source, final ByteBuffer target) {
+            this.source = source;
+            this.target = target.asFloatBuffer();
+        }
+
+        /** Write bytes. */
+        @Override public int write() {
+            reset(source, target);
+            target.put(source);
+            return target.position() * Float.BYTES;
+        }
+    }
+
+    /** Writer for {@code long} values. */
+    private static final class Longs extends ByteWriter {
+        /** The buffers between which to transfer data. */
+        private final LongBuffer source, target;
+
+        /** Creates a new writer for the given source. */
+        Longs(final LongBuffer source, final ByteBuffer target) {
+            this.source = source;
+            this.target = target.asLongBuffer();
+        }
+
+        /** Write bytes. */
+        @Override public int write() {
+            reset(source, target);
+            target.put(source);
+            return target.position() * Long.BYTES;
+        }
+    }
+
+    /** Writer for {@code int} values. */
+    private static final class Integers extends ByteWriter {
+        /** The buffers between which to transfer data. */
+        private final IntBuffer source, target;
+
+        /** Creates a new writer for the given source. */
+        Integers(final IntBuffer source, final ByteBuffer target) {
+            this.source = source;
+            this.target = target.asIntBuffer();
+        }
+
+        /** Write bytes. */
+        @Override public int write() {
+            reset(source, target);
+            target.put(source);
+            return target.position() * Integer.BYTES;
+        }
+    }
+
+    /** Writer for {@code short} values. */
+    private static final class Shorts extends ByteWriter {
+        /** The buffers between which to transfer data. */
+        private final ShortBuffer source, target;
+
+        /** Creates a new writer for the given source. */
+        Shorts(final ShortBuffer source, final ByteBuffer target) {
+            this.source = source;
+            this.target = target.asShortBuffer();
+        }
+
+        /** Write bytes. */
+        @Override public int write() {
+            reset(source, target);
+            target.put(source);
+            return target.position() * Short.BYTES;
+        }
+    }
+
+    /** Writer for {@code byte} values. */
+    private static final class Bytes extends ByteWriter {
+        /** The buffers between which to transfer data. */
+        private final ByteBuffer source, target;
+
+        /** Creates a new writer for the given source. */
+        Bytes(final ByteBuffer source, final ByteBuffer target) {
+            this.source = source;
+            this.target = target;
+        }
+
+        /** Write bytes. */
+        @Override public int write() {
+            reset(source, target);
+            target.put(source);
+            return target.position() * Byte.BYTES;
+        }
+    }
+}
diff --git a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/package-info.java b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/package-info.java
index e0d90c3..79e0b07 100644
--- a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/package-info.java
+++ b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/io/package-info.java
@@ -24,7 +24,7 @@
  * may change in incompatible ways in any future version without notice.
  *
  * @author  Martin Desruisseaux (Geomatys)
- * @version 0.8
+ * @version 1.0
  * @since   0.3
  * @module
  */


Mime
View raw message