sis-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From desruisse...@apache.org
Subject [sis] 02/02: Partial decoding of HYCOM netCDF files.
Date Thu, 22 Nov 2018 19:57:09 GMT
This is an automated email from the ASF dual-hosted git repository.

desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git

commit 367df41cd5dfe7e156402ceb147442cb62f12e26
Author: Martin Desruisseaux <martin.desruisseaux@geomatys.com>
AuthorDate: Thu Nov 22 20:56:32 2018 +0100

    Partial decoding of HYCOM netCDF files.
---
 .../org/apache/sis/coverage/grid/GridGeometry.java |   2 +-
 .../java/org/apache/sis/internal/netcdf/Axis.java  |  48 ++++----
 .../java/org/apache/sis/internal/netcdf/Grid.java  |   2 +
 .../org/apache/sis/internal/netcdf/Variable.java   |  40 ++++++-
 .../sis/internal/netcdf/impl/ChannelDecoder.java   |  83 ++++++++-----
 .../apache/sis/internal/netcdf/impl/GridInfo.java  |  31 ++++-
 .../org/apache/sis/internal/netcdf/impl/HYCOM.java | 129 +++++++++++++++++++++
 .../sis/internal/netcdf/impl/VariableInfo.java     |  38 +++++-
 .../sis/internal/netcdf/ucar/VariableWrapper.java  |  12 +-
 .../sis/storage/netcdf/NetcdfStoreProvider.java    |   4 +-
 10 files changed, 325 insertions(+), 64 deletions(-)

diff --git a/core/sis-raster/src/main/java/org/apache/sis/coverage/grid/GridGeometry.java
b/core/sis-raster/src/main/java/org/apache/sis/coverage/grid/GridGeometry.java
index 04b34cc..e19eb8f 100644
--- a/core/sis-raster/src/main/java/org/apache/sis/coverage/grid/GridGeometry.java
+++ b/core/sis-raster/src/main/java/org/apache/sis/coverage/grid/GridGeometry.java
@@ -980,7 +980,7 @@ public class GridGeometry implements Serializable {
                     while (nonLinearDimensions != 0) {
                         final int i = Long.numberOfTrailingZeros(nonLinearDimensions);
                         nonLinearDimensions &= ~(1L << i);
-                        buffer.append(separator).append(cs != null ? cs.getAxis(i).getName()
: String.valueOf(i));
+                        buffer.append(separator).append(cs != null ? cs.getAxis(i).getName().getCode()
: String.valueOf(i));
                         separator = ", ";
                     }
                     buffer.append(lineSeparator);
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java
index a1e533d..bae678d 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Axis.java
@@ -356,28 +356,32 @@ public final class Axis extends NamedElement {
          */
         if (sourceDimensions.length == 2) {
             Vector data = coordinates.read();
-            final int[] repetitions = data.repetitions();           // Detects repetitions
as illustrated above.
-            for (int i=0; i<sourceDimensions.length; i++) {
-                final int srcDim = srcEnd - sourceDimensions[i];    // "Natural" order is
reverse of netCDF order.
-                final int length = sourceSizes[i];
-                int step = 1;
-                for (int j=0; j<sourceDimensions.length; j++) {
-                    int previous = srcEnd - sourceDimensions[j];
-                    if (previous < srcDim) step *= sourceSizes[j];
-                }
-                final boolean condition;
-                switch (srcDim) {
-                    case 0:  condition = repetitions.length > 1 && (repetitions[1]
% length) == 0; break;
-                    case 1:  condition = repetitions.length > 0 && (repetitions[0]
% step)   == 0; break;
-                    default: throw new AssertionError();        // I don't know yet how to
generalize to n dimensions.
-                }
-                if (condition) {                                // Repetition length shall
be grid size (or a multiple).
-                    data = data.subSampling(0, step, length);
-                    if (coordinates.trySetTransform(gridToCRS, srcDim, tgtDim, data)) {
-                        return true;
-                    } else {
-                        nonLinears.add(MathTransforms.interpolate(null, data.doubleValues()));
-                        return false;
+            if (!coordinates.readTriesToCompress() || data.getClass().getSimpleName().equals("RepeatedVector"))
{
+                final int[] repetitions = data.repetitions();       // Detects repetitions
as illustrated above.
+                if (repetitions.length != 0) {
+                    for (int i=0; i<sourceDimensions.length; i++) {
+                        final int srcDim = srcEnd - sourceDimensions[i];    // "Natural"
order is reverse of netCDF order.
+                        final int length = sourceSizes[i];
+                        int step = 1;
+                        for (int j=0; j<sourceDimensions.length; j++) {
+                            int previous = srcEnd - sourceDimensions[j];
+                            if (previous < srcDim) step *= sourceSizes[j];
+                        }
+                        final boolean condition;
+                        switch (srcDim) {
+                            case 0:  condition = repetitions.length > 1 && (repetitions[1]
% length) == 0; break;
+                            case 1:  condition =                           (repetitions[0]
% step)   == 0; break;
+                            default: throw new AssertionError();        // I don't know yet
how to generalize to n dimensions.
+                        }
+                        if (condition) {                                // Repetition length
shall be grid size (or a multiple).
+                            data = data.subSampling(0, step, length);
+                            if (coordinates.trySetTransform(gridToCRS, srcDim, tgtDim, data))
{
+                                return true;
+                            } else {
+                                nonLinears.add(MathTransforms.interpolate(null, data.doubleValues()));
+                                return false;
+                            }
+                        }
                     }
                 }
             }
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Grid.java b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Grid.java
index e8a4c50..c8e1d2f 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Grid.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Grid.java
@@ -53,6 +53,8 @@ import org.apache.sis.util.NullArgumentException;
 public abstract class Grid extends NamedElement {
     /**
      * The axes, created when first needed.
+     * The ordering of axes is based on the order in which dimensions are declared for variables
using this grid.
+     * This is not necessarily the same order than the order in which variables are listed
in the netCDF file.
      *
      * @see #getAxes()
      */
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
index 412d345..8d8ca0d 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/Variable.java
@@ -42,8 +42,11 @@ import org.apache.sis.util.resources.Errors;
 public abstract class Variable extends NamedElement {
     /**
      * The pattern to use for parsing temporal units of the form "days since 1970-01-01 00:00:00".
+     *
+     * @see #parseUnit(String)
+     * @see Decoder#numberToDate(String, Number[])
      */
-    protected static final Pattern TIME_PATTERN = Pattern.compile("(.+)\\Wsince\\W(.+)",
Pattern.CASE_INSENSITIVE);
+    public static final Pattern TIME_UNIT_PATTERN = Pattern.compile("(.+)\\Wsince\\W(.+)",
Pattern.CASE_INSENSITIVE);
 
     /**
      * Minimal number of dimension for accepting a variable as a coverage variable.
@@ -54,18 +57,23 @@ public abstract class Variable extends NamedElement {
      * The unit of measurement, parsed from {@link #getUnitsString()} when first needed.
      * We do not try to parse the unit at construction time because this variable may be
      * never requested by the user.
+     *
+     * @see #getUnit()
      */
     private Unit<?> unit;
 
     /**
      * If the unit is a temporal unit of the form "days since 1970-01-01 00:00:00", the epoch.
-     * Otherwise {@code null}. This value can be set
+     * Otherwise {@code null}. This value can be set by subclasses as a side-effect of their
+     * {@link #parseUnit(String)} method implementation.
      */
     protected Instant epoch;
 
     /**
      * Whether an attempt to parse the unit has already be done. This is used for avoiding
      * to report the same failure many times when {@link #unit} stay null.
+     *
+     * @see #getUnit()
      */
     private boolean unitParsed;
 
@@ -129,6 +137,26 @@ public abstract class Variable extends NamedElement {
     protected abstract Unit<?> parseUnit(String symbols) throws Exception;
 
     /**
+     * Sets the unit of measurement and the epoch to the same value than the given variable.
+     * This method is not used in CF-compliant files; it is reserved for the handling of
some
+     * particular conventions, for example HYCOM.
+     *
+     * @param  other      the variable from which to copy unit and epoch, or {@code null}
if none.
+     * @param  overwrite  if non-null, set to the given unit instead than the unit of {@code
other}.
+     * @return the epoch (may be {@code null}).
+     */
+    public final Instant setUnit(final Variable other, Unit<?> overwrite) {
+        if (other != null) {
+            unit  = other.getUnit();        // May compute the epoch as a side effect.
+            epoch = other.epoch;
+        }
+        if (overwrite != null) {
+            unit = overwrite;
+        }
+        return epoch;
+    }
+
+    /**
      * Returns the unit of measurement for this variable, or {@code null} if unknown.
      * This method parse the units from {@link #getUnitsString()} when first needed
      * and sets {@link #epoch} as a side-effect if the unit is temporal.
@@ -291,6 +319,14 @@ public abstract class Variable extends NamedElement {
     public abstract String getAttributeString(final String attributeName);
 
     /**
+     * Whether {@link #read()} invoked {@link Vector#compress(double)} on the returned vector.
+     * This information is used for avoiding to do twice some potentially costly operations.
+     *
+     * @return whether {@link #read()} invokes {@link Vector#compress(double)}.
+     */
+    protected abstract boolean readTriesToCompress();
+
+    /**
      * Reads all the data for this variable and returns them as an array of a Java primitive
type.
      * Multi-dimensional variables are flattened as a one-dimensional array (wrapped in a
vector).
      * Example:
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
index c946987..033fcd0 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ChannelDecoder.java
@@ -25,11 +25,10 @@ import java.util.LinkedHashSet;
 import java.util.LinkedHashMap;
 import java.util.IdentityHashMap;
 import java.util.List;
-import java.util.Arrays;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.Locale;
-import java.util.regex.Pattern;
+import java.util.regex.Matcher;
 import java.time.DateTimeException;
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -111,14 +110,6 @@ public final class ChannelDecoder extends Decoder {
      */
     static final Locale NAME_LOCALE = Locale.US;
 
-    /**
-     * The pattern to use for separating the component of a time unit.
-     * An example of time unit is <cite>"days since 1970-01-01T00:00:00Z"</cite>.
-     *
-     * @see #numberToDate(String, Number[])
-     */
-    private static final Pattern TIME_UNIT_PATTERN = Pattern.compile("\\s+since\\s+", Pattern.CASE_INSENSITIVE);
-
     /*
      * NOTE: the names of the static constants below this point match the names used in the
Backus-Naur Form (BNF)
      *       definitions in the netCDF Classic and 64-bit Offset Format (1.0) specification
(link in class javdoc),
@@ -626,6 +617,18 @@ public final class ChannelDecoder extends Decoder {
         return variables;
     }
 
+    /**
+     * Checks and potentially modifies the content of this dataset for conventions other
than CF-conventions.
+     * This method should be invoked after construction for handling the particularities
of some datasets
+     * (HYCOM, …).
+     *
+     * @throws IOException if an error occurred while reading the channel.
+     * @throws DataStoreContentException if an error occurred while interpreting the netCDF
file content.
+     */
+    public final void applyOtherConventions() throws IOException, DataStoreContentException
{
+        HYCOM.convert(this, variables);
+    }
+
 
 
     // --------------------------------------------------------------------------------------------
@@ -791,10 +794,10 @@ public final class ChannelDecoder extends Decoder {
     @Override
     public Date[] numberToDate(final String symbol, final Number... values) {
         final Date[] dates = new Date[values.length];
-        final String[] parts = TIME_UNIT_PATTERN.split(symbol);
-        if (parts.length == 2) try {
-            final UnitConverter converter = Units.valueOf(parts[0]).getConverterToAny(Units.MILLISECOND);
-            final long epoch = StandardDateFormat.toDate(StandardDateFormat.FORMAT.parse(parts[1])).getTime();
+        final Matcher parts = Variable.TIME_UNIT_PATTERN.matcher(symbol);
+        if (parts.matches()) try {
+            final UnitConverter converter = Units.valueOf(parts.group(1)).getConverterToAny(Units.MILLISECOND);
+            final long epoch = StandardDateFormat.toDate(StandardDateFormat.FORMAT.parse(parts.group(2))).getTime();
             for (int i=0; i<values.length; i++) {
                 final Number value = values[i];
                 if (value != null) {
@@ -851,7 +854,7 @@ public final class ChannelDecoder extends Decoder {
              * First, find all variables which are used as coordinate system axis. The keys
in the map are
              * the grid dimensions which are the domain of the variable (i.e. the sources
of the conversion
              * from grid coordinates to CRS coordinates). For each key there is usually only
one value, but
-             * we try to make this code robust to unusual netCDF files.
+             * more complicated netCDF files (e.g. using two-dimensional localisation grids)
also exist.
              */
             final Map<Dimension, List<VariableInfo>> dimToAxes = new IdentityHashMap<>();
             for (final VariableInfo variable : variables) {
@@ -862,25 +865,34 @@ public final class ChannelDecoder extends Decoder {
                 }
             }
             /*
-             * For each variables, get the list of all axes associated to their dimensions.
The association
-             * is given by the above 'dimToVars' map. More than one variable may have the
same dimensions,
-             * and consequently the same axes, so we will remember the previously created
instances in order
-             * to share them.
+             * For each variable, get its list of axes. More than one variable may have the
same list of axes,
+             * so we remember the previously created instances in order to share the grid
geometry instances.
              */
             final Set<VariableInfo> axes = new LinkedHashSet<>(4);
-            final Map<List<Dimension>, GridInfo> dimsToGG = new LinkedHashMap<>();
+            final Map<GridInfo,GridInfo> shared = new LinkedHashMap<>();
 nextVar:    for (final VariableInfo variable : variables) {
                 if (variable.isCoordinateSystemAxis() || variable.dimensions.length == 0)
{
                     continue;
                 }
-                final List<Dimension> dimensions = Arrays.asList(variable.dimensions);
-                GridInfo gridGeometry = dimsToGG.get(dimensions);
-                if (gridGeometry == null) {
-                    /*
-                     * Found a new list of dimensions for which no axes have been created
yet.
-                     * If and only if we can find all axes, then create the GridGeometryInfo.
-                     * This is a "all or nothing" operation.
-                     */
+                /*
+                 * The axes can be inferred in two ways: if the variable contains a "coordinates"
attribute,
+                 * that attribute lists explicitly the variables to use as axes. Otherwise
we have to infer
+                 * the axes from the variable dimensions, using the 'dimToVars' map computed
at the beginning
+                 * of this method. If and only if we can find all axes, we create the GridGeometryInfo.
+                 * This is a "all or nothing" operation.
+                 */
+                final CharSequence[] coordinates = variable.getCoordinateVariables();
+                if (coordinates.length != 0) {
+                    for (int i=coordinates.length; --i >= 0;) {
+                        final VariableInfo axis = findVariable(coordinates[i].toString());
+                        if (axis == null) {
+                            axes.clear();
+                            break;
+                        }
+                        axes.add(axis);
+                    }
+                }
+                if (axes.isEmpty()) {
                     for (final Dimension dimension : variable.dimensions) {
                         final List<VariableInfo> axis = dimToAxes.get(dimension); 
     // Should have only 1 element.
                         if (axis == null) {
@@ -889,13 +901,20 @@ nextVar:    for (final VariableInfo variable : variables) {
                         }
                         axes.addAll(axis);
                     }
-                    gridGeometry = new GridInfo(variable.dimensions, axes.toArray(new VariableInfo[axes.size()]));
-                    dimsToGG.put(dimensions, gridGeometry);
-                    axes.clear();
+                }
+                /*
+                 * Creates the grid geometry using the given domain and range,
+                 * reusing existing instance if one exists.
+                 */
+                GridInfo gridGeometry = new GridInfo(variable.dimensions, axes.toArray(new
VariableInfo[axes.size()]));
+                GridInfo existing = shared.putIfAbsent(gridGeometry, gridGeometry);
+                if (existing != null) {
+                    gridGeometry = existing;
                 }
                 variable.gridGeometry = gridGeometry;
+                axes.clear();
             }
-            gridGeometries = dimsToGG.values().toArray(new Grid[dimsToGG.size()]);
+            gridGeometries = shared.values().toArray(new Grid[shared.size()]);
         }
         return gridGeometries;
     }
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/GridInfo.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/GridInfo.java
index 452a19a..033c74a 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/GridInfo.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/GridInfo.java
@@ -17,6 +17,7 @@
 package org.apache.sis.internal.netcdf.impl;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Locale;
 import java.util.Map;
 import java.util.HashMap;
@@ -29,6 +30,7 @@ import org.apache.sis.internal.netcdf.Grid;
 import org.apache.sis.internal.netcdf.Resources;
 import org.apache.sis.storage.DataStoreContentException;
 import org.apache.sis.storage.DataStoreException;
+import org.apache.sis.measure.Units;
 import ucar.nc2.constants.CF;
 
 
@@ -188,7 +190,7 @@ final class GridInfo extends Grid {
         /*
          * Process the variables in the order the appear in the sequence of bytes that make
the netCDF files.
          * This is often the same order than the indices, but not necessarily. The intent
is to reduce the
-         * amount of disk seek operations.
+         * amount of disk seek operations. Data loading may happen in this method through
Axis constructor.
          */
         final SortedMap<VariableInfo,Integer> variables = new TreeMap<>();
         for (int i=0; i<range.length; i++) {
@@ -208,6 +210,11 @@ final class GridInfo extends Grid {
             char abbreviation = getAxisType(axis.getAxisType());
             if (abbreviation == 0) {
                 abbreviation = getAxisType(axis.getName());
+                if (abbreviation == 0) {
+                    if (Units.isTemporal(axis.getUnit())) {
+                        abbreviation = 't';
+                    }
+                }
             }
             /*
              * Get the grid dimensions (part of the "domain" in UCAR terminology) used for
computing
@@ -245,4 +252,26 @@ final class GridInfo extends Grid {
         final int n = v.dimensions[0].length;
         return v.read().doubleValue(j + n*i);
     }
+
+    /**
+     * Returns a hash code for this grid. A map of {@code GridInfo} is used by
+     * {@link ChannelDecoder#getGridGeometries()} for sharing existing instances.
+     */
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(domain) ^ Arrays.hashCode(range);
+    }
+
+    /**
+     * Compares the grid with the given object for equality.
+     */
+    @Override
+    public boolean equals(final Object other) {
+        if (other instanceof GridInfo) {
+            final GridInfo that = (GridInfo) other;
+            return Arrays.equals(domain, that.domain) &&
+                   Arrays.equals(range,  that.range);
+        }
+        return false;
+    }
 }
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/HYCOM.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/HYCOM.java
new file mode 100644
index 0000000..3e8ec40
--- /dev/null
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/HYCOM.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sis.internal.netcdf.impl;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.GregorianCalendar;
+import java.util.TimeZone;
+import org.apache.sis.math.Vector;
+import org.apache.sis.measure.Units;
+import org.apache.sis.internal.util.StandardDateFormat;
+import org.apache.sis.storage.DataStoreContentException;
+
+
+/**
+ * Handles particularity of HYCOM format. It is not yet clear whether those particularities
are used elsewhere or not.
+ * We handle them in a separated class for now and may refactor later in a more general mechanism
for providing extensions.
+ *
+ * @author  Martin Desruisseaux (Geomatys)
+ * @version 1.0
+ *
+ * @see <a href="https://issues.apache.org/jira/browse/SIS-315">SIS-315</a>
+ *
+ * @since 1.0
+ * @module
+ */
+final class HYCOM {
+    /**
+     * The pattern to use for identifying temporal units of the form "day as %Y%m%d.%f".
+     * "%Y" is year formatted as at least four digits, "%m" is month formatted as two digits,
+     * and "%d" is day of month formatted as two digits.
+     *
+     * Example: 20181017.0000 for 2018-10-17.
+     */
+    private static final Pattern DATE_PATTERN = Pattern.compile("days?\\s+as\\s+(?-i)%Y%m%d.*",
Pattern.CASE_INSENSITIVE);
+
+    /**
+     * Do not allow instantiation of this class.
+     */
+    private HYCOM() {
+    }
+
+    /**
+     * If any variable uses the "day as %Y%m%d.%f" pseudo-units, converts to a number of
days since the epoch.
+     * The epoch is taken from the unit of the dimension. Example of netCDF file header:
+     *
+     * {@preformat text
+     *     dimensions:
+     *         MT = UNLIMITED ; // (1 currently)
+     *         Y = 3298 ;
+     *         X = 4500 ;
+     *     variables:
+     *         double MT(MT) ;
+     *             MT:long_name = "time" ;
+     *             MT:units = "days since 1900-12-31 00:00:00" ;
+     *             MT:calendar = "standard" ;
+     *             MT:axis = "T" ;
+     *         double Date(MT) ;
+     *             Date:long_name = "date" ;
+     *             Date:units = "day as %Y%m%d.%f" ;
+     *             Date:C_format = "%13.4f" ;
+     *             Date:FORTRAN_format = "(f13.4)" ;
+     *     data:
+     *         MT = 43024 ;
+     *         Date = 20181017.0000 ;
+     * }
+     *
+     * In this example, the real units of {@code Date(MT)} will be taken from {@code MT(MT)},
which is
+     * "days since 1900-12-31 00:00:00".
+     */
+    static void convert(final ChannelDecoder decoder, final VariableInfo[] variables) throws
IOException, DataStoreContentException {
+        Matcher matcher = null;
+        for (final VariableInfo variable : variables) {
+            if (variable.dimensions.length == 1) {
+                final String units = variable.getUnitsString();
+                if (units != null) {
+                    if (matcher == null) {
+                        matcher = DATE_PATTERN.matcher(units);
+                    } else {
+                        matcher.reset(units);
+                    }
+                    if (matcher.matches()) {
+                        Instant epoch = variable.setUnit(decoder.findVariable(variable.dimensions[0].name),
Units.DAY);
+                        if (epoch == null) {
+                            epoch = Instant.EPOCH;
+                        }
+                        final long origin = epoch.toEpochMilli();
+                        /*
+                         * Convert all dates into numbers of days since the epoch.
+                         */
+                        Vector values = variable.read();
+                        final double[] times = new double[values.size()];
+                        final GregorianCalendar calendar = new GregorianCalendar(TimeZone.getTimeZone(StandardDateFormat.UTC),
Locale.US);
+                        calendar.clear();
+                        for (int i=0; i<times.length; i++) {
+                            double time = values.doubleValue(i);                        
   // Date encoded as a double (e.g. 20181017)
+                            long date = (long) time;                                    
   // Round toward zero.
+                            time -= date;                                               
   // Fractional part of the day.
+                            int day   = (int) (date % 100); date /= 100;
+                            int month = (int) (date % 100); date /= 100;
+                            calendar.set(Math.toIntExact(date), month - 1, day, 0, 0, 0);
+                            date = calendar.getTimeInMillis() - origin;                 
   // Milliseconds since epoch.
+                            time += date / (double) StandardDateFormat.MILLISECONDS_PER_DAY;
+                            times[i] = time;
+                        }
+                        variable.setValues(times);
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
index b2d6561..38054cc 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/VariableInfo.java
@@ -300,7 +300,7 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo>
{
         boolean isUnknown    = false;           // True if 'total' is actually unknown.
         for (final VariableInfo variable : variables) {
             // Opportunistically store names of all axes listed in "coordinates" attributes
of all variables.
-            referencedAsAxis.addAll(Arrays.asList(CharSequences.split(variable.getAttributeString(CF.COORDINATES),
' ')));
+            referencedAsAxis.addAll(Arrays.asList(variable.getCoordinateVariables()));
             if (variable.isUnlimited()) {
                 final long paddedSize = variable.paddedSize();
                 unlimited[count++] = variable;
@@ -393,7 +393,7 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo>
{
      */
     @Override
     protected Unit<?> parseUnit(String symbols) {
-        final Matcher parts = TIME_PATTERN.matcher(symbols);
+        final Matcher parts = TIME_UNIT_PATTERN.matcher(symbols);
         if (parts.matches()) {
             /*
              * If we enter in this block, the unit is of the form "days since 1970-01-01
00:00:00".
@@ -452,6 +452,13 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo>
{
     }
 
     /**
+     * Returns the names of variables to use as axes for this variable, or an empty array
if none.
+     */
+    final CharSequence[] getCoordinateVariables() {
+        return CharSequences.split(getAttributeString(CF.COORDINATES), ' ');
+    }
+
+    /**
      * Returns the grid geometry for this variable, or {@code null} if this variable is not
a data cube.
      * The grid geometries are opportunistically cached in {@code VariableInfo} instances
after they have
      * been computed by {@link ChannelDecoder#getGridGeometries()}.
@@ -607,6 +614,26 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo>
{
     }
 
     /**
+     * Whether {@link #read()} invokes {@link Vector#compress(double)} on the returned vector.
+     *
+     * @return {@code true}.
+     */
+    @Override
+    protected boolean readTriesToCompress() {
+        return true;
+    }
+
+    /**
+     * Sets the values in this variable. The values are normally read from the netCDF file
by the {@link #read()} method,
+     * but this {@code setValues(Object)} method may also be invoked if we want to overwrite
those values.
+     *
+     * @param  array  the values as an array of primitive type (for example {@code float[]}.
+     */
+    final void setValues(final Object array) {
+        values = createDecimalVector(array, dataType.isUnsigned).compress(0);
+    }
+
+    /**
      * Reads all the data for this variable and returns them as an array of a Java primitive
type.
      * Multi-dimensional variables are flattened as a one-dimensional array (wrapped in a
vector).
      * The vector is cached and returned as-is in all future invocation of this method.
@@ -630,7 +657,7 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo>
{
             }
             final Region region = new Region(upper, lower, upper, subsampling);
             applyUnlimitedDimensionStride(region);
-            values = createDecimalVector(reader.read(region), dataType.isUnsigned).compress(0);
+            setValues(reader.read(region));
         }
         return values;
     }
@@ -664,10 +691,13 @@ final class VariableInfo extends Variable implements Comparable<VariableInfo>
{
      * @throws ArithmeticException if the size of the region to read exceeds {@link Integer#MAX_VALUE},
or other overflow occurs.
      */
     @Override
-    public Vector read(int[] areaLower, int[] areaUpper, int[] subsampling) throws IOException,
DataStoreContentException {
+    public Vector read(int[] areaLower, int[] areaUpper, int[] subsampling) throws IOException,
DataStoreException {
         if (reader == null) {
             throw new DataStoreContentException(unknownType());
         }
+        if (values != null) {
+            throw new DataStoreException();     // TODO: create a view.
+        }
         /*
          * NetCDF sorts datas in reverse dimension order. Example:
          *
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
index 6bc71dc..0f09b30 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/VariableWrapper.java
@@ -131,7 +131,7 @@ final class VariableWrapper extends Variable {
      */
     @Override
     protected Unit<?> parseUnit(final String symbols) throws Exception {
-        if (TIME_PATTERN.matcher(symbols).matches()) {
+        if (TIME_UNIT_PATTERN.matcher(symbols).matches()) {
             /*
              * UCAR library has two methods for getting epoch: getDate() and getDateOrigin().
              * The former adds to the origin the number that may appear before the unit,
for example
@@ -317,6 +317,16 @@ final class VariableWrapper extends Variable {
     }
 
     /**
+     * Whether {@link #read()} invokes {@link Vector#compress(double)} on the returned vector.
+     *
+     * @return {@code false}.
+     */
+    @Override
+    protected boolean readTriesToCompress() {
+        return false;
+    }
+
+    /**
      * Reads all the data for this variable and returns them as an array of a Java primitive
type.
      * Multi-dimensional variables are flattened as a one-dimensional array (wrapped in a
vector).
      * This method may cache the returned vector, at UCAR library choice.
diff --git a/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStoreProvider.java
b/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStoreProvider.java
index e456e86..8eb744a 100644
--- a/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStoreProvider.java
+++ b/storage/sis-netcdf/src/main/java/org/apache/sis/storage/netcdf/NetcdfStoreProvider.java
@@ -273,7 +273,9 @@ public class NetcdfStoreProvider extends DataStoreProvider {
         Object keepOpen;
         final ChannelDataInput input = connector.getStorageAs(ChannelDataInput.class);
         if (input != null) try {
-            decoder = new ChannelDecoder(input, connector.getOption(OptionKey.ENCODING),
geomlib, listeners);
+            final ChannelDecoder cd = new ChannelDecoder(input, connector.getOption(OptionKey.ENCODING),
geomlib, listeners);
+            cd.applyOtherConventions();
+            decoder = cd;
             keepOpen = input;
         } catch (DataStoreException | ArithmeticException e) {
             final String path = connector.getStorageAs(String.class);


Mime
View raw message