sis-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From desruisse...@apache.org
Subject svn commit: r1800217 - in /sis/branches/JDK8/storage: sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/ sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/ sis-storage/src/m...
Date Thu, 29 Jun 2017 00:21:33 GMT
Author: desruisseaux
Date: Thu Jun 29 00:21:33 2017
New Revision: 1800217

URL: http://svn.apache.org/viewvc?rev=1800217&view=rev
Log:
Partial support of moving features in NetCDF.

Modified:
    sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/DiscreteSampling.java
    sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/FeaturesInfo.java
    sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/FeaturesWrapper.java
    sis/branches/JDK8/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/csv/Store.java

Modified: sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/DiscreteSampling.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/DiscreteSampling.java?rev=1800217&r1=1800216&r2=1800217&view=diff
==============================================================================
--- sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/DiscreteSampling.java
[UTF-8] (original)
+++ sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/DiscreteSampling.java
[UTF-8] Thu Jun 29 00:21:33 2017
@@ -16,6 +16,10 @@
  */
 package org.apache.sis.internal.netcdf;
 
+// Branch-dependent imports
+import java.util.stream.Stream;
+import org.opengis.feature.Feature;
+
 
 /**
  * Returns the features encoded in the NetCDF files when they are encoded as discrete sampling.
@@ -34,4 +38,11 @@ public abstract class DiscreteSampling {
      */
     protected DiscreteSampling() {
     }
+
+    /**
+     * Returns the stream of features.
+     *
+     * @return the stream of features.
+     */
+    public abstract Stream<Feature> features();
 }

Modified: sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/FeaturesInfo.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/FeaturesInfo.java?rev=1800217&r1=1800216&r2=1800217&view=diff
==============================================================================
--- sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/FeaturesInfo.java
[UTF-8] (original)
+++ sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/impl/FeaturesInfo.java
[UTF-8] Thu Jun 29 00:21:33 2017
@@ -16,16 +16,35 @@
  */
 package org.apache.sis.internal.netcdf.impl;
 
-import java.util.ArrayList;
+import java.util.Map;
 import java.util.List;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.io.IOException;
 import org.apache.sis.math.Vector;
 import org.apache.sis.internal.netcdf.DataType;
 import org.apache.sis.internal.netcdf.DiscreteSampling;
 import org.apache.sis.internal.netcdf.Resources;
+import org.apache.sis.internal.feature.Geometries;
+import org.apache.sis.internal.feature.MovingFeature;
 import org.apache.sis.storage.DataStoreException;
+import org.apache.sis.feature.DefaultFeatureType;
+import org.apache.sis.feature.DefaultAttributeType;
+import org.apache.sis.util.collection.BackingStoreException;
 import ucar.nc2.constants.CF;
 
+// Branch-dependent imports
+import java.util.Spliterator;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+import java.util.function.Consumer;
+import org.opengis.feature.Feature;
+import org.opengis.feature.FeatureType;
+import org.opengis.feature.PropertyType;
+import org.opengis.feature.AttributeType;
+
 
 /**
  * Implementations of the discrete sampling features decoder. This implementation shall be
able to decode at least the
@@ -44,18 +63,85 @@ final class FeaturesInfo extends Discret
 
     /**
      * The moving feature identifiers ("mfIdRef").
+     * The amount of identifiers shall be the same than the length of the {@link #counts}
vector.
      */
     private final VariableInfo identifiers;
 
     /**
+     * The variable that contains time.
+     */
+    private final VariableInfo time;
+
+    /**
+     * The variable that contains <var>x</var> and <var>y</var> ordinate
values (typically longitudes and latitudes).
+     * All variables in this array shall have the same length, and that length shall be the
same than {@link #time}.
+     */
+    private final VariableInfo[] coordinates;
+
+    /**
+     * Any custom properties.
+     */
+    private final VariableInfo[] properties;
+
+    /**
+     * The type of all features to be read by this {@code FeaturesInfo}.
+     */
+    private final FeatureType type;
+
+    private final Geometries<?> factory = Geometries.implementation(null);        
 // TODO: shall be given by the store.
+
+    /**
      * Creates a new discrete sampling parser for features identified by the given variable.
      *
      * @param  counts       the count of instances per feature.
      * @param  identifiers  the feature identifiers.
      */
-    private FeaturesInfo(final Vector counts, final VariableInfo identifiers) {
+    @SuppressWarnings("rawtypes")                               // Because of generic array
creation.
+    private FeaturesInfo(final Vector counts, final VariableInfo identifiers, final VariableInfo
time,
+            final Collection<VariableInfo> coordinates, final Collection<VariableInfo>
properties)
+    {
         this.counts      = counts;
         this.identifiers = identifiers;
+        this.coordinates = coordinates.toArray(new VariableInfo[coordinates.size()]);
+        this.properties  = properties .toArray(new VariableInfo[properties .size()]);
+        this.time        = time;
+        /*
+         * Creates a description of the features to be read.
+         */
+        final Map<String,Object> info = new HashMap<>(4);
+        final PropertyType[] pt = new PropertyType[this.properties.length + 2];
+        AttributeType[] characteristics = null;
+        for (int i=0; i<pt.length; i++) {
+            final VariableInfo variable;
+            final Class<?> valueClass;
+            int minOccurs = 1;
+            int maxOccurs = 1;
+            switch (i) {
+                case 0: {
+                    variable        = identifiers;
+                    valueClass      = Integer.class;
+                    break;
+                }
+                case 1: {
+                    variable        = null;
+                    valueClass      = factory.polylineClass;
+                    characteristics = new AttributeType[] {MovingFeature.TIME};
+                    break;
+                }
+                default: {
+                    variable        = this.properties[i-2];
+                    valueClass      = Number.class;           // TODO: use more accurate
value class.
+                    minOccurs       = 0;
+                    maxOccurs       = Integer.MAX_VALUE;
+                    break;
+                }
+            }
+            info.put(DefaultAttributeType.NAME_KEY, (variable != null) ? variable.getName()
: "trajectory");
+            // TODO: add description.
+            pt[i] = new DefaultAttributeType<>(info, valueClass, minOccurs, maxOccurs,
null, characteristics);
+        }
+        info.put(DefaultAttributeType.NAME_KEY, "Feature");     // TODO: find a better name.
+        type = new DefaultFeatureType(info, false, null, pt);
     }
 
     /**
@@ -148,12 +234,154 @@ search: for (final VariableInfo counts :
                         }
                     }
                     /*
-                     * At this point, all information have been verified as valid.
+                     * At this point, all information have been verified as valid. Now search
all variables having
+                     * the expected sample dimension. Those variable contains the actual
data. For example if the
+                     * sample dimension name is "points", then we may have:
+                     *
+                     *     double longitude(points);
+                     *         longitude:axis = "X";
+                     *         longitude:standard_name = "longitude";
+                     *         longitude:units = "degrees_east";
+                     *     double latitude(points);
+                     *         latitude:axis = "Y";
+                     *         latitude:standard_name = "latitude";
+                     *         latitude:units = "degrees_north";
+                     *     double time(points);
+                     *         time:axis = "T";
+                     *         time:standard_name = "time";
+                     *         time:units = "minutes since 2014-11-29 00:00:00";
+                     *     short myCustomProperty(points);
                      */
-                    features.add(new FeaturesInfo(counts.read().compress(0), identifiers));
+                    final Map<String,VariableInfo> coordinates = new LinkedHashMap<>();
+                    final List<VariableInfo> properties  = new ArrayList<>();
+                    for (final VariableInfo data : decoder.variables) {
+                        if (data.dimensions.length == 1 && data.dimensions[0] ==
sampleDimension) {
+                            final Object axisType = data.getAttributeValue(CF.AXIS);
+                            if (axisType == null) {
+                                properties.add(data);
+                            } else if (coordinates.put(axisType.toString(), data) != null)
{
+                                continue search;    // Two axes of the same type: abort.
+                            }
+                        }
+                    }
+                    final VariableInfo time = coordinates.remove("T");
+                    if (time != null) {
+                        features.add(new FeaturesInfo(counts.read().compress(0), identifiers,
time, coordinates.values(), properties));
+                    }
                 }
             }
         }
         return features.toArray(new FeaturesInfo[features.size()]);
     }
+
+    /**
+     * Returns the stream of features.
+     */
+    @Override
+    public Stream<Feature> features() {
+        return StreamSupport.stream(new Iter(), false);
+    }
+
+    /**
+     * Implementation of the iterator returned by {@link #features()}.
+     */
+    private final class Iter implements Spliterator<Feature> {
+        /**
+         * Index of the next feature to read.
+         */
+        private int index;
+
+        /**
+         * Position in the data vectors of the next feature to read.
+         * This is the sum of the length of data in all previous features.
+         */
+        private int position;
+
+        /**
+         * Creates a new iterator.
+         */
+        Iter() {
+        }
+
+        /**
+         * Executes the given action only on the next feature, if any.
+         *
+         * @todo current reading process implies lot of seeks, which is inefficient.
+         */
+        @Override
+        public boolean tryAdvance(final Consumer<? super Feature> action) {
+            final int   length = counts.intValue(index);
+            final int[] lower  = {position};
+            final int[] upper  = {position + length};
+            final int[] step   = {1};
+            final Vector   id, t;
+            final Vector[] coords = new Vector[coordinates.length];
+            final Vector[] props  = new Vector[properties.length];
+            try {
+                id = identifiers.read();                    // Efficiency should be okay
because of cached value.
+                t = time.read(lower, upper, step);
+                for (int i=0; i<coordinates.length; i++) {
+                    coords[i] = coordinates[i].read(lower, upper, step);
+                }
+                for (int i=0; i<properties.length; i++) {
+                    props[i] = properties[i].read(lower, upper, step);
+                }
+            } catch (IOException | DataStoreException e) {
+                throw new BackingStoreException(canNotReadFile(), e);
+            }
+            final Feature feature = type.newInstance();
+            feature.setPropertyValue(identifiers.getName(), id.intValue(index));
+            for (int i=0; i<properties.length; i++) {
+                feature.setPropertyValue(properties[i].getName(), props[i]);
+                // TODO: set time characteristic.
+            }
+            // TODO: temporary hack - to be replaced by support in Vector.
+            final int dimension = coordinates.length;
+            final double[] tmp = new double[length * dimension];
+            for (int i=0; i<tmp.length; i++) {
+                tmp[i] = coords[i % dimension].doubleValue(i / dimension);
+            }
+            feature.setPropertyValue("trajectory", factory.createPolyline(dimension, Vector.create(tmp,
false)));
+            action.accept(feature);
+            position = Math.addExact(position, length);
+            return ++index < counts.size();
+        }
+
+        /**
+         * Current implementation can not split this iterator.
+         */
+        @Override
+        public Spliterator<Feature> trySplit() {
+            return null;
+        }
+
+        /**
+         * Returns the number of features.
+         */
+        @Override
+        public long estimateSize() {
+            return counts.size();
+        }
+
+        /**
+         * Returns the characteristics of the iteration over feature instances.
+         * The iteration is assumed {@link #ORDERED} in the declaration order in the NetCDF
file.
+         * The iteration is {@link #NONNULL} (i.e. {@link #tryAdvance(Consumer)} is not allowed
+         * to return null value) and {@link #IMMUTABLE} (i.e. we do not support modification
of
+         * the NetCDF file while an iteration is in progress).
+         *
+         * @return characteristics of iteration over the features in the NetCDF file.
+         */
+        @Override
+        public int characteristics() {
+            return ORDERED | NONNULL | IMMUTABLE | SIZED;
+        }
+    }
+
+    /**
+     * Returns the error message for a file that can not be read.
+     */
+    final String canNotReadFile() {
+        return null;    // TODO
+    }
 }

Modified: sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/FeaturesWrapper.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/FeaturesWrapper.java?rev=1800217&r1=1800216&r2=1800217&view=diff
==============================================================================
--- sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/FeaturesWrapper.java
[UTF-8] (original)
+++ sis/branches/JDK8/storage/sis-netcdf/src/main/java/org/apache/sis/internal/netcdf/ucar/FeaturesWrapper.java
[UTF-8] Thu Jun 29 00:21:33 2017
@@ -19,6 +19,10 @@ package org.apache.sis.internal.netcdf.u
 import org.apache.sis.internal.netcdf.DiscreteSampling;
 import ucar.nc2.ft.FeatureCollection;
 
+// Branch-dependent imports
+import java.util.stream.Stream;
+import org.opengis.feature.Feature;
+
 
 /**
  * A wrapper around the UCAR {@code ucar.nc2.ft} package.
@@ -41,5 +45,12 @@ final class FeaturesWrapper extends Disc
         this.features = features;
     }
 
-    // TODO
+
+    /**
+     * Returns the stream of features.
+     */
+    @Override
+    public Stream<Feature> features() {
+        throw new UnsupportedOperationException();      // TODO
+    }
 }

Modified: sis/branches/JDK8/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/csv/Store.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK8/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/csv/Store.java?rev=1800217&r1=1800216&r2=1800217&view=diff
==============================================================================
--- sis/branches/JDK8/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/csv/Store.java
[UTF-8] (original)
+++ sis/branches/JDK8/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/csv/Store.java
[UTF-8] Thu Jun 29 00:21:33 2017
@@ -126,7 +126,7 @@ public final class Store extends Feature
 
     /**
      * Index of the column containing trajectory coordinates.
-     * Columns before the trajectory are Moving Feature identifier {@code mfidref}, start
time and end time.
+     * Columns before the trajectory are Moving Feature identifier {@code mfIdRef}, start
time and end time.
      */
     private static final int TRAJECTORY_COLUMN = 3;
 
@@ -458,7 +458,7 @@ public final class Store extends Feature
      */
     @SuppressWarnings("rawtypes")               // "rawtypes" because of generic array creation.
     private FeatureType parseFeatureType(final List<String> elements) throws DataStoreException
{
-        AttributeType[] characteristics = {};
+        AttributeType[] characteristics = null;
         final int size = elements.size();
         final List<PropertyType> properties = new ArrayList<>();
         for (int i=1; i<size; i++) {
@@ -797,14 +797,14 @@ public final class Store extends Feature
                 for (int i=0; i<n; i++) {
                     values[i] = converters[i].apply((String) values[i]);
                 }
-                final String  mfidref   =  (String)  values[0];
+                final String  mfIdRef   =  (String)  values[0];
                 final long    startTime = ((Instant) values[1]).toEpochMilli();
                 final long    endTime   = ((Instant) values[2]).toEpochMilli();
                 String        publish   = null;
-                if (!mfidref.equals(identifier)) {
+                if (!mfIdRef.equals(identifier)) {
                     publish    = identifier;
-                    identifier = mfidref;
-                    builder    = builders.computeIfAbsent(mfidref, (k) -> new MovingFeature(np));
+                    identifier = mfIdRef;
+                    builder    = builders.computeIfAbsent(mfIdRef, (k) -> new MovingFeature(np));
                 }
                 builder.addTimeRange(startTime, endTime);
                 for (int i=0; i<np; i++) {



Mime
View raw message