sis-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mlebi...@apache.org
Subject svn commit: r1647105 - in /sis/branches/JDK8/storage/sis-shapefile/src/main: java/org/apache/sis/storage/shapefile/ resources/org/apache/sis/storage/shapefile/
Date Sun, 21 Dec 2014 08:19:31 GMT
Author: mlebihan
Date: Sun Dec 21 08:19:30 2014
New Revision: 1647105

URL: http://svn.apache.org/r1647105
Log:
Refactoring for SIS-184 : MappedByteBuffer is ejected from Database class and put into internal
classes in order to be upgrated to something better later.

Added:
    sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/AbstractByteReader.java
    sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/ByteReader.java
    sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/MappedByteReader.java
Modified:
    sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/Database.java
    sis/branches/JDK8/storage/sis-shapefile/src/main/resources/org/apache/sis/storage/shapefile/Database.properties

Added: sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/AbstractByteReader.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/AbstractByteReader.java?rev=1647105&view=auto
==============================================================================
--- sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/AbstractByteReader.java
(added)
+++ sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/AbstractByteReader.java
Sun Dec 21 08:19:30 2014
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sis.storage.shapefile;
+
+import java.io.File;
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
+import java.text.MessageFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Objects;
+import java.util.logging.Level;
+
+import org.apache.sis.util.logging.AbstractAutoChecker;
+
+/**
+ * The Abstract Byte Reader.
+ * @author Marc LE BIHAN
+ */
+abstract class AbstractByteReader extends AbstractAutoChecker implements ByteReader {
+    /** Database filename. */
+    protected String dbfFile;
+
+    /** Number of bytes in the header. */
+    protected short DbaseHeaderBytes;
+
+    /** Number of bytes in the record. */
+    protected short DbaseRecordBytes;
+    
+    /** Reserved (dBASE IV) Filled with 00h. */
+    protected byte[] reservedFiller1 = new byte[2];
+    
+    /** 
+     * Reserved : Incomplete transaction (dBASE IV).
+     * 00h : Transaction ended (or rolled back).
+     * 01h : Transaction started. 
+     */
+    protected byte reservedIncompleteTransaction;
+
+    /**
+     * Reserved : Encryption flag (dBASE IV).
+     * 00h : Not encrypted. 
+     * 01h : Data encrypted.
+     */
+    protected byte reservedEncryptionFlag;
+    
+    /** Reserved : Free record thread (for LAN only). */
+    protected byte[] reservedFreeRecordThread = new byte[4];
+    
+    /** Reserved : For multi-user (DBase 3+). */
+    protected byte[] reservedMultiUser = new byte[8];
+
+    /** Reserved : MDX flag (dBASE IV). */
+    protected byte reservedMDXFlag;
+    
+    /** Binary code page value. */
+    protected byte codePage;
+    
+    /** Reserved (dBASE IV) Filled with 00h. */
+    protected byte[] reservedFiller2 = new byte[2];
+    
+    /** Marks the end of the descriptor : must be 0x0D. */
+    protected byte descriptorTerminator;
+
+    /** Valid dBASE III PLUS table file (03h without a memo .DBT file; 83h with a memo).
*/
+    protected byte DbaseVersion;
+
+    /** Number of records in the table. */
+    protected int recordCount;
+    
+    /** Database charset. */
+    protected Charset charset;
+
+    /** Date of last update; in YYMMDD format. */
+    protected byte[] DbaseLastUpdate = new byte[3];
+
+    /** Fields descriptor. */
+    protected FieldsDescriptors fieldsDescriptors = new FieldsDescriptors();
+
+    /** Current row rumber. */
+    protected int rowNum;
+    
+    /**
+     * Returns the charset.
+     * @return Charset.
+     */
+    @Override public Charset getCharset() {
+        return charset;
+    }
+
+    /**
+     * Returns the fields descriptors.
+     * @return Fields descriptors.
+     */
+    @Override public FieldsDescriptors getFieldsDescriptors() {
+        return fieldsDescriptors;
+    }
+    
+    /**
+     * Returns the database last update date.
+     * @return Date of the last update.
+     */
+    @Override public Date getDateOfLastUpdate() {
+        return toDate(DbaseLastUpdate);
+    }
+
+    /**
+     * Returns the record count.
+     * @return Record count.
+     */
+    @Override public int getRecordCount() {
+        return recordCount;
+    }
+    
+    /**
+     * Returns the current record number.
+     * @return Current record number.
+     */
+    @Override public int getRowNum() {
+        return rowNum;
+    }
+
+    /**
+     * Convert the binary code page value of the Dbase 3 file to a recent Charset.
+     * @param codePageBinaryValue page code binary value.
+     * @return Charset.
+     * @throws InvalidDbaseFileFormatException if the binary value is not one of the standard
values that the DBF file should carry : the Dbase 3
+     * file might be corrupted.
+     * @throws UnsupportedCharsetException if the code page as no representation in recents
Charset (legacy DOS or macintosh charsets).
+     */
+    protected Charset toCharset(byte codePageBinaryValue) throws InvalidDbaseFileFormatException,
UnsupportedCharsetException {
+        // Attempt to find a known conversion.
+        String dbfCodePage = toCodePage(codePageBinaryValue);
+        
+        // If no conversion has been found, decide if the cause is an unsupported value or
an illegal value to choose the good exception to return.
+        if (dbfCodePage == null) {
+            switch(Byte.toUnsignedInt(codePageBinaryValue)) {
+                case 0x04: dbfCodePage = "unsupported"; break;
+                case 0x68: dbfCodePage = "unsupported"; break; // Kamenicky (Czech) MS-DOS
+                case 0x69: dbfCodePage = "unsupported"; break; // Mazovia (Polish) MS-DOS
+                case 0x96: dbfCodePage = "unsupported"; break; // russian mac
+                case 0x97: dbfCodePage = "unsupported"; break; // eastern european macintosh
+                case 0x98: dbfCodePage = "unsupported"; break; // greek macintosh
+                case 0xC8: dbfCodePage = "unsupported"; break; // windows ee
+                default: dbfCodePage = "invalid"; break;
+            }
+        }
+        
+        assert dbfCodePage != null;
+        
+        // If the code page is invalid, the database itself has chances to be invalid too.
+        if (dbfCodePage.equals("invalid")) {
+            String message = format(Level.SEVERE, "excp.illegal_codepage", codePageBinaryValue,
new File(dbfFile).getAbsolutePath());
+            throw new InvalidDbaseFileFormatException(message);
+        }
+        
+        // If the code page cannot find a match for a more recent Charset, we wont be able
to handle this DBF.
+        if (dbfCodePage.equals("unsupported")) {
+            String message = format(Level.SEVERE, "excp.unsupported_codepage", dbfCodePage,
new File(dbfFile).getAbsolutePath());
+            throw new UnsupportedCharsetException(message);
+        }
+        
+        try {
+            return Charset.forName(dbfCodePage);
+        }
+        catch(IllegalArgumentException e) {
+            // If this happens here, it means that we have selected a wrong charset. We have
a bug.
+            String message = format(Level.SEVERE, "assert.wrong_charset_selection", dbfCodePage,
new File(dbfFile).getAbsolutePath());
+            throw new RuntimeException(message);
+        }
+    }
+    
+    /**
+     * Return a Charset code page from a binary code page value.
+     * @param pageCodeBinaryValue binary code page value.
+     * @return Page code.
+     */
+    private String toCodePage(byte pageCodeBinaryValue) {
+        // From http://trac.osgeo.org/gdal/ticket/2864
+        HashMap<Integer, String> knownConversions = new HashMap<>();
+        knownConversions.put(0x01, "cp437"); //  U.S. MS–DOS
+        knownConversions.put(0x02, "cp850"); // International MS–DOS
+        knownConversions.put(0x03, "cp1252"); // Windows ANSI
+        knownConversions.put(0x08, "cp865"); //  Danish OEM
+        knownConversions.put(0x09, "cp437"); //  Dutch OEM
+        knownConversions.put(0x0a, "cp850"); //  Dutch OEM*
+        knownConversions.put(0x0b, "cp437"); //  Finnish OEM
+        knownConversions.put(0x0d, "cp437"); //  French OEM
+        knownConversions.put(0x0e, "cp850"); //  French OEM*
+        knownConversions.put(0x0f, "cp437"); //  German OEM
+        knownConversions.put(0x10, "cp850"); //  German OEM*
+        knownConversions.put(0x11, "cp437"); //  Italian OEM
+        knownConversions.put(0x12, "cp850"); //  Italian OEM*
+        knownConversions.put(0x13, "cp932"); //  Japanese Shift-JIS
+        knownConversions.put(0x14, "cp850"); //  Spanish OEM*
+        knownConversions.put(0x15, "cp437"); //  Swedish OEM
+        knownConversions.put(0x16, "cp850"); //  Swedish OEM*
+        knownConversions.put(0x17, "cp865"); //  Norwegian OEM
+        knownConversions.put(0x18, "cp437"); //  Spanish OEM
+        knownConversions.put(0x19, "cp437"); //  English OEM (Britain)
+        knownConversions.put(0x1a, "cp850"); //  English OEM (Britain)*
+        knownConversions.put(0x1b, "cp437"); //  English OEM (U.S.)
+        knownConversions.put(0x1c, "cp863"); //  French OEM (Canada)
+        knownConversions.put(0x1d, "cp850"); //  French OEM*
+        knownConversions.put(0x1f, "cp852"); //  Czech OEM
+        knownConversions.put(0x22, "cp852"); //  Hungarian OEM
+        knownConversions.put(0x23, "cp852"); //  Polish OEM
+        knownConversions.put(0x24, "cp860"); //  Portuguese OEM
+        knownConversions.put(0x25, "cp850"); //  Portuguese OEM*
+        knownConversions.put(0x26, "cp866"); //  Russian OEM
+        knownConversions.put(0x37, "cp850"); //  English OEM (U.S.)*
+        knownConversions.put(0x40, "cp852"); //  Romanian OEM
+        knownConversions.put(0x4d, "cp936"); //  Chinese GBK (PRC)
+        knownConversions.put(0x4e, "cp949"); //  Korean (ANSI/OEM)
+        knownConversions.put(0x4f, "cp950"); //  Chinese Big5 (Taiwan)
+        knownConversions.put(0x50, "cp874"); //  Thai (ANSI/OEM)
+        knownConversions.put(0x57, "cp1252"); // ANSI
+        knownConversions.put(0x58, "cp1252"); // Western European ANSI
+        knownConversions.put(0x59, "cp1252"); // Spanish ANSI
+        knownConversions.put(0x64, "cp852"); //  Eastern European MS–DOS
+        knownConversions.put(0x65, "cp866"); //  Russian MS–DOS
+        knownConversions.put(0x66, "cp865"); //  Nordic MS–DOS
+        knownConversions.put(0x67, "cp861"); //  Icelandic MS–DOS
+        knownConversions.put(0x6a, "cp737"); //  Greek MS–DOS (437G)
+        knownConversions.put(0x6b, "cp857"); //  Turkish MS–DOS
+        knownConversions.put(0x6c, "cp863"); //  French–Canadian MS–DOS
+        knownConversions.put(0x78, "cp950"); //  Taiwan Big 5
+        knownConversions.put(0x79, "cp949"); //  Hangul (Wansung)
+        knownConversions.put(0x7a, "cp936"); //  PRC GBK
+        knownConversions.put(0x7b, "cp932"); //  Japanese Shift-JIS
+        knownConversions.put(0x7c, "cp874"); //  Thai Windows/MS–DOS
+        knownConversions.put(0x86, "cp737"); //  Greek OEM
+        knownConversions.put(0x87, "cp852"); //  Slovenian OEM
+        knownConversions.put(0x88, "cp857"); //  Turkish OEM
+        knownConversions.put(0xc8, "cp1250"); // Eastern European Windows
+        knownConversions.put(0xc9, "cp1251"); // Russian Windows
+        knownConversions.put(0xca, "cp1254"); // Turkish Windows
+        knownConversions.put(0xcb, "cp1253"); // Greek Windows
+        knownConversions.put(0xcc, "cp1257"); // Baltic Windows
+        
+        return(knownConversions.get(Byte.toUnsignedInt(pageCodeBinaryValue)));
+    }
+
+    /**
+     * Return a date from a byte array.
+     * @param yymmdd byte[3] with byte[0] = year (2 digits), [1] = month, [2] = day. 
+     * @return Date.
+     */
+    private Date toDate(byte[] yymmdd) {
+        Objects.requireNonNull(yymmdd, "the yymmdd bytes cannot be null");
+        
+        if (yymmdd.length != 3)
+            throw new IllegalArgumentException(MessageFormat.format("Database:toDate() works
only on a 3 bytes YY MM DD date. this array has {0} length", yymmdd.length));
+        
+        Objects.requireNonNull(yymmdd[0], "the year byte cannot be null");
+        Objects.requireNonNull(yymmdd[1], "the month byte cannot be null");
+        Objects.requireNonNull(yymmdd[2], "the day byte cannot be null");
+        
+        int year = yymmdd[0] < 70 ? 100 + yymmdd[0] : yymmdd[0];
+        int month = yymmdd[1];
+        int day = yymmdd[2];
+        
+        @SuppressWarnings("deprecation") // But everything is deprecated in DBF files...

+        Date date = new Date(year, month, day);
+        return date;
+    }
+}

Added: sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/ByteReader.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/ByteReader.java?rev=1647105&view=auto
==============================================================================
--- sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/ByteReader.java
(added)
+++ sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/ByteReader.java
Sun Dec 21 08:19:30 2014
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sis.storage.shapefile;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Date;
+import java.util.HashMap;
+
+import org.opengis.feature.Feature;
+
+/**
+ * Database byte reader contract. Used to allow refactoring of core byte management of a
DBase file.
+ * @author Marc LE BIHAN
+ */
+interface ByteReader {
+    /**
+     * Close the MappedByteReader.
+     * @throws IOException if the close operation fails.
+     */
+    public void close() throws IOException;
+    
+    /**
+     * Returns the charset.
+     * @return Charset.
+     */
+    public Charset getCharset();
+    
+    /**
+     * Returns the fields descriptors.
+     * @return Fields descriptors.
+     */
+    public FieldsDescriptors getFieldsDescriptors();
+    
+    /**
+     * Returns the database last update date.
+     * @return Date of the last update.
+     */
+    public Date getDateOfLastUpdate();
+    
+    /**
+     * Returns the record count.
+     * @return Record count.
+     */
+    public int getRecordCount();
+
+    /**
+     * Returns the current record number.
+     * @return Current record number.
+     */
+    public int getRowNum();
+    
+    /**
+     * Load a row into a feature.
+     * @param feature Feature to fill.
+     */
+    public void loadRowIntoFeature(Feature feature);
+    
+    /**
+     * Read the next row as a set of objects.
+     * @return Map of field name / object value.
+     */
+    public HashMap<String, Object> readNextRowAsObjects();
+}

Modified: sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/Database.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/Database.java?rev=1647105&r1=1647104&r2=1647105&view=diff
==============================================================================
--- sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/Database.java
[UTF-8] (original)
+++ sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/Database.java
[UTF-8] Sun Dec 21 08:19:30 2014
@@ -16,15 +16,13 @@
  */
 package org.apache.sis.storage.shapefile;
 
-import java.io.*;
-import java.nio.ByteOrder;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.nio.charset.Charset;
-import java.nio.charset.UnsupportedCharsetException;
-import java.text.MessageFormat;
-import java.util.*;
-import java.util.logging.Level;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Objects;
 
 import org.apache.sis.util.logging.AbstractAutoChecker;
 import org.opengis.feature.Feature;
@@ -46,74 +44,9 @@ public class Database extends AbstractAu
 
     /** Indicates is the database file is closed or not. */
     private boolean isClosed;
-
-    /** Valid dBASE III PLUS table file (03h without a memo .DBT file; 83h with a memo).
*/
-    public byte DbaseVersion;
-
-    /** Date of last update; in YYMMDD format. */
-    public byte[] DbaseLastUpdate = new byte[3];
-
-    /** Number of records in the table. */
-    public int recordCount;
-
-    /** Number of bytes in the header. */
-    public short DbaseHeaderBytes;
-
-    /** Number of bytes in the record. */
-    public short DbaseRecordBytes;
     
-    /** Reserved (dBASE IV) Filled with 00h. */
-    private byte[] reservedFiller1 = new byte[2];
-    
-    /** 
-     * Reserved : Incomplete transaction (dBASE IV).
-     * 00h : Transaction ended (or rolled back).
-     * 01h : Transaction started. 
-     */
-    private byte reservedIncompleteTransaction;
-
-    /**
-     * Reserved : Encryption flag (dBASE IV).
-     * 00h : Not encrypted. 
-     * 01h : Data encrypted.
-     */
-    private byte reservedEncryptionFlag;
-    
-    /** Reserved : Free record thread (for LAN only). */
-    private byte[] reservedFreeRecordThread = new byte[4];
-    
-    /** Reserved : For multi-user (DBase 3+). */
-    private byte[] reservedMultiUser = new byte[8];
-
-    /** Reserved : MDX flag (dBASE IV). */
-    private byte reservedMDXFlag;
-    
-    /** Binary code page value. */
-    private byte codePage;
-    
-    /** Database charset. */
-    private Charset charset;
-    
-    /** Reserved (dBASE IV) Filled with 00h. */
-    private byte[] reservedFiller2 = new byte[2];
-    
-    /** Marks the end of the descriptor : must be 0x0D. */
-    private byte descriptorTerminator;
-
-    /** Fields descriptor. */
-    public FieldsDescriptors fieldsDescriptors = new FieldsDescriptors();
-
-    /** Input Stream on the DBF. */
-    private FileInputStream fis;
-
-    /** File channel on the file. */
-    private FileChannel fc;
-
-    /** Buffer reader. */
-    private MappedByteBuffer df;
-
-    /** Current row rumber. */
-    private int rowNum;
+    /** Database binary reader. */
+    private ByteReader binaryReader; 
     
     /**
      * Load a database file.
@@ -124,13 +57,18 @@ public class Database extends AbstractAu
     public Database(String file) throws FileNotFoundException, InvalidDbaseFileFormatException
{
         Objects.requireNonNull(file, "The database file to load cannot be null.");
         dbfFile = file;
-
-        fis = new FileInputStream(file);
-        fc = fis.getChannel();
-        rowNum = 0;
-        isClosed = false;
+        binaryReader = new MappedByteReader(file);
         
-        loadDescriptor();
+        isClosed = false;
+    }
+
+    /**
+     * @see java.lang.AutoCloseable#close()
+     */
+    @Override
+    public void close() throws IOException {
+        binaryReader.close();
+        isClosed = true;
     }
 
     /**
@@ -138,7 +76,7 @@ public class Database extends AbstractAu
      * @return Charset.
      */
     public Charset getCharset() {
-        return this.charset;
+        return binaryReader.getCharset();
     }
     
     /**
@@ -146,7 +84,7 @@ public class Database extends AbstractAu
      * @return Record count.
      */
     public int getRecordCount() {
-        return this.recordCount;
+        return binaryReader.getRecordCount();
     }
 
     /**
@@ -154,98 +92,15 @@ public class Database extends AbstractAu
      * @return Field descriptor.
      */
     public ArrayList<FieldDescriptor> getFieldsDescriptor() {
-        return this.fieldsDescriptors;
-    }
-
-    /**
-     * Return the mapped byte buffer currently used to read that database file.
-     * @return Database file.
-     */
-    public MappedByteBuffer getByteBuffer() {
-        return this.df;
-    }
-
-    /**
-     * Loading the database file content from binary .dbf file.
-     * @throws InvalidDbaseFileFormatException if descriptor is not readable. 
-     */
-    private void loadDescriptor() throws InvalidDbaseFileFormatException {
-        try {
-            int fsize = (int) fc.size();
-            df = fc.map(FileChannel.MapMode.READ_ONLY, 0, fsize);
-    
-            this.DbaseVersion = df.get();
-            df.get(this.DbaseLastUpdate);
-    
-            df.order(ByteOrder.LITTLE_ENDIAN);
-            this.recordCount = df.getInt();
-            this.DbaseHeaderBytes = df.getShort();
-            this.DbaseRecordBytes = df.getShort();
-            df.order(ByteOrder.BIG_ENDIAN);
-            
-            df.get(reservedFiller1);
-            this.reservedIncompleteTransaction = df.get();
-            this.reservedEncryptionFlag = df.get();
-            df.get(reservedFreeRecordThread);
-            df.get(reservedMultiUser);
-            reservedMDXFlag = df.get();
-            
-            // Translate code page value to a known charset.
-            this.codePage = df.get();
-            this.charset = toCharset(this.codePage);             
-            
-            df.get(reservedFiller2); 
-    
-            while(df.position() < this.DbaseHeaderBytes - 1) {
-                FieldDescriptor fd = toFieldDescriptor();
-                this.fieldsDescriptors.add(fd);
-                // loop until you hit the 0Dh field terminator
-            }
-            
-            this.descriptorTerminator = df.get();
-
-            // If the last character read after the field descriptor isn't 0x0D, the expected
mark has not been found and the DBF is corrupted.
-            if (descriptorTerminator != 0x0D) {
-                String message = format(Level.SEVERE, "excp.filedescriptor_problem", new
File(dbfFile).getAbsolutePath(), "Character marking the end of the fields descriptors (0x0D)
has not been found.");
-                throw new InvalidDbaseFileFormatException(message);
-            }
-        }
-        catch(IOException e) {
-            // This exception doesn't denote a trouble of file opening because the file has
been checked before 
-            // the calling of this private function.
-            // Therefore, an internal structure problem cause maybe a premature End of file
or anything else, but the only thing
-            // we can conclude is : we are not before a device trouble, but a file format
trouble.
-            String message = format(Level.SEVERE, "excp.filedescriptor_problem", new File(dbfFile).getAbsolutePath(),
e.getMessage());
-            throw new InvalidDbaseFileFormatException(message);
-        }
+        return binaryReader.getFieldsDescriptors();
     }
 
     /**
-     * Read the next row as a set of objects.
-     * @return Map of field name / object value.
+     * Return the database as a {@link java.io.File}.
+     * @return File.
      */
-    public HashMap<String, Object> readNextRowAsObjects() {
-        // TODO: ignore deleted records
-        byte isDeleted = df.get(); // denotes whether deleted or current
-        // read first part of record
-
-        HashMap<String, Object> fieldsValues = new HashMap<>();
-
-        for (FieldDescriptor fd : this.fieldsDescriptors) {
-            byte[] data = new byte[fd.getLength()];
-            df.get(data);
-
-            int length = data.length;
-            while (length != 0 && data[length - 1] <= ' ') {
-                length--;
-            }
-
-            String value = new String(data, 0, length);
-            fieldsValues.put(fd.getName(), value);
-        }
-
-        rowNum ++;
-        return fieldsValues;
+    public File getFile() {
+        return(new File(dbfFile));
     }
 
     /**
@@ -253,89 +108,7 @@ public class Database extends AbstractAu
      * @return Row number (zero based) or -1 if reading has not started.
      */
     public int getRowNum() {
-        return rowNum;
-    }
-
-    /**
-     * Load a row into a feature.
-     * @param feature Feature to fill.
-     */
-    public void loadRowIntoFeature(Feature feature) {
-        // TODO: ignore deleted records
-        df.get(); // denotes whether deleted or current
-        // read first part of record
-
-        for (FieldDescriptor fd : this.fieldsDescriptors) {
-            byte[] data = new byte[fd.getLength()];
-            df.get(data);
-
-            int length = data.length;
-            while (length != 0 && data[length - 1] <= ' ') {
-                length--;
-            }
-
-            String value = new String(data, 0, length);
-            feature.setPropertyValue(fd.getName(), value);
-        }
-
-        rowNum ++;
-    }
-
-    /**
-     * Create a field descriptor from the current position of the binary stream.
-     * @return FieldDescriptor or null if there is no more available.
-     */
-    private FieldDescriptor toFieldDescriptor() {
-        // If there is no more field description available, return null.
-        if (df.position() >= this.DbaseHeaderBytes - 1)
-            return null;
-
-        FieldDescriptor fd = new FieldDescriptor();
-
-        // Field name.
-        df.get(fd.FieldName);
-
-        // Field type.
-        char dt = (char) df.get();
-        fd.FieldType = DataType.valueOfDataType(dt);
-
-        // Field address.
-        df.get(fd.FieldAddress);
-
-        // Length and scale.
-        fd.FieldLength = df.get();
-        fd.FieldDecimalCount = df.get();
-
-        df.getShort(); // reserved
-
-        df.get(fd.DbasePlusLanReserved2);
-
-        // Work area id.
-        fd.WorkAreaID = df.get();
-
-        df.get(fd.DbasePlusLanReserved3);
-
-        // Fields.
-        fd.SetFields = df.get();
-
-        byte[] data = new byte[6];
-        df.get(data); // reserved
-
-        return fd;
-    }
-
-    /**
-     * @see java.lang.AutoCloseable#close()
-     */
-    @Override
-    public void close() throws IOException {
-        if (fc != null)
-            fc.close();
-
-        if (fis != null)
-            fis.close();
-
-        isClosed = true;
+        return binaryReader.getRowNum();
     }
 
     /**
@@ -347,157 +120,18 @@ public class Database extends AbstractAu
     }
     
     /**
-
-     * Return a date from a byte array.
-     * @param yymmdd byte[3] with byte[0] = year (2 digits), [1] = month, [2] = day. 
-     * @return Date.
+     * Load a row into a feature.
+     * @param feature Feature to fill.
      */
-    private Date toDate(byte[] yymmdd) {
-        Objects.requireNonNull(yymmdd, "the yymmdd bytes cannot be null");
-        
-        if (yymmdd.length != 3)
-            throw new IllegalArgumentException(MessageFormat.format("Database:toDate() works
only on a 3 bytes YY MM DD date. this array has {0} length", yymmdd.length));
-        
-        Objects.requireNonNull(yymmdd[0], "the year byte cannot be null");
-        Objects.requireNonNull(yymmdd[1], "the month byte cannot be null");
-        Objects.requireNonNull(yymmdd[2], "the day byte cannot be null");
-        
-        int year = yymmdd[0] < 70 ? 100 + yymmdd[0] : yymmdd[0];
-        int month = yymmdd[1];
-        int day = yymmdd[2];
-        
-        @SuppressWarnings("deprecation") // But everything is deprecated in DBF files...

-        Date date = new Date(year, month, day);
-        return date;
-    }
-
-    /**
-     * Convert the binary code page value of the Dbase 3 file to a recent Charset.
-     * @param codePageBinaryValue page code binary value.
-     * @return Charset.
-     * @throws InvalidDbaseFileFormatException if the binary value is not one of the standard
values that the DBF file should carry : the Dbase 3
-     * file might be corrupted.
-     * @throws UnsupportedCharsetException if the code page as no representation in recents
Charset (legacy DOS or macintosh charsets).
-     */
-    private Charset toCharset(byte codePageBinaryValue) throws InvalidDbaseFileFormatException,
UnsupportedCharsetException {
-        // Attempt to find a known conversion.
-        String dbfCodePage = toCodePage(codePageBinaryValue);
-        
-        // If no conversion has been found, decide if the cause is an unsupported value or
an illegal value to choose the good exception to return.
-        if (dbfCodePage == null) {
-            switch(Byte.toUnsignedInt(codePageBinaryValue)) {
-                case 0x04: dbfCodePage = "unsupported"; break;
-                case 0x68: dbfCodePage = "unsupported"; break; // Kamenicky (Czech) MS-DOS
-                case 0x69: dbfCodePage = "unsupported"; break; // Mazovia (Polish) MS-DOS
-                case 0x96: dbfCodePage = "unsupported"; break; // russian mac
-                case 0x97: dbfCodePage = "unsupported"; break; // eastern european macintosh
-                case 0x98: dbfCodePage = "unsupported"; break; // greek macintosh
-                case 0xC8: dbfCodePage = "unsupported"; break; // windows ee
-                default: dbfCodePage = "invalid"; break;
-            }
-        }
-        
-        assert dbfCodePage != null;
-        
-        // If the code page is invalid, the database itself has chances to be invalid too.
-        if (dbfCodePage.equals("invalid")) {
-            String message = format(Level.SEVERE, "excp.illegal_codepage", codePageBinaryValue,
new File(dbfFile).getAbsolutePath());
-            throw new InvalidDbaseFileFormatException(message);
-        }
-        
-        // If the code page cannot find a match for a more recent Charset, we wont be able
to handle this DBF.
-        if (dbfCodePage.equals("unsupported")) {
-            String message = format(Level.SEVERE, "excp.unsupported_codepage", dbfCodePage,
new File(dbfFile).getAbsolutePath());
-            throw new UnsupportedCharsetException(message);
-        }
-        
-        try {
-            return Charset.forName(dbfCodePage);
-        }
-        catch(IllegalArgumentException e) {
-            // If this happens here, it means that we have selected a wrong charset. We have
a bug.
-            String message = format(Level.SEVERE, "assert.wrong_charset_selection", dbfCodePage,
new File(dbfFile).getAbsolutePath());
-            throw new RuntimeException(message);
-        }
-    }
-
-    /**
-     * Return a Charset code page from a binary code page value.
-     * @param pageCodeBinaryValue binary code page value.
-     * @return Page code.
-     */
-    private String toCodePage(byte pageCodeBinaryValue) {
-        // From http://trac.osgeo.org/gdal/ticket/2864
-        HashMap<Integer, String> knownConversions = new HashMap<>();
-        knownConversions.put(0x01, "cp437"); //  U.S. MS–DOS
-        knownConversions.put(0x02, "cp850"); // International MS–DOS
-        knownConversions.put(0x03, "cp1252"); // Windows ANSI
-        knownConversions.put(0x08, "cp865"); //  Danish OEM
-        knownConversions.put(0x09, "cp437"); //  Dutch OEM
-        knownConversions.put(0x0a, "cp850"); //  Dutch OEM*
-        knownConversions.put(0x0b, "cp437"); //  Finnish OEM
-        knownConversions.put(0x0d, "cp437"); //  French OEM
-        knownConversions.put(0x0e, "cp850"); //  French OEM*
-        knownConversions.put(0x0f, "cp437"); //  German OEM
-        knownConversions.put(0x10, "cp850"); //  German OEM*
-        knownConversions.put(0x11, "cp437"); //  Italian OEM
-        knownConversions.put(0x12, "cp850"); //  Italian OEM*
-        knownConversions.put(0x13, "cp932"); //  Japanese Shift-JIS
-        knownConversions.put(0x14, "cp850"); //  Spanish OEM*
-        knownConversions.put(0x15, "cp437"); //  Swedish OEM
-        knownConversions.put(0x16, "cp850"); //  Swedish OEM*
-        knownConversions.put(0x17, "cp865"); //  Norwegian OEM
-        knownConversions.put(0x18, "cp437"); //  Spanish OEM
-        knownConversions.put(0x19, "cp437"); //  English OEM (Britain)
-        knownConversions.put(0x1a, "cp850"); //  English OEM (Britain)*
-        knownConversions.put(0x1b, "cp437"); //  English OEM (U.S.)
-        knownConversions.put(0x1c, "cp863"); //  French OEM (Canada)
-        knownConversions.put(0x1d, "cp850"); //  French OEM*
-        knownConversions.put(0x1f, "cp852"); //  Czech OEM
-        knownConversions.put(0x22, "cp852"); //  Hungarian OEM
-        knownConversions.put(0x23, "cp852"); //  Polish OEM
-        knownConversions.put(0x24, "cp860"); //  Portuguese OEM
-        knownConversions.put(0x25, "cp850"); //  Portuguese OEM*
-        knownConversions.put(0x26, "cp866"); //  Russian OEM
-        knownConversions.put(0x37, "cp850"); //  English OEM (U.S.)*
-        knownConversions.put(0x40, "cp852"); //  Romanian OEM
-        knownConversions.put(0x4d, "cp936"); //  Chinese GBK (PRC)
-        knownConversions.put(0x4e, "cp949"); //  Korean (ANSI/OEM)
-        knownConversions.put(0x4f, "cp950"); //  Chinese Big5 (Taiwan)
-        knownConversions.put(0x50, "cp874"); //  Thai (ANSI/OEM)
-        knownConversions.put(0x57, "cp1252"); // ANSI
-        knownConversions.put(0x58, "cp1252"); // Western European ANSI
-        knownConversions.put(0x59, "cp1252"); // Spanish ANSI
-        knownConversions.put(0x64, "cp852"); //  Eastern European MS–DOS
-        knownConversions.put(0x65, "cp866"); //  Russian MS–DOS
-        knownConversions.put(0x66, "cp865"); //  Nordic MS–DOS
-        knownConversions.put(0x67, "cp861"); //  Icelandic MS–DOS
-        knownConversions.put(0x6a, "cp737"); //  Greek MS–DOS (437G)
-        knownConversions.put(0x6b, "cp857"); //  Turkish MS–DOS
-        knownConversions.put(0x6c, "cp863"); //  French–Canadian MS–DOS
-        knownConversions.put(0x78, "cp950"); //  Taiwan Big 5
-        knownConversions.put(0x79, "cp949"); //  Hangul (Wansung)
-        knownConversions.put(0x7a, "cp936"); //  PRC GBK
-        knownConversions.put(0x7b, "cp932"); //  Japanese Shift-JIS
-        knownConversions.put(0x7c, "cp874"); //  Thai Windows/MS–DOS
-        knownConversions.put(0x86, "cp737"); //  Greek OEM
-        knownConversions.put(0x87, "cp852"); //  Slovenian OEM
-        knownConversions.put(0x88, "cp857"); //  Turkish OEM
-        knownConversions.put(0xc8, "cp1250"); // Eastern European Windows
-        knownConversions.put(0xc9, "cp1251"); // Russian Windows
-        knownConversions.put(0xca, "cp1254"); // Turkish Windows
-        knownConversions.put(0xcb, "cp1253"); // Greek Windows
-        knownConversions.put(0xcc, "cp1257"); // Baltic Windows
-        
-        return(knownConversions.get(Byte.toUnsignedInt(pageCodeBinaryValue)));
+    public void loadRowIntoFeature(Feature feature) {
+        binaryReader.loadRowIntoFeature(feature);
     }
-
     /**
-     * Return the database as a {@link java.io.File}.
-     * @return File.
+     * Read the next row as a set of objects.
+     * @return Map of field name / object value.
      */
-    public File getFile() {
-        return(new File(dbfFile));
+    public HashMap<String, Object> readNextRowAsObjects() {
+        return binaryReader.readNextRowAsObjects();
     }
     
     /**
@@ -505,11 +139,7 @@ public class Database extends AbstractAu
      */
     @Override
     public String toString() {
-        return format("toString", System.getProperty("line.separator", "\n"),
-               DbaseVersion, toDate(DbaseLastUpdate), recordCount, fieldsDescriptors, DbaseHeaderBytes,
DbaseRecordBytes, charset);
-        
-        /*StringBuilder s = new StringBuilder();
-        s.append("DbasePlusLanReserved: ").append(DbasePlusLanReserved).append(lineSeparator);
-        */
+        return format("toString", System.getProperty("line.separator", "\n"), 
+                binaryReader.getDateOfLastUpdate(), binaryReader.getFieldsDescriptors(),
binaryReader.getCharset());
     }
 }

Added: sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/MappedByteReader.java
URL: http://svn.apache.org/viewvc/sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/MappedByteReader.java?rev=1647105&view=auto
==============================================================================
--- sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/MappedByteReader.java
(added)
+++ sis/branches/JDK8/storage/sis-shapefile/src/main/java/org/apache/sis/storage/shapefile/MappedByteReader.java
Sun Dec 21 08:19:30 2014
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sis.storage.shapefile;
+
+import java.io.*;
+import java.nio.ByteOrder;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.HashMap;
+import java.util.Objects;
+import java.util.logging.Level;
+
+import org.opengis.feature.Feature;
+
+/**
+ * Reader of a Database Binary content by the way of a {@link java.nio.MappedByteBuffer}
+ * @author Marc LE BIHAN
+ */
+class MappedByteReader extends AbstractByteReader {
+    /** Input Stream on the DBF. */
+    private FileInputStream fis;
+
+    /** File channel on the file. */
+    private FileChannel fc;
+
+    /** Buffer reader. */
+    private MappedByteBuffer df;
+
+    /**
+     * Construct a mapped byte reader on a file.
+     * @param fileName File name.
+     * @throws FileNotFoundException the file name cannot be null.
+     * @throws InvalidDbaseFileFormatException if the database seems to be invalid.
+     */
+    public MappedByteReader(String fileName) throws FileNotFoundException, InvalidDbaseFileFormatException
{
+        Objects.requireNonNull(fileName, "The filename cannot be null.");
+        
+        dbfFile = fileName;
+        fis = new FileInputStream(fileName);
+        fc = fis.getChannel();
+        loadDescriptor();
+    }
+
+    /**
+     * Close the MappedByteReader.
+     * @throws IOException if the close operation fails.
+     */
+    @Override public void close() throws IOException {
+        if (fc != null)
+            fc.close();
+
+        if (fis != null)
+            fis.close();
+    }
+
+    /**
+     * Load a row into a feature.
+     * @param feature Feature to fill.
+     */
+    @Override public void loadRowIntoFeature(Feature feature) {
+        // TODO: ignore deleted records
+        df.get(); // denotes whether deleted or current
+        // read first part of record
+
+        for (FieldDescriptor fd : fieldsDescriptors) {
+            byte[] data = new byte[fd.getLength()];
+            df.get(data);
+
+            int length = data.length;
+            while (length != 0 && data[length - 1] <= ' ') {
+                length--;
+            }
+
+            String value = new String(data, 0, length);
+            feature.setPropertyValue(fd.getName(), value);
+        }
+
+        rowNum ++;
+    }
+
+    /**
+     * Read the next row as a set of objects.
+     * @return Map of field name / object value.
+     */
+    @Override public HashMap<String, Object> readNextRowAsObjects() {
+        // TODO: ignore deleted records
+        byte isDeleted = df.get(); // denotes whether deleted or current
+        // read first part of record
+
+        HashMap<String, Object> fieldsValues = new HashMap<>();
+
+        for (FieldDescriptor fd : fieldsDescriptors) {
+            byte[] data = new byte[fd.getLength()];
+            df.get(data);
+
+            int length = data.length;
+            while (length != 0 && data[length - 1] <= ' ') {
+                length--;
+            }
+
+            String value = new String(data, 0, length);
+            fieldsValues.put(fd.getName(), value);
+        }
+
+        rowNum ++;
+        return fieldsValues;
+    }
+    
+    /**
+     * Loading the database file content from binary .dbf file.
+     * @throws InvalidDbaseFileFormatException if descriptor is not readable. 
+     */
+    private void loadDescriptor() throws InvalidDbaseFileFormatException {
+        try {
+            int fsize = (int) fc.size();
+            df = fc.map(FileChannel.MapMode.READ_ONLY, 0, fsize);
+    
+            this.DbaseVersion = df.get();
+            df.get(this.DbaseLastUpdate);
+    
+            df.order(ByteOrder.LITTLE_ENDIAN);
+            this.recordCount = df.getInt();
+            this.DbaseHeaderBytes = df.getShort();
+            this.DbaseRecordBytes = df.getShort();
+            df.order(ByteOrder.BIG_ENDIAN);
+            
+            df.get(reservedFiller1);
+            this.reservedIncompleteTransaction = df.get();
+            this.reservedEncryptionFlag = df.get();
+            df.get(reservedFreeRecordThread);
+            df.get(reservedMultiUser);
+            reservedMDXFlag = df.get();
+            
+            // Translate code page value to a known charset.
+            this.codePage = df.get();
+            this.charset = toCharset(this.codePage);             
+            
+            df.get(reservedFiller2); 
+    
+            while(df.position() < this.DbaseHeaderBytes - 1) {
+                FieldDescriptor fd = toFieldDescriptor();
+                this.fieldsDescriptors.add(fd);
+                // loop until you hit the 0Dh field terminator
+            }
+            
+            this.descriptorTerminator = df.get();
+
+            // If the last character read after the field descriptor isn't 0x0D, the expected
mark has not been found and the DBF is corrupted.
+            if (descriptorTerminator != 0x0D) {
+                String message = format(Level.SEVERE, "excp.filedescriptor_problem", new
File(dbfFile).getAbsolutePath(), "Character marking the end of the fields descriptors (0x0D)
has not been found.");
+                throw new InvalidDbaseFileFormatException(message);
+            }
+        }
+        catch(IOException e) {
+            // This exception doesn't denote a trouble of file opening because the file has
been checked before 
+            // the calling of this private function.
+            // Therefore, an internal structure problem cause maybe a premature End of file
or anything else, but the only thing
+            // we can conclude is : we are not before a device trouble, but a file format
trouble.
+            String message = format(Level.SEVERE, "excp.filedescriptor_problem", new File(dbfFile).getAbsolutePath(),
e.getMessage());
+            throw new InvalidDbaseFileFormatException(message);
+        }
+    }
+
+    /**
+     * Create a field descriptor from the current position of the binary stream.
+     * @return FieldDescriptor or null if there is no more available.
+     */
+    private FieldDescriptor toFieldDescriptor() {
+        // If there is no more field description available, return null.
+        if (df.position() >= this.DbaseHeaderBytes - 1)
+            return null;
+
+        FieldDescriptor fd = new FieldDescriptor();
+
+        // Field name.
+        df.get(fd.FieldName);
+
+        // Field type.
+        char dt = (char) df.get();
+        fd.FieldType = DataType.valueOfDataType(dt);
+
+        // Field address.
+        df.get(fd.FieldAddress);
+
+        // Length and scale.
+        fd.FieldLength = df.get();
+        fd.FieldDecimalCount = df.get();
+
+        df.getShort(); // reserved
+
+        df.get(fd.DbasePlusLanReserved2);
+
+        // Work area id.
+        fd.WorkAreaID = df.get();
+
+        df.get(fd.DbasePlusLanReserved3);
+
+        // Fields.
+        fd.SetFields = df.get();
+
+        byte[] data = new byte[6];
+        df.get(data); // reserved
+
+        return fd;
+    }
+}

Modified: sis/branches/JDK8/storage/sis-shapefile/src/main/resources/org/apache/sis/storage/shapefile/Database.properties
URL: http://svn.apache.org/viewvc/sis/branches/JDK8/storage/sis-shapefile/src/main/resources/org/apache/sis/storage/shapefile/Database.properties?rev=1647105&r1=1647104&r2=1647105&view=diff
==============================================================================
--- sis/branches/JDK8/storage/sis-shapefile/src/main/resources/org/apache/sis/storage/shapefile/Database.properties
[ISO-8859-1] (original)
+++ sis/branches/JDK8/storage/sis-shapefile/src/main/resources/org/apache/sis/storage/shapefile/Database.properties
[ISO-8859-1] Sun Dec 21 08:19:30 2014
@@ -20,11 +20,8 @@ assert.wrong_charset_selection=[Internal
 
 # Database:toString() 
 #0 : Line separator.
-#1 : Database Version.
-#2 : Last update (Date).
-#3 : Feature count (Integer).
-#4 : Field descriptor (Embedded List).
-#5 : Header length (in bytes).
-#6 : Record length (in bytes).
-#7 : Charset (converted from binary code page value).
-toString='{'{0}DBase version : {1}{0}Last update : {2,date,medium}{0}Feature count : {3,number,integer}{0}Field
descriptor : {4}{0}Header length : {5}, Record length : {6}, Charset : {7}'}'
+#1 : Last update (Date).
+#2 : Feature count (Integer).
+#3 : Field descriptor (Embedded List).
+#4 : Charset (converted from binary code page value).
+toString='{'{0}Last update : {1,date,medium}{0}Feature count : {2,number,integer}{0}Field
descriptor : {3}, Charset : {4}'}'



Mime
View raw message