indexedshapefiledatastoretest.java

shape file read and write

/*
 *    GeoTools - OpenSource mapping toolkit
 *    http://geotools.org
 *    (C) 2002-2006, GeoTools Project Managment Committee (PMC)
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License as published by the Free Software Foundation;
 *    version 2.1 of the License.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 */
package org.geotools.data.shapefile.indexed;

import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.nio.charset.UnsupportedCharsetException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;

import org.geotools.TestData;
import org.geotools.data.DataUtilities;
import org.geotools.data.DefaultQuery;
import org.geotools.data.DefaultTransaction;
import org.geotools.data.FeatureSource;
import org.geotools.data.FeatureStore;
import org.geotools.data.FeatureWriter;
import org.geotools.data.Query;
import org.geotools.data.Transaction;
import org.geotools.data.shapefile.ShapefileDataStore;
import org.geotools.data.shapefile.ShpFileType;
import org.geotools.data.shapefile.TestCaseSupport;
import org.geotools.factory.CommonFactoryFinder;
import org.geotools.factory.FactoryRegistryException;
import org.geotools.feature.FeatureCollection;
import org.geotools.feature.FeatureCollections;
import org.geotools.feature.FeatureIterator;
import org.geotools.feature.simple.SimpleFeatureBuilder;
import org.geotools.feature.simple.SimpleFeatureTypeBuilder;
import org.geotools.filter.IllegalFilterException;
import org.geotools.geometry.jts.ReferencedEnvelope;
import org.opengis.feature.simple.SimpleFeature;
import org.opengis.feature.simple.SimpleFeatureType;
import org.opengis.feature.type.AttributeDescriptor;
import org.opengis.filter.Filter;
import org.opengis.filter.FilterFactory2;
import org.opengis.geometry.BoundingBox;
import org.opengis.referencing.crs.CoordinateReferenceSystem;

import com.vividsolutions.jts.geom.Coordinate;
import com.vividsolutions.jts.geom.Envelope;
import com.vividsolutions.jts.geom.Geometry;
import com.vividsolutions.jts.geom.GeometryCollection;
import com.vividsolutions.jts.geom.GeometryFactory;
import com.vividsolutions.jts.geom.Point;

/**
 * 
 * @source $URL:
 *         http://svn.geotools.org/geotools/branches/shpLazyLoadingIndex/ext/shape/test/org/geotools/data/shapefile/indexed/ShapefileDataStoreTest.java $
 * @version $Id: IndexedShapefileDataStoreTest.java 29672 2008-03-19 18:33:05Z desruisseaux $
 * @author Ian Schneider
 */
public class IndexedShapefileDataStoreTest extends TestCaseSupport {
    final static String STATE_POP = "shapes/statepop.shp";

    final static String STREAM = "shapes/stream.shp";

    final static String DANISH = "shapes/danish_point.shp";

    final static String CHINESE = "shapes/chinese_poly.shp";

    public IndexedShapefileDataStoreTest(String testName) throws IOException {
        super(testName);
    }

    protected FeatureCollection<SimpleFeatureType, SimpleFeature> loadFeatures(String resource, Query q)
            throws Exception {
        if (q == null) {
            q = new DefaultQuery();
        }

        URL url = TestData.url(resource);
        IndexedShapefileDataStore s = new IndexedShapefileDataStore(url);
        FeatureSource<SimpleFeatureType, SimpleFeature> fs = s.getFeatureSource(s.getTypeNames()[0]);

        FeatureCollection<SimpleFeatureType, SimpleFeature> features = fs.getFeatures(q);

        s.dispose();
        
        return features;
    }

    protected FeatureCollection<SimpleFeatureType, SimpleFeature> loadFeatures(String resource, Charset charset,
            Query q) throws Exception {
        if (q == null)
            q = new DefaultQuery();
        URL url = TestData.url(resource);
        ShapefileDataStore s = new IndexedShapefileDataStore(url, null, false,
                true, IndexType.QIX, charset);
        FeatureSource<SimpleFeatureType, SimpleFeature> fs = s.getFeatureSource(s.getTypeNames()[0]);
        FeatureCollection<SimpleFeatureType, SimpleFeature> features = fs.getFeatures(q);
        s.dispose();
        return features;
    }

    protected FeatureCollection<SimpleFeatureType, SimpleFeature> loadFeatures(IndexedShapefileDataStore s)
            throws Exception {
        return s.getFeatureSource(s.getTypeNames()[0]).getFeatures();
    }

    public void testLoad() throws Exception {
        loadFeatures(STATE_POP, null);
    }

    public void testLoadDanishChars() throws Exception {
        FeatureCollection<SimpleFeatureType, SimpleFeature> fc = loadFeatures(DANISH, null);
        SimpleFeature first = firstFeature(fc);
        // Charl�tte, if you can read it with your OS charset
        assertEquals("Charl\u00F8tte", first.getAttribute("TEKST1"));
    }

    public void testLoadChineseChars() throws Exception {
        try {
            FeatureCollection<SimpleFeatureType, SimpleFeature> fc = loadFeatures(CHINESE, Charset
                    .forName("GB18030"), null);
            SimpleFeature first = firstFeature(fc);
            String name = (String) first.getAttribute("NAME");
            assertEquals("\u9ed1\u9f99\u6c5f\u7701", name);
        } catch (UnsupportedCharsetException no) {
            // this JDK has not been installed with the required
            // lanaguage
        }
    }

    public void testSchema() throws Exception {
        URL url = TestData.url(STATE_POP);
        IndexedShapefileDataStore s = new IndexedShapefileDataStore(url);
        SimpleFeatureType schema = s.getSchema(s.getTypeNames()[0]);
        List<AttributeDescriptor> types = schema.getAttributes();
        assertEquals("Number of Attributes", 253, types.size());
        assertNotNull(schema.getCRS());
    }

    public void testSpacesInPath() throws Exception {
        URL u = TestData.url(TestCaseSupport.class, "folder with spaces/pointtest.shp");
        File f = new File(URLDecoder.decode(u.getFile(), "UTF-8"));
        assertTrue(f.exists());

        IndexedShapefileDataStore s = new IndexedShapefileDataStore(u);
        loadFeatures(s);
        s.dispose();
    }

    /**
     * Test envelope versus old DataSource
     */
    public void testEnvelope() throws Exception {
        FeatureCollection<SimpleFeatureType, SimpleFeature> features = loadFeatures(STATE_POP, null);
        testEnvelope(features, IndexType.QIX);
        testEnvelope(features, IndexType.NONE);
    }

    private void testEnvelope(FeatureCollection<SimpleFeatureType, SimpleFeature> features, IndexType treeType)
            throws MalformedURLException, IOException {
        IndexedShapefileDataStore s = new IndexedShapefileDataStore(TestData
                .url(STATE_POP), null, true, true, treeType);
        String typeName = s.getTypeNames()[0];
        FeatureCollection<SimpleFeatureType, SimpleFeature> all = s.getFeatureSource(typeName).getFeatures();

        assertEquals(features.getBounds(), all.getBounds());
        s.dispose();
    }

    public void testCreateAndReadQIX() throws Exception {
        File shpFile = copyShapefiles(STATE_POP);
        URL url = shpFile.toURL();
        String filename = url.getFile();
        filename = filename.substring(0, filename.lastIndexOf("."));

        File file = new File(filename + ".qix");

        if (file.exists()) {
            file.delete();
        }
        file.deleteOnExit();

        IndexedShapefileDataStore ds = new IndexedShapefileDataStore(url, null,
                true, true, IndexType.QIX);
        FeatureCollection<SimpleFeatureType, SimpleFeature> features = ds.getFeatureSource().getFeatures();
        FeatureIterator<SimpleFeature> indexIter = features.features();

        GeometryFactory factory = new GeometryFactory();
        double area = Double.MAX_VALUE;
        SimpleFeature smallestFeature = null;
        while (indexIter.hasNext()) {
            SimpleFeature newFeature = indexIter.next();

            BoundingBox bounds = newFeature.getBounds();
            Geometry geometry = factory.toGeometry(new ReferencedEnvelope(
                    bounds));
            double newArea = geometry.getArea();

            if (smallestFeature == null || newArea < area) {
                smallestFeature = newFeature;
                area = newArea;
            }
        }
        indexIter.close();

        IndexedShapefileDataStore ds2 = new IndexedShapefileDataStore(url,
                null, false, false, IndexType.NONE);

        Envelope newBounds = ds.getBounds(Query.ALL);
        double dx = newBounds.getWidth() / 4;
        double dy = newBounds.getHeight() / 4;
        newBounds = new Envelope(newBounds.getMinX() + dx, newBounds.getMaxX()
                - dx, newBounds.getMinY() + dy, newBounds.getMaxY() - dy);

        CoordinateReferenceSystem crs = features.getSchema().getCRS();

        performQueryComparison(ds, ds2, new ReferencedEnvelope(newBounds, crs));
        performQueryComparison(ds, ds2, new ReferencedEnvelope(smallestFeature
                .getBounds()));

        assertTrue(file.exists());
        ds.dispose();
        ds2.dispose();
    }

    private ArrayList performQueryComparison(
            IndexedShapefileDataStore indexedDS,
            IndexedShapefileDataStore baselineDS, ReferencedEnvelope newBounds)
            throws FactoryRegistryException, IllegalFilterException,
            IOException {
        FeatureCollection<SimpleFeatureType, SimpleFeature> features;
        FeatureIterator<SimpleFeature> indexIter;
        FilterFactory2 fac = CommonFactoryFinder.getFilterFactory2(null);
        String geometryName = indexedDS.getSchema().getDefaultGeometry()
                .getLocalName();

        Filter filter = fac.bbox(fac.property(geometryName), newBounds);

        features = indexedDS.getFeatureSource().getFeatures(filter);
        FeatureCollection<SimpleFeatureType, SimpleFeature> features2 = baselineDS.getFeatureSource()
                .getFeatures(filter);

        FeatureIterator<SimpleFeature> baselineIter = features2.features();
        indexIter = features.features();

        ArrayList baselineFeatures = new ArrayList();
        ArrayList indexedFeatures = new ArrayList();

        try {
            while (baselineIter.hasNext()) {
                baselineFeatures.add(baselineIter.next());
            }
            while (indexIter.hasNext()) {
                indexedFeatures.add(indexIter.next());
            }
            assertFalse(indexIter.hasNext());
            assertFalse(baselineIter.hasNext());
            assertEquals(baselineFeatures, indexedFeatures);
        } finally {
            indexIter.close();
            baselineIter.close();
        }
        return indexedFeatures;
    }
//
//    public void testCreateAndReadGRX() throws Exception {
//        URL url = TestData.url(STATE_POP);
//        String filename = url.getFile();
//        filename = filename.substring(0, filename.lastIndexOf("."));
//
//        File file = new File(filename + ".grx");
//
//        if (file.exists()) {
//            file.delete();
//        }
//
//        IndexedShapefileDataStore ds = new IndexedShapefileDataStore(url, null,
//                true, true, IndexType.EXPERIMENTAL_UNSUPPORTED_GRX);
//        FeatureCollection<SimpleFeatureType, SimpleFeature> features = ds.getFeatureSource().getFeatures();
//        Iterator iter = features.iterator();
//
//        while (iter.hasNext()) {
//            iter.next();
//        }
//
//        // TODO: The following assertion fails
//        // assertTrue(file.exists());
//    }

    public void testLoadAndVerify() throws Exception {
        FeatureCollection<SimpleFeatureType, SimpleFeature> features = loadFeatures(STATE_POP, null);

        int count = features.size();
        assertTrue("Got Features", count > 0);
        // assertEquals("Number of Features loaded", 49, count); // FILE CORRECT
        // assertEquals("Number of Features loaded", 3, count); // JAR WRONG

        SimpleFeatureType schema = firstFeature(features).getFeatureType();
        assertNotNull(schema.getDefaultGeometry());
        assertEquals("Number of Attributes", 253, schema.getAttributeCount());
        assertEquals("Value of statename is wrong", firstFeature(features)
                .getAttribute("STATE_NAME"), "Illinois");
        assertEquals("Value of land area is wrong", ((Double) firstFeature(
                features).getAttribute("LAND_KM")).doubleValue(), 143986.61,
                0.001);
    }

    private IndexedShapefileDataStore createDataStore(File f) throws Exception {
        FeatureCollection<SimpleFeatureType, SimpleFeature> fc = createFeatureCollection();
        f.createNewFile();

        IndexedShapefileDataStore sds = new IndexedShapefileDataStore(f.toURL());
        writeFeatures(sds, fc);

        return sds;
    }

    private IndexedShapefileDataStore createDataStore() throws Exception {
        return createDataStore(getTempFile());
    }

    /**
     * Create a set of features, then remove every other one, updating the
     * remaining. Test for removal and proper update after reloading...
     */
    public void testUpdating() throws Throwable {
        try {
            IndexedShapefileDataStore sds = createDataStore();
            loadFeatures(sds);

            FeatureWriter<SimpleFeatureType, SimpleFeature> writer = null;

            try {