sesameupload.java

这是外国一个开源推理机

/*  Sesame - Storage and Querying architecture for RDF and RDF Schema
 *  Copyright (C) 2001-2005 Aduna
 *
 *  Contact: 
 *  	Aduna
 *  	Prinses Julianaplein 14 b
 *  	3817 CS Amersfoort
 *  	The Netherlands
 *  	tel. [phone]
 *  	fax. [phone]
 *
 *  	http://aduna.biz/
 *  	http://www.openrdf.org/
 *  
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2.1 of the License, or (at your option) any later version.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

package org.openrdf.sesame.admin;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;

import org.openrdf.util.io.IOUtil;
import org.openrdf.util.log.ThreadLog;

import org.openrdf.model.Resource;
import org.openrdf.model.URI;
import org.openrdf.model.Value;
import org.openrdf.model.ValueFactory;
import org.openrdf.model.impl.StatementImpl;
import org.openrdf.model.impl.ValueFactoryImpl;

import org.openrdf.rio.ParseException;
import org.openrdf.rio.Parser;
import org.openrdf.rio.StatementHandlerException;
import org.openrdf.rio.ntriples.NTriplesParser;
import org.openrdf.rio.rdfxml.RdfXmlParser;
import org.openrdf.rio.turtle.TurtleParser;

import org.openrdf.sesame.constants.RDFFormat;
import org.openrdf.sesame.repository.local.LocalService;
import org.openrdf.sesame.sail.RdfRepository;
import org.openrdf.sesame.sail.SailInternalException;
import org.openrdf.sesame.sail.SailUpdateException;

public class SesameUpload extends Upload {

	private AdminListener _report;
	private LocalService _service;

	/**
	 * Creates a new SesameUpload that will add any data to the supplied
	 * RdfRepository.
	 *
	 * @exception IllegalArgumentException If the supplied rdfRepository is
	 * equal to null.
	 **/
	public SesameUpload(RdfRepository rdfRepository, LocalService service) {
		super(rdfRepository);
		_service = service;
	}

	/**
	 * Reads and parses RDF data from the supplied InputStream and adds the
	 * triples to the RdfRepository of this SesameUpload object.
	 *
	 * @return the number of processed statements
	 **/
	public synchronized int addRdfModel(InputStream inputStream, String baseURL,
		AdminListener report, RDFFormat dataFormat, boolean verifyData)
		throws IOException, UpdateException
	{
		_report = report;
		File tmpFile = null;
		byte[] data = null;
		int statementCount = 0;

		if (verifyData) {
			// Try storing the data to disk to prevent double downloads, use
			// main memory as backup in case no tmp file could be created.
			try {
				if (_service != null) {
					tmpFile = _service.createTmpFile("upload", ".rdf");
				}
				else {
					ThreadLog.error("Unable to create tmp file: could not init service");
				}
			}
			catch (IOException e) {
				ThreadLog.error("Unable to create tmp file", e);
			}

			_report.status("Loading data", -1, -1);
			ThreadLog.trace("Loading data");

			int totalBytes = 0;
			if (tmpFile != null) {
				// Store data to disk
				totalBytes = IOUtil.writeToFile(inputStream, tmpFile);
				inputStream.close();
			}
			else {
				// Store data in memory
				_report.warning("Unable to store data in tmp file, attempting to store it in main memory", -1, -1, null);
				data = IOUtil.readFully(inputStream);
				inputStream.close();
			}

			_report.status("Data loaded (" + _nf.format(totalBytes) + " bytes)", -1, -1);

			// Open an InputStream on the cached data for verification
			if (tmpFile != null) {
				inputStream = new FileInputStream(tmpFile);
			} else {
				inputStream = new ByteArrayInputStream(data);
			}

			// Verify data
			try {
				statementCount = _verifyData(inputStream, baseURL, dataFormat);
			}
			finally {
				inputStream.close();
			}

			if (statementCount == 0) {
				_report.status("Data does not contain any statements", -1, -1);
				ThreadLog.trace("No statements found");
				return 0;
			}

			String msg = "Data is correct and contains " + _nf.format(statementCount) + " statements";
			_report.status(msg, -1, -1);
			ThreadLog.trace(msg);

			// Open an InputStream on the cached data for the actual processing
			if (tmpFile != null) {
				inputStream = new FileInputStream(tmpFile);
			} else {
				inputStream = new ByteArrayInputStream(data);
			}
		}

		// Process the data
		try {
			statementCount = _processStatements(inputStream, baseURL, dataFormat);
		}
		finally {
			inputStream.close();
		}

		// Delete the tmp file.
		if (tmpFile != null) {
			tmpFile.delete();
		}

		return statementCount;
	}

	/**
	 * Reads and parses RDF data from the supplied Reader and adds the
	 * triples to the RdfRepository of this SesameUpload object.
	 *
	 * @return the number of processed statements
	 **/
	public synchronized int addRdfModel(Reader reader, String baseURL,
		AdminListener report, RDFFormat dataFormat, boolean verifyData)
		throws IOException, UpdateException
	{
		_report = report;
		File tmpFile = null;
		String data = null;
		int statementCount = 0;

		if (verifyData) {
			// Try storing the data to disk to prevent double downloads, use
			// main memory as backup in case no tmp file could be created.
			try {
				if (_service != null) {
					tmpFile = _service.createTmpFile("upload", ".rdf");
				}
				else {
					ThreadLog.error("Unable to create tmp file: could not init service");
				}
			}
			catch (IOException e) {
				ThreadLog.error("Unable to create tmp file", e);
			}

			_report.status("Loading data", -1, -1);
			ThreadLog.trace("Loading data");

			int totalChars = 0;
			if (tmpFile != null) {
				try {
					FileOutputStream out = new FileOutputStream(tmpFile);
					Writer writer = new OutputStreamWriter(out, "UTF-8");
					totalChars = IOUtil.transfer(reader, writer);
					reader.close();
					writer.close();
				}
				catch (UnsupportedEncodingException e) {
					// UTF-8 should be supported on all platforms...
					ThreadLog.error("Unable to write data to tmp file using UTF-8", e);
					throw new RuntimeException(e);
				}
			}
			else {
				// Store data in memory
				_report.warning("Unable to store data in tmp file, attempting to store it in main memory", -1, -1, null);
				data = IOUtil.readFully(reader);
				reader.close();
			}

			_report.status("Data loaded (" + _nf.format(totalChars) + " characters)", -1, -1);

			// Open a Reader on the cached data for verification
			if (tmpFile != null) {
				try {
					FileInputStream in = new FileInputStream(tmpFile);
					reader = new InputStreamReader(in, "UTF-8");
				}
				catch (UnsupportedEncodingException e) {
					// UTF-8 should be supported on all platforms...
					ThreadLog.error("Unable to read data from tmp file using UTF-8", e);
					throw new RuntimeException(e);
				}
			}
			else {
				reader = new StringReader(data);
			}

			// Verify data
			statementCount = _verifyData(reader, baseURL, dataFormat);
			reader.close();

			if (statementCount == 0) {
				_report.status("Data does not contain any statements", -1, -1);
				ThreadLog.trace("No statements found");
				return 0;
			}

			String msg = "Data is correct and contains " + _nf.format(statementCount) + " statements";
			_report.status(msg, -1, -1);
			ThreadLog.trace(msg);

			// Open a Reader on the cached data for the actual processing
			if (tmpFile != null) {
				try {
					FileInputStream in = new FileInputStream(tmpFile);
					reader = new InputStreamReader(in, "UTF-8");
				}
				catch (UnsupportedEncodingException e) {
					// UTF-8 should be supported on all platforms...
					ThreadLog.error("Unable to read data from tmp file using UTF-8", e);
					throw new RuntimeException(e);
				}
			}
			else {
				reader = new StringReader(data);
			}
		}

		// Process the data
		statementCount = _processStatements(reader, baseURL, dataFormat);
		reader.close();

		// Delete the tmp file.
		if (tmpFile != null) {
			tmpFile.delete();
		}

		return statementCount;
	}

	// Returns the number of statements found in the data.
	private int _verifyData(InputStream inputStream, String baseURL, RDFFormat dataFormat)
		throws UpdateException, IOException
	{
		_report.status("Checking data for errors", -1, -1);
		ThreadLog.trace("Checking data for errors");

		// The RDF Parser
		Parser rdfParser = _createParser(dataFormat, baseURL, new ValueFactoryImpl());
		rdfParser.setVerifyData(true);
		rdfParser.setStopAtFirstError(false);
		rdfParser.setDatatypeHandling(Parser.DT_VERIFY);

		// The StatementHandler and ErrorListener
		ErrorHandler errorHandler = new ErrorHandler();
		rdfParser.setStatementHandler(errorHandler);
		rdfParser.setParseErrorListener(errorHandler);

		try {
			rdfParser.parse(inputStream, baseURL);
		}
		catch (IOException e) {
			ThreadLog.warning("unable to verify data", e);
			throw new UpdateException(e);
		}
		catch (ParseException e) {
			throw new UpdateException(e);
		}
		catch (StatementHandlerException e) {
			throw new UpdateException(e);
		}

		int errorCount = errorHandler.getErrorCount();
		int statementCount = errorHandler.getStatementCount();

		// Check number of errors
		if (errorCount > 0) {
			throw new UpdateException(errorCount + " errors found, please fix these first");
		}