filesetdocumentcollection.java

MG4J (Managing Gigabytes for Java) is a free full-text search engine for large document collections

package it.unimi.dsi.mg4j.document;

/*		 
 * MG4J: Managing Gigabytes for Java
 *
 * Copyright (C) 2005-2007 Paolo Boldi and Sebastiano Vigna 
 *
 *  This library is free software; you can redistribute it and/or modify it
 *  under the terms of the GNU Lesser General Public License as published by the Free
 *  Software Foundation; either version 2.1 of the License, or (at your option)
 *  any later version.
 *
 *  This library is distributed in the hope that it will be useful, but
 *  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 *  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
 *  for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 */

import it.unimi.dsi.fastutil.io.BinIO;
import it.unimi.dsi.fastutil.objects.ObjectArrayList;
import it.unimi.dsi.fastutil.objects.Reference2ObjectArrayMap;
import it.unimi.dsi.fastutil.objects.Reference2ObjectMap;
import it.unimi.dsi.io.FileLinesCollection;
import it.unimi.dsi.io.NullInputStream;
import it.unimi.dsi.lang.MutableString;
import it.unimi.dsi.mg4j.document.PropertyBasedDocumentFactory.MetadataKeys;
import it.unimi.dsi.mg4j.util.MG4JClassParser;
import it.unimi.dsi.mg4j.util.MimeTypeResolver;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Serializable;
import java.lang.reflect.InvocationTargetException;
import java.util.Collection;

import com.martiansoftware.jsap.FlaggedOption;
import com.martiansoftware.jsap.JSAP;
import com.martiansoftware.jsap.JSAPException;
import com.martiansoftware.jsap.JSAPResult;
import com.martiansoftware.jsap.Parameter;
import com.martiansoftware.jsap.SimpleJSAP;
import com.martiansoftware.jsap.UnflaggedOption;

/** A {@link it.unimi.dsi.mg4j.document.DocumentCollection} corresponding to
 *  a given set of files.
 * 
 * <P>This class provides a main method with a flexible syntax that serialises
 * into a document collection a list of files given on the command line or
 * piped into standard input. Optionally, you can provide a parallel list of URIs
 * that will be associated to each file.
 */
public class FileSetDocumentCollection extends AbstractDocumentCollection implements Serializable {

	private static final long serialVersionUID = 0L;
	
	/** The files in this collection. */
	private final String[] file;
	/** URIs for each file in this collection, or <code>null</code>, in which case the filename will be used as URI. */
	private final String[] uri;
	/** The factory to be used by this collection. */
	private final DocumentFactory factory;
	/** The last returned file input stream. */
	private InputStream last = NullInputStream.getInstance();

	/** Builds a document collection corresponding to a given set of files specified as an array.
	 * 
	 *  <p><strong>Beware.</strong> This class is not guaranteed to work if files are
	 *  deleted or modified after creation!
	 * 
	 * @param file an array containing the files that will be contained in the collection.
	 * @param factory the factory that will be used to create documents.
	 */
	public FileSetDocumentCollection( final String[] file, final DocumentFactory factory ) {
		this( file, null, factory );
	}
	
	/** Builds a document collection corresponding to a given set of files specified as an array and
	 * a parallel array of URIs, one for each file.
	 * 
	 *  <p><strong>Beware.</strong> This class is not guaranteed to work if files are
	 *  deleted or modified after creation!
	 * 
	 * @param file an array containing the files that will be contained in the collection.
	 * @param uri an array, parallel to <code>file</code>, containing URIs to be associated to each element of <code>file</code>.
	 * @param factory the factory that will be used to create documents.
	 */
	public FileSetDocumentCollection( final String[] file, final String uri[], final DocumentFactory factory ) {
		this.file = file;
		this.uri = uri;
		this.factory = factory;
	}

	public DocumentFactory factory() {
		return factory;
	}
	
	public int size() {
		return file.length;
	}

	public Reference2ObjectMap<Enum<?>,Object> metadata( final int index ) {
		ensureDocumentIndex( index );
		final Reference2ObjectArrayMap<Enum<?>,Object> metadata = new Reference2ObjectArrayMap<Enum<?>,Object>( 2 );
		metadata.put( MetadataKeys.TITLE, file[ index ] );
		if ( uri != null ) metadata.put( MetadataKeys.URI, uri[ index ] );
		else metadata.put( MetadataKeys.URI, new File( file[ index ] ).toURI().toString() );
		metadata.put( MetadataKeys.MIMETYPE, MimeTypeResolver.getContentType( file[ index ] ) );
		return metadata;
	}

	public Document document( final int index ) throws IOException {
		return factory.getDocument( stream( index ), metadata( index ) );
	}
	
	public InputStream stream( final int index ) throws IOException {
		ensureDocumentIndex( index );
		last.close();
		return last = new FileInputStream( file[ index ] );
	}
	
	public FileSetDocumentCollection copy() {
		return new FileSetDocumentCollection( file, uri, factory.copy() ); 
	}
	
	public void close() throws IOException {
		last.close();
		super.close();
	}
	
	public static void main( final String[] arg ) throws IOException, JSAPException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {

		SimpleJSAP jsap = new SimpleJSAP( FileSetDocumentCollection.class.getName(), "Saves a serialised document collection based on a set of files.",
				new Parameter[] {
					new FlaggedOption( "factory", MG4JClassParser.getParser(), IdentityDocumentFactory.class.getName(), JSAP.NOT_REQUIRED, 'f', "factory", "A document factory with a standard constructor." ),
					new FlaggedOption( "property", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'p', "property", "A 'key=value' specification, or the name of a property file" ).setAllowMultipleDeclarations( true ),
					new FlaggedOption( "uris", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'u', "uris", "A file containing a list of URIs in ASCII encoding, one per line, that will be associated to each file" ),
					new UnflaggedOption( "collection", JSAP.STRING_PARSER, JSAP.REQUIRED, "The filename for the serialised collection." ),
					new UnflaggedOption( "file", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, JSAP.GREEDY, "A list of files that will be indexed. If missing, a list of files will be read from standard input." )
				}
		);
		

		JSAPResult jsapResult = jsap.parse( arg );
		if ( jsap.messagePrinted() ) return;
		
		String uri[] = null;
		if ( jsapResult.getString( "uris" ) != null ) {
			Collection<MutableString> lines = new FileLinesCollection( jsapResult.getString( "uris" ), "ASCII" ).allLines();
			uri = new String[ lines.size() ];
			int i = 0;
			for( Object l: lines ) uri[ i++ ] = l.toString();
		}
		
		final DocumentFactory factory = PropertyBasedDocumentFactory.getInstance( jsapResult.getClass( "factory" ), jsapResult.getStringArray( "property" ) ); 
		
		String[] file = (String[])jsapResult.getObjectArray( "file", new String[ 0 ] );
		if ( file.length == 0 ) {
			final ObjectArrayList<String> files = new ObjectArrayList<String>();
			BufferedReader bufferedReader = new BufferedReader( new InputStreamReader( System.in ) );
			String s;
			while( ( s = bufferedReader.readLine() ) != null ) files.add( s );
			file = files.toArray( new String[ 0 ] );
		}
		
		if ( file.length == 0 ) System.err.println( "WARNING: empty file set." );
		if ( uri != null && file.length != uri.length ) throw new IllegalArgumentException( "The number of files (" + file.length + ") and the number of URIs (" + uri.length + ") differ" );
		BinIO.storeObject( new FileSetDocumentCollection( file, uri, factory ), jsapResult.getString( "collection" ) );
	}
}