bitstreamindexreader.c

MG4J (Managing Gigabytes for Java) is a free full-text search engine for large document collections

			if ( ASSERTS ) ensureCurrentDocument();
			return currentDocument;
		}

		public Payload payload() throws IOException {
			if ( DEBUG ) System.err.println( this + ".payload()" );
			if ( ASSERTS ) ensureCurrentDocument();
#if GENERIC
			if ( ! hasPayloads )
#endif
#if GENERIC || ! PAYLOADS
				throw new UnsupportedOperationException( "This index ("+ index + ") does not contain payloads" );
#endif

#if GENERIC || PAYLOADS
			if ( state <= BEFORE_PAYLOAD ) {
#if GENERIC || SKIPS
				if ( state == BEFORE_TOWER ) readTower();
#endif

				payload.read( ibs );

#if GENERIC
				state = hasCounts ? BEFORE_COUNT : BEFORE_POINTER;
#elif #counts(NONE)
				state = BEFORE_POINTER;
#else
				state = BEFORE_COUNT;
#endif
			}
			return payload;
#endif
		}
		
		public int count() throws IOException {
			if ( DEBUG ) System.err.println( this + ".count()" );
			if ( count != -1 ) return count;
			if ( ASSERTS ) ensureCurrentDocument();
#if GENERIC
			if ( ! hasCounts ) 
#endif
#if GENERIC || #counts(NONE)
				throw new UnsupportedOperationException( "This index (" + index + ") does not contain counts" );
#endif

#if GENERIC || ! #counts(NONE)
#if GENERIC || SKIPS
			if ( state == BEFORE_TOWER ) readTower();
#endif
#if GENERIC || PAYLOADS
			if ( state == BEFORE_PAYLOAD ) payload.read( ibs ); 
#endif

		{
#if GENERIC
			if ( ASSERTS && state != BEFORE_COUNT ) throw new IllegalStateException();
			state = hasPositions ? BEFORE_POSITIONS : BEFORE_POINTER;
#elif ! #positions(NONE)
			state = BEFORE_POSITIONS;
#else
			state = BEFORE_POINTER;
#endif
			
#if GENERIC	
			switch( countCoding ) {
			case UNARY: 
#endif
#if GENERIC || #counts(UNARY)
				count = ibs.readUnary() + 1;
#endif
#if GENERIC	
				break;
			case SHIFTED_GAMMA: 
#endif
#if GENERIC || #counts(SHIFTED_GAMMA)
				count = ibs.readShiftedGamma() + 1;
#endif
#if GENERIC	
				break;
			case GAMMA: 
#endif
#if GENERIC || #counts(GAMMA)
				count = ibs.readGamma() + 1;
#endif
#if GENERIC	
				break;
			case DELTA: 
#endif
#if GENERIC || #counts(DELTA)
				count = ibs.readDelta() + 1;
#endif
#if GENERIC	
				break;
			default: throw new IllegalStateException( "The required count coding (" + countCoding + ") is not supported." );
			}
#endif
		}

			return count;
#endif
		}
		
		

		/** We read positions, assuming state <= BEFORE_POSITIONS */
		@SuppressWarnings("unused")
		protected void updatePositionCache() throws IOException {
			if ( ASSERTS ) assert state <= BEFORE_POSITIONS;
#if GENERIC
			if ( ! hasPositions ) 
#endif
#if GENERIC || #positions(NONE)
				throw new UnsupportedOperationException( "This index(" + index + ") does not contain positions" );
#endif
#if ! #positions(NONE)
			if ( state < BEFORE_POSITIONS ) {
#if GENERIC || SKIPS
				if ( state == BEFORE_TOWER ) readTower();
#endif
#if GENERIC || PAYLOADS
				if ( state == BEFORE_PAYLOAD ) payload.read( ibs );
#endif
				if ( state == BEFORE_COUNT )
		{
#if GENERIC
			if ( ASSERTS && state != BEFORE_COUNT ) throw new IllegalStateException();
#elif ! #positions(NONE)
			state = BEFORE_POSITIONS;
#else
			state = BEFORE_POINTER;
#endif
			
#if GENERIC	
			switch( countCoding ) {
			case UNARY: 
#endif
#if GENERIC || #counts(UNARY)
				count = ibs.readUnary() + 1;
#endif
#if GENERIC	
				break;
			case SHIFTED_GAMMA: 
#endif
#if GENERIC || #counts(SHIFTED_GAMMA)
				count = ibs.readShiftedGamma() + 1;
#endif
#if GENERIC	
				break;
			case GAMMA: 
#endif
#if GENERIC || #counts(GAMMA)
				count = ibs.readGamma() + 1;
#endif
#if GENERIC	
				break;
			case DELTA: 
#endif
#if GENERIC || #counts(DELTA)
				count = ibs.readDelta() + 1;
#endif
#if GENERIC	
				break;
			default: throw new IllegalStateException( "The required count coding (" + countCoding + ") is not supported." );
			}
#endif
		}
				
			}
				if ( count > positionCache.length ) positionCache = new int[ Math.max( positionCache.length * 2, count ) ];
				final int[] occ = positionCache;
				state = BEFORE_POINTER;

#if GENERIC	
				switch( positionCoding ) {
				case SHIFTED_GAMMA:
#endif
#if GENERIC	|| #positions(SHIFTED_GAMMA)
					ibs.readShiftedGammas( occ, count );
					for( int i = 1; i < count; i++ ) occ[ i ] += occ[ i - 1 ] + 1;
#endif
#if GENERIC	
					return;
				case GAMMA:
#endif
#if GENERIC	|| #positions(GAMMA)
					ibs.readGammas( occ, count );
					for( int i = 1; i < count; i++ ) occ[ i ] += occ[ i - 1 ] + 1;
#endif
#if GENERIC	
					return;
				case DELTA:
#endif
#if GENERIC	|| #positions(DELTA)
					ibs.readDeltas( occ, count );
					for( int i = 1; i < count; i++ ) occ[ i ] += occ[ i - 1 ] + 1;
#endif
#if GENERIC	
					return;
				case GOLOMB:
#endif
#if GENERIC	|| #positions(GOLOMB)
					if ( ASSERTS ) assert index.sizes != null;
					int docSize =  index.sizes.getInt( currentDocument );
					if ( count < 3 ) for( int i = 0; i < count; i++ ) occ[ i ] = ibs.readMinimalBinary( docSize );
					else {
						final int bb = BitStreamIndex.golombModulus( count, docSize );
						int prev = -1;
						if ( bb != 0 ) {
							final int log2bb = Fast.mostSignificantBit( bb );
							for( int i = 0; i < count; i++ ) occ[ i ] = prev = ibs.readGolomb( bb, log2bb ) + prev + 1;
						}
						else for ( int i = 0; i < count; i++ ) occ[ i ] = i;
					}
#endif
#if GENERIC	
					return;
				case SKEWED_GOLOMB:
#endif
#if GENERIC	|| #positions(SKEWED_GOLOMB)
					if ( ASSERTS ) assert index.sizes != null;
					int docSize2 =  index.sizes.getInt( currentDocument );
					if ( count < 3 ) for( int i = 0; i < count; i++ ) occ[ i ] = ibs.readMinimalBinary( docSize2 );
					else {
						final int sb = ibs.readMinimalBinary( docSize2 ) + 1;
	
						int prev2 = -1;
						for( int i = 0; i < count; i++ ) occ[ i ] = prev2 = ibs.readSkewedGolomb( sb ) + prev2 + 1;
					}
#endif
#if GENERIC
					return;
				case INTERPOLATIVE:
#endif
#if GENERIC	|| #positions(INTERPOLATIVE)
					it.unimi.dsi.mg4j.io.InterpolativeCoding.read( ibs, occ, 0, count, 0, index.sizes.getInt( currentDocument ) - 1 );
#endif
#if GENERIC	
					return;
				default:
					throw new IllegalStateException( "The required position coding (" + index.positionCoding + ") is not supported." );
				}
#endif
#endif
		}

		public IntIterator positions() throws IOException {
			if ( ASSERTS ) ensureCurrentDocument();
			if ( state <= BEFORE_POSITIONS ) updatePositionCache();
			return IntIterators.wrap( positionCache, 0, count );
		}

		public int[] positionArray() throws IOException {
			if ( ASSERTS ) ensureCurrentDocument();
			if ( state <= BEFORE_POSITIONS ) updatePositionCache();
			return positionCache;
		}

		// TODO: check who's using this (positionArray() is actually faster now)
		public int positions( final int[] position ) throws IOException {
			if ( ASSERTS ) ensureCurrentDocument();
			if ( state <= BEFORE_POSITIONS ) updatePositionCache(); // And also that positions have been read
			if ( position.length < count ) return -count;
			for( int i = count; i-- != 0; ) position[ i ] = this.positionCache[ i ];
			return count;
		}

		public int nextDocument() throws IOException {
			if ( DEBUG ) System.err.println( "{" + this + "} nextDocument()" );

#if GENERIC || PAYLOADS || ! #counts(NONE)
			if ( state != BEFORE_POINTER ) {
#endif
#if GENERIC || SKIPS
				if ( state == BEFORE_TOWER ) readTower();
#endif
#if GENERIC || PAYLOADS
				if ( state == BEFORE_PAYLOAD ) payload.read( ibs );
#endif
#if GENERIC || ! #counts(NONE)
				if ( state == BEFORE_COUNT )
		{
#if GENERIC
			if ( ASSERTS && state != BEFORE_COUNT ) throw new IllegalStateException();
			state = hasPositions ? BEFORE_POSITIONS : BEFORE_POINTER;
#elif ! #positions(NONE)
			state = BEFORE_POSITIONS;
#else
			state = BEFORE_POINTER;
#endif
			
#if GENERIC	
			switch( countCoding ) {
			case UNARY: 
#endif
#if GENERIC || #counts(UNARY)
				count = ibs.readUnary() + 1;
#endif
#if GENERIC	
				break;
			case SHIFTED_GAMMA: 
#endif
#if GENERIC || #counts(SHIFTED_GAMMA)
				count = ibs.readShiftedGamma() + 1;
#endif
#if GENERIC	
				break;
			case GAMMA: 
#endif
#if GENERIC || #counts(GAMMA)
				count = ibs.readGamma() + 1;
#endif
#if GENERIC	
				break;
			case DELTA: 
#endif
#if GENERIC || #counts(DELTA)
				count = ibs.readDelta() + 1;
#endif
#if GENERIC	
				break;
			default: throw new IllegalStateException( "The required count coding (" + countCoding + ") is not supported." );
			}
#endif
		}

#endif

#if GENERIC || ! #positions(NONE)
				if ( state == BEFORE_POSITIONS ) {
					// Here we just skip; note that the state change is necessary if endOfList() is true
					state = BEFORE_POINTER;

#if GENERIC	
					switch( positionCoding ) {
					case SHIFTED_GAMMA:
#endif
#if GENERIC	|| #positions(SHIFTED_GAMMA)
						ibs.skipShiftedGammas( count );
#endif
#if GENERIC	
						break;
			case GAMMA:
#endif
#if GENERIC	|| #positions(GAMMA)
						ibs.skipGammas( count );
#endif
#if GENERIC	
						break;
					case DELTA:
#endif
#if GENERIC	|| #positions(DELTA)
						ibs.skipDeltas( count );
#endif
#if GENERIC	
				break;
					case GOLOMB:
#endif
#if GENERIC	|| #positions(GOLOMB)
						if ( ASSERTS ) assert index.sizes != null;
						int docSize =  index.sizes.getInt( currentDocument );
						if ( count < 3 ) for( int i = 0; i < count; i++ ) ibs.readMinimalBinary( docSize );
						else {
							final int bb = BitStreamIndex.golombModulus( count, docSize );

							if ( bb != 0 ) {
								final int log2bb = Fast.mostSignificantBit( bb );
								for( int i = 0; i < count; i++ ) ibs.readGolomb( bb, log2bb );
							}
						}
#endif
#if GENERIC	
						break;
					case SKEWED_GOLOMB:
#endif
#if GENERIC	|| #positions(SKEWED_FOLOMB)
						if ( ASSERTS ) assert index.sizes != null;
						docSize =  index.sizes.getInt( currentDocument );
						if ( count < 3 ) for( int i = 0; i < count; i++ ) ibs.readMinimalBinary( docSize );
						else {
							final int sb = ibs.readMinimalBinary( docSize ) + 1;
							for( int i = 0; i < count; i++ ) ibs.readSkewedGolomb( sb );
						}
#endif
#if GENERIC	
						break;
					case INTERPOLATIVE:
#endif
#if GENERIC	|| #positions(INTERPOLATIVE)
						it.unimi.dsi.mg4j.io.InterpolativeCoding.read( ibs, null, 0, count, 0, index.sizes.getInt( currentDocument ) - 1 );
#endif
#if GENERIC	
						break;
					default:
						throw new IllegalStateException( "The required position coding (" + positionCoding + ") is not supported." );
					}
#endif
				}
#endif
#if GENERIC || PAYLOADS || ! #counts(NONE)
			}
#endif
			
			if ( endOfList() ) return -1;

			if ( hasPointers ) {// We do not write pointers for everywhere occurring terms.
#if GENERIC	
				switch( pointerCoding ) {
				case SHIFTED_GAMMA:
#endif
#if GENERIC || #pointers(SHIFTED_GAMMA)
					currentDocument += ibs.readShiftedGamma() + 1;
#endif
#if GENERIC	
					break;
				case GAMMA:
#endif
#if GENERIC || #pointers(GAMMA)
					currentDocument += ibs.readGamma() + 1;
#endif
#if GENERIC	
					break;
				case DELTA:
#endif
#if GENERIC || #pointers(DELTA)
					currentDocument += ibs.readDelta() + 1;
#endif
#if GENERIC	
					break;
				case GOLOMB:
#endif
#if GENERIC || #pointers(GOLOMB)
					currentDocument += ibs.readGolomb( b, log2b ) + 1;
#endif
#if GENERIC