shpackio.c

speech signal process tools

#include <stdio.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <math.h>

/**** Added byt JGF  ****/
#if defined(NARCH_SUN) || defined(NARCH_HP)	/* cth */
#include <sys/types.h>
#include <netinet/in.h>
#endif
#include <sp/sphere.h>
#include <util/min.h>
#include <setjmp.h>

extern jmp_buf	exitenv;

#ifdef fread
#       undef fread
#endif
#define fread(a,b,c,d)          fob_fread((void *)(a),(b),(c),(d))
 
#ifdef fwrite
#       undef fwrite
#endif
#define fwrite(a,b,c,d)         fob_fwrite((void *)(a),(b),(c),(d))
 
#ifdef putc
#       undef putc
#endif
#define putc(a,b)               fob_putc((a),(b))

#ifdef getc
#       undef getc
#endif
#define getc(a)                 fob_getc((a))

#ifdef fgetc
#       undef fgetc
#endif
#define fgetc(a)                fob_getc((a))

#ifdef getw
#       undef getw
#endif
#define getw(a)                 fob_getw((a))
 
#ifdef fputc
#       undef fputc
#endif
#define fputc(a,b)               fob_putc((a),(b))

/**** End of FOB additions ****/

#ifndef TRUE
#define		FALSE		0
#define		TRUE		1
#endif

#define		COMPRESS	1
#define		UNCOMPRESS	2
#define		INFO		3
#define		SWAP_BYTES	1
#define 	ALL_ONES	0xffffffff

static short log2s[] = {1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024,
2048, 4096, 8192, 16384, 32768};

/*keep*/
static char ieee_order = -1; /*-1 means not set yet*/

static char *SccsId = "@(#) %W% %D% MIT/LCS/SLS";

int write_shortpacked_data(short int *, int, FOB *);
int read_shortpacked_data(short int *, int, FOB *);
void write_shortpack_reading_from_file(FOB *, FOB *, int);
void read_shortpack_writing_to_file(FOB *, FOB *, int);
int find_bits_needed(short int );
int find_next_chunk_to_compress(short int *, int, int , int *, int );
int better_to_chop_here_vs_using_more_bits(int, int , int );
int better_to_chop_here_and_use_fewer_bits_for_future(int, int , short int *, short int *, int, int , int , int );
void pack_and_write_chunk_to_file(short int *, int , int, int, FOB *, int );
int read_shortpack_chunk_from_file(FOB *, short int *, int , int, int );
int pack_short_array_into_buffer(short int *, int , int, int , int , short int *);
int unpack_short_array_into_buffer(short int *, int , int , int, int , short int *, int);

/***************************************************************************

	shortpack_io.c

	Peter Daly			[e-mail]
	Spoken Language Systems Group
	MIT Laboratory for Computer Science

	Creation Date: 	12-JUL-1990

	This program compresses utterance files made up of short integers.  
	Refer to the manpage for more information.
	This segment comprises the "guts" of the shortpack algorithim.

	Revision History:

	08-OCT-1990	Modified to not compress files which have an odd 
	                number of bytes, since they are obviously not made
		        up of short integers (pdaly).

	28-AUG-1991	Rewrote many routines to make implementation clearer
	                (mp)

	Compression algorithim by Mike Phillips, MIT/LCS/SLS

	Copyright (C) 1990, 1991	Massachusetts Institute of Technology
					All Rights Reserved

**************************************************************************/


/*  

/*ok, here's how shortpack works:

The idea is to pack short integers into as few bits as required for
sequences of speech samples.  For each sequence, a 16 bit header is
written that says "here comes n words using m bits per word".  The
compression code needs to decide where to break up the sequences to
optimize the tradeoff between transmitting fewer bits for the sequence
vs the cost of transmitting another header for another sequence break.
The present algorithm does an OK job, but if you went to more work in
the compression algorithm, you could probably save a percent or two of
the file size.

If you just want to read and write shortpacked data from your
programs, use the following two functions:

read_shortpacked_data(array, num_elements, fp) - use this for reading
  shortpacked data into an array (works like fread())
write_shortpacked_data(array, num_elements, fp) - use this for writing
  shortpacked data from an array (works like fwrite())

If you want to do file-to-file compression (like the shortpack utility
does), use write_shortpack_reading_from_file() and
read_shortpack_writing_to_file()

These routines use a circular buffer to hold data between input and
output of the shortpacked data.

The file-to-file and file-to-memory routines use the same low level
routines (so the low level routines support the use of circular
buffers (they use a mask before indexing the buffer array - the
memory-to-file routines just use ALL_ONES as a mask))


All of these routines should work with machines with either byte order
(the routines load and dump shortpack file format data to and from the
machines short representation).

If you have any problems, let me know - mike ([e-mail])

*/
#define MAX_HEADER_FIELD 100
#define MAX_SEQUENCE_SIZE 255

/*this goes through the input array, looks for chunks to pack into so
number of bit, compresses those into a temporary array, and writes a
header followed by the chunk of data*/



int write_shortpacked_data(short int *array, int num_elements, FOB *fp)
{ int i,j,k;
  int input_pointer;
  int num_in_chunk;
  int bits;
  input_pointer = 0;

  while(input_pointer < num_elements)
  { num_in_chunk = find_next_chunk_to_compress(array, input_pointer, 
					       num_elements, &bits, ALL_ONES);
    pack_and_write_chunk_to_file(array, input_pointer, num_in_chunk,
				 bits, fp, ALL_ONES);
    input_pointer += num_in_chunk;
  }
  return input_pointer;
}

int read_shortpacked_data(short int *array, int num_elements, FOB *fp)
{ int i,j,k;
  int input_pointer;
  int num_in_chunk;

  input_pointer = 0;

  while(input_pointer < num_elements)
  { num_in_chunk =  read_shortpack_chunk_from_file(fp, array, input_pointer,
						   num_elements, ALL_ONES);
    input_pointer += num_in_chunk;
  }
  return input_pointer;
}

#define CBUF_SIZE 512
#define CBUF_MASK 511

void write_shortpack_reading_from_file(FOB *ifp, FOB *ofp, int swap)
{ int i,j,k;
  short cbuf[CBUF_SIZE];
  int next_write,next_read; /*pointers_to_cbuf*/
  int num_read;
  short test_short = 1;
  int how_much_to_write;
  char *char_ptr;
  char temp_char;
  char temp_char2;
  int num_to_read;
  int num_in_chunk;
  int bits;

  if(ieee_order == -1) /*set this if not set already*/
  { ieee_order = (sp_htons(test_short) == test_short);
  }

  next_read = next_write = 0;
  while(1)
  { /*first read data from file into cbuf, swapping bytes if needed*/
    if(next_read <= next_write)
    { how_much_to_write = CBUF_SIZE - next_write;
      num_read = fread(&(cbuf[next_write]), sizeof(short), how_much_to_write,
		       ifp);
      if((num_read == 0) && (next_read == next_write)) break;

/*      printf("num_read1 %d\n", num_read);*/
      if(!(swap ^ ieee_order))
      { char_ptr = (char*)&(cbuf[next_write]);
	for(i=0;i<num_read;i++)
	{ temp_char = *char_ptr;
	  temp_char2 = *(char_ptr+1);
	  *(char_ptr++) = temp_char2;
	  *(char_ptr++) = temp_char;
	}
      }
      if(num_read < how_much_to_write)
      { next_write += num_read;
	
      }
      else
      { next_write = 0;
      }
    }
    if(next_read > next_write)
    { how_much_to_write = next_read - next_write;
      num_read = fread(&(cbuf[next_write]), sizeof(short), how_much_to_write, 
		       ifp);
/*      printf("num_read2 %d\n", num_read);*/
      if(!(swap ^ ieee_order))
      { char_ptr = (char*)&(cbuf[next_write]);
	for(i=0;i<num_read;i++)
	{ temp_char = *char_ptr;
	  temp_char2 = *(char_ptr+1);
	  *(char_ptr++) = temp_char2;
	  *(char_ptr++) = temp_char;
	}
      }
      next_write += num_read;
    }
    /*now compress and write next chunk*/
    if(next_write == next_read)
    { num_to_read = CBUF_SIZE;
    }
    else
    { if(next_write > next_read)
      { num_to_read = next_write - next_read;
      }
      else
      { num_to_read = CBUF_SIZE - (next_read - next_write);
      }
    }
    num_in_chunk = find_next_chunk_to_compress(cbuf, next_read, 
					       next_read + num_to_read, &bits,
					       CBUF_MASK);
    pack_and_write_chunk_to_file(cbuf, next_read, num_in_chunk, bits, ofp,
				 CBUF_MASK);
    next_read += num_in_chunk;
    next_read &= CBUF_MASK;
  }
}

void read_shortpack_writing_to_file(FOB *ifp, FOB *ofp, int swap)
{ int i,j,k;
  short cbuf[CBUF_SIZE];
  int next_write,next_read; /*pointers_to_cbuf*/
  short test_short = 1;
  int how_much_to_write;
  int num_in_chunk;
  char *char_ptr;
  char temp_char;
  char temp_char2;
  int num_written;
  int room_in_cbuf;
  if(ieee_order == -1) /*set this if not set already*/
  { ieee_order = (sp_htons(test_short) == test_short);
  }

  next_read = next_write = 0;

  while(1)
  { if(next_write == next_read)
    { room_in_cbuf = CBUF_SIZE;
    }
    else
    { if(next_write > next_read)
      { room_in_cbuf = next_write - next_read;
      }
      else
      { room_in_cbuf = CBUF_SIZE - (next_read - next_write);
      }
    }

    num_in_chunk =  read_shortpack_chunk_from_file(ifp, cbuf, next_write, 
						   next_write + room_in_cbuf, 
						   CBUF_MASK);
    if(num_in_chunk < 0) break;

    next_write += num_in_chunk;
    next_write &= CBUF_MASK;

    /*write data from cbuf to file, swapping bytes if needed*/
    if(next_read >= next_write)
    { how_much_to_write = CBUF_SIZE - next_read;
      if(!(swap ^ ieee_order))
      { char_ptr = (char*)&(cbuf[next_read]);
	for(i=0;i<how_much_to_write;i++)
	{ temp_char = *char_ptr;
	  temp_char2 = *(char_ptr+1);
	  *(char_ptr++) = temp_char2;
	  *(char_ptr++) = temp_char;
	}
      }
      num_written = fwrite(&(cbuf[next_read]), sizeof(short), 
			   how_much_to_write, ofp);
      if(num_written < how_much_to_write)
      { fprintf(stderr,"read_shortpack_writing_to_file: HAD_TROUBLE_WRITING!!\n");
	longjmp(exitenv, -1);
/*	exit (-1); */
      }
      else
      { next_read = 0;
      }
    }
    if(next_read < next_write)
    { how_much_to_write = next_write - next_read;
      if(!(swap ^ ieee_order))
      { char_ptr = (char*)&(cbuf[next_read]);
	for(i=0;i<how_much_to_write;i++)
	{ temp_char = *char_ptr;
	  temp_char2 = *(char_ptr+1);
	  *(char_ptr++) = temp_char2;
	  *(char_ptr++) = temp_char;
	}
      }
      num_written = fwrite(&(cbuf[next_read]), sizeof(short), 
			   how_much_to_write, ofp);
      if(num_written < how_much_to_write)
      { fprintf(stderr,"read_shortpack_writing_to_file: HAD_TROUBLE_WRITING!!\n");
	longjmp(exitenv,-1);
/*	exit (-1); */
      }
      next_read += how_much_to_write;
    }
  }
}

/*
 * find_bits_needed(): deterimine the minimum number of bits needed to to 
 represent a number.
 */

int
find_bits_needed(short int num)
{
  int i;

  num = abs(num);
  for (i = 14; i >= 0; i--)
    if ((num & log2s[i]) != 0)
      return (i + 1);

  return (0);
}

/*
 * find_next_chunk_to_compress: (compression) look ahead in the uncompressed 
 data to determine how many bits 
 will be needed to represent the next chunk of data.
 */
int find_next_chunk_to_compress(short int *buffer, int start, int num_elements, int *pbits, int mask)
{ int i,j,k;