sphere.doc

speech signal process tools

The first line specifies the header type and the second line specifies the
header length.  Each of these lines are 8 bytes long (including new-line) and
are structured to identify the header as well as allow those who do not wish
to read the subsequent header information to programmatically skip over it. 

The remaining object-oriented variable portion is composed of
object-type-value "triple" lines which have the following format:


<LINE> --> <TRIPLE><new-line> |
           <COMMENT><new-line> | 
           <TRIPLE><COMMENT><new-line> | 

  <TRIPLE> --> <OBJECT><space><TYPE><space><VALUE><OPT-SPACES>

    <OBJECT> --> <PRIMARY-SUBOBJECT> | 
                 <PRIMARY-SUBOBJECT><SECONDARY-SUBOBJECT>

    <PRIMARY-SUBOBJECT> --> <ALPHA> | <ALPHA><ALPHA-NUM-STRING>
    <SECONDARY-SUBOBJECT> --> _<ALPHA-NUM-STRING> | 
                              _<ALPHA-NUM-STRING><SECONDARY-SUBOBJECT>

    <TYPE> --> -<INTEGER-FLAG> | -<REAL-FLAG> | -<STRING-FLAG>

      <INTEGER-FLAG> --> i
      <REAL-FLAG> --> r
      <STRING-FLAG> --> s<DIGIT-STRING>
      
    <VALUE> --> <INTEGER> | <REAL> | <STRING>  (depending on object type)

      <INTEGER> --> <SIGN><DIGIT-STRING>
      <REAL> --> <SIGN><DIGIT-STRING>.<DIGIT-STRING> 

    <OPT-SPACES> --> <SPACES> | NULL

  <COMMENT> --> ;<STRING>  (excluding embedded new-lines)

<ALPHA-NUM-STRING> --> <ALPHA-NUM> | <ALPHA-NUM><ALPHA-NUM-STRING>
<ALPHA-NUM> --> <DIGIT> | <ALPHA>
<ALPHA> --> a | ... | z | A | ... | Z
<DIGIT-STRING> --> <DIGIT> | <DIGIT><DIGIT-STRING>
<DIGIT> --> 0 | ... | 9
<SIGN> --> + | - | NULL
<SPACES> --> <space> | <SPACES><space>
<STRING> -->  <CHARACTER> | <CHARACTER><STRING>
<CHARACTER> --> char(0) | char(1) | ... | char(255)

The following fields are required for proper SPHERE I/O handling.
These fields completely describe the geometry of the data contained in
the SPHERE file.

        Field_name         Field_type
	----------         ----------
	sample_count           -i      (Defined to be the number of samples
					per channel)
	sample_n_bytes         -i
	channel_count          -i

The following fields are optional:

        Field_name         Field_type
	----------         ----------
	sample_checksum        -i       
        sample_coding          -s       (If missing, defaults to 'pcm')
        sample_byte_format     -s       (If missing, defaults to the Host's
                                         natural byte format.)

The following fields are conditionally required:

        Field_name         Field_type   Required if:
	----------         ----------   ------------
	sample_rate            -i       (the 'sample_coding' field is
					 missing, or contains 'pcm' or 'ulaw')

Possible values for these fields are:

	sample_count -> 1 .. MAXINT
	sample_n_bytes -> 1 | 2
	channel_count -> 1 .. 32
	sample_byte_format -> 01 | 10 | 1 (* See below for exception)
	sample_checksum -> 0 .. 32767
	sample_rate -> 1 .. MAXINT
	sample_coding -> pcm | ulaw | pcm,embedded-shorten-vX.X | 
			 ulaw,embedded-shorten-vX.X |
			 pcm,embedded-wavpack-X.X | 
			 ulaw,embedded-wavpack-X.X |
			 pcm,embedded-shortpack-X.X (Read Only)

	* Note: The original recordable CD-ROM release of WSJ0 incorporated
	        "shortpack" waveform compression.  The compression was 
		indicated by the now outmoded presence of the string,
		'shortpack-v0',	in the 'sample_byte_format' field.
		Since this software attempts to support all previous
		NIST releases of speech corpora, this "non-standard"
		field value is permitted.  Please note that this
		software will not generate files with this field value
		and that the LDC is producing a CD-ROM release of WSJ0
		which conforms to the specifications defined in SPHERE 2.0.

The single object "end_head" marks the end of the active header and the
remaining unused header space is undefined. A sample header is included
below.

Example SPHERE header from the TIMIT corpus (NIST Speech Disc 1-1.1):

NIST_1A
   1024
database_id -s5 TIMIT
database_version -s3 1.0
utterance_id -s8 aks0_sa1
channel_count -i 1
sample_count -i 63488
sample_rate -i 16000
sample_min -i -6967
sample_max -i 7710
sample_n_bytes -i 2
sample_byte_format -s2 01
sample_sig_bits -i 16
end_head


Example SPHERE header from the CSR Phase II corpus:

NIST_1A
   1024
microphone -s21     Sennheiser HMD414
recording_site -s4  SRI
database_id -s8     wsj1
database_version -s3 1.0
recording_environment -s38 quiet office, door closed (room EJ186)
speaker_session_number -s2 01
session_utterance_number -s2 01
prompt_id -s8 adapt.01
utterance_id -s8 460a0101
speaking_mode -s15 read-adaptation
speaker_id -s3 460
sample_count -i 82561
sample_min -i -560
sample_max -i 655
sample_checksum -i 27320
recording_date -s11 11-Nov-1992
recording_time -s11 12:14:16.00
channel_count -i 1
sample_rate -i 16000
sample_n_bytes -i 2
sample_byte_format -s2 01
sample_sig_bits -i 16
sample_coding -s25 pcm,embedded-shorten-v1.1
end_head



IV.  File Pointer Structure

The following is a description of the file pointer structure used by
SPHERE to address opened files.

The structure includes pointers to the parsed header, the waveform,
and a structure used by SPHERE to track the status of various
operations on the opened file.

NOTE:   The SPHERE file pointer structure has been designed to be used
	exclusively by the SPHERE library functions.  The programmer
	should only include the structure in argument lists passed to
	SPHERE function calls.  The programmer should NOT attempt to
	manipulate this structure directly.


    SPHERE Structure 'SP_FILE' 

	typedef struct sphere_t SP_FILE;

	struct sphere_t {
       		struct header_t *header;
                struct waveform_t *waveform;
		struct spfile_status_t *status;
	};

        struct waveform_t {
	        /* This structure is intended for use only by */
		/* the SPHERE-library functions               */
        };

        struct spfile_status_t {
	        /* This structure is intended for use only by */
		/* the SPHERE-library functions               */
	};

	struct header_t {
	        /* This structure is intended for use only by */
		/* the SPHERE-library functions               */
	};



V.  Installation

To install SPHERE on a Unix system, cd into the 'nist' directory of
the SPHERE distribution hierarchy and type the command:

	sh src/scripts/install

The SPHERE libraries will be created and placed in the 'lib'
directory, and the sample programs, described in the 'System-Level
Utilities' section, will be compiled and placed in the 'bin'
directory.  Installation on non-Unix systems without "make" and "sh"
will probably require manual compilation.  The next release of SPHERE
will support compilation of SPHERE on PC's.

Included with this release is a functional interface test program
which exercises various header manipulation and waveform
compression/decompression functions.  To run the test, execute the
command from the 'nist' directory in the distribution hierarchy:

	tsphere -d lib/data

The program requires the files in the 'lib/data' directory of the
SPHERE distribution hierarchy.  If all is well, the program will run
without errors.  If an error occurs, send a bug report as specified in
the 'Bug Reports' section.



VI.  Linking to the SPHERE library

User programs are linked with two object code libraries:

	libsp.a       - user level SPHERE functions
			include compression code
	libutil.a     - SPHERE-internal functions

Both libraries must be 'linked' for successful use of SPHERE.
During the installation process, the source code for these
libraries is placed in the 'lib' directory of the distribution
hierarchy.   

User programs must include the following line in order to link
with the SPHERE libraries:

	#include <sp/sphere.h>

This include file references all of the required library files.

If the user's programs were written using a SPHERE version 2.0 beta 1
to SPHERE version 2.1, the new syntax for sp_read_data and
sp_write_data needs to be taken into consideration, (see Section II
for function descriptions).  There are two ways to use the old code,
either change the code to conform to the new syntax, or do not
re-write the code, but set a pre-processor variable, "SPHERE_PRE_2_2".
The pre-processor variable can be set in two locations, the choice
of location depends on the choice of the programmer. 

	1. The first location is in each source file which includes
	   the "sp/sphere.h" file.  Before the include statement for
	   "sp/sphere.h", put in the define statement
	   "#define SPHERE_PRE_2_2".  This allows flexibility for each
	   source code file.

	2. The second location is in the "sp/sphere.h" file itself.
	   Simply uncomment the define for "SPHERE_PRE_2_2" on line 36.
	   Now every program which includes "sp/sphere.h" must use
	   the old syntax of sp_read_data and sp_write_data.

In either case, the include file will then properly handle all
references to sp_read_data and sp_write_data.

To compile a program, such as the command-line SPHERE utility,
'w_decode', which is found in the 'src/prog' directory, execute
the following command:

	cc -L<INSTALL_DIR>/nist/lib -I<INSTALL_DIR>/nist/include \
		-o w_decode w_decode.c -lsp -lutil -lm

        where "INSTALL_DIR" is the directory where the SPHERE library
        source code is located.

If an unresolvable error occurs in compiling a program which uses the
SPHERE library, you may send a bug report as specified in the 'Bug
Reports' section and someone will assist you.



VII.  Example Interface Library Usage

Example 1: To load the embedded shorten-compressed 2-byte-per-sample
PCM file, "file.wav", into memory in its uncompressed form.  This version
uses the SPHERE 2.2 and later function call to sp_read_data.

    SP_FILE *sp;
    short *waveform;
    SP_INTEGER sample_count;
    int wave_byte_size, total_samples;

    if ((sp = sp_open("file.wav","r")) == (SP_FILE *)0) {
        fprintf(stderr,"Error: Unable to open SPHERE file %s\n","file.wav");
        return(-1);
    }

    if (sp_h_get_field(sp,"sample_count",T_INTEGER,&sample_count) > 0)
	return(-1);

    if ((waveform=(short *)sp_data_alloc(-1)) == (short *)0){
        fprintf(stderr,"Error: Unable to allocate waveform memory\n");
        exit(-1);
    }
    
    if (sp_read_data(waveform,total_samples,sp) != samples_count){
        fprintf(stderr,"Error: reading speech waveform\n");
	sp_return_status(stderr);
        exit(-1);
    }
                     .
                     .
                     .

    sp_data_free(sp,(void *)waveform);
    sp_close(sp);  /* deallocates the header & buffers */


Example 2: To load a single-channel ulaw file, "ulaw.wav", and convert
it to a 2-byte-per-sample PCM format.  This 

    #define SPHERE_PRE_2_2
    #include <sp/sphere.h>
    SP_FILE *sp=SPNULL;
    short *waveform;
    SP_INTEGER channel_count, sample_n_bytes, sample_count;
    int wave_byte_size, total_samples, samp_read;

    printf("-- Documentation Example 2\n");
    if ((sp = sp_open(EX4_ULAW,"r")) == SPNULL) {
        fprintf(stderr,"Error: Unable to open SPHERE file %s\n",EXAMPLE4_ULAW);
	sp_print_return_status(stdout);
	exit(-1);
    }

    if (sp_set_data_mode(sp, "SE-PCM-2") > 0){
	sp_print_return_status(stdout);
	sp_close(sp);
	exit(-1);
    }
    if (sp_h_get_field(sp,"channel_count",T_INTEGER,&channel_count) > 0){
	fprintf(stderr,"Error: Unable to get the '%s' field\n",
		"channel_count");
	sp_close(sp);
	exit(-1);
    }
    /*  When the sd_set_data_mode() function is called to convert the file */
    /*  from ulaw to pcm, the sample_n_bytes in the header is automatically*/
    /*  changed to 2                                                       */
    if (sp_h_get_field(sp,"sample_n_bytes",T_INTEGER,&sample_n_bytes) > 0){
	fprintf(stderr,"Error: Unable to get the '%s' field\n",
		"sample_n_bytes");
	sp_close(sp);
	exit(-1);
    }
    if (sp_h_get_field(sp,"sample_count",T_INTEGER,&sample_count) > 0){
	fprintf(stderr,"Error: Unable to get the '%s' field\n","sample_count");
	sp_close(sp);
	exit(-1);
    }

    total_samples=sample_count * channel_count;
    wave_byte_size=sample_n_bytes * total_samples;

    printf("---- Example 2: Expected channel_count=1,      Actually=%d\n",
		channel_count);
    printf("---- Example 2: Expected sample_n_bytes=2,     Actually=%d\n",
		sample_n_bytes);
    printf("---- Example 2: Expected sample_count=16000,   Actually=%d\n",
		sample_count);
    printf("---- Example 2: Expected total_samples=16000,  Actually=%d\n",
		total_samples);
    printf("---- Example 2: Expected wave_byte_size=32000, Actually=%d\n",
	wave_byte_size);

    if ((waveform=(short *)malloc(wave_byte_size)) == (short *)0){
        fprintf(stderr,"Error: Unable to allocate %d bytes for the waveform\n",
                       wave_byte_size);
	sp_close(sp);
	exit(-1);
    }
    
    if ((samp_read=sp_read_data(waveform,sample_n_bytes,total_samples,sp)) !=
                     total_samples){
        fprintf(stderr,"Error: Unable to read speech waveform, ");
	fprintf(stderr,"%d samples read\n",samp_read);
	sp_print_return_status(stderr);
	sp_close(sp);
	exit(-1);
    }
    sp_close(sp); /* deallocates the header & buffers */