xdelta3-main.h

Linux下一个可以比较二进制文件的工具xdelta3.0u的源码。

/* xdelta 3 - delta compression tools and library
 * Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007,
 * Joshua P. MacDonald
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/* This is all the extra stuff you need for convenience to users in a
 * command line application.  It contains these major components:
 *
 * 1. VCDIFF tools 2. external compression support (this is
 * POSIX-specific).  3. a general read/write loop that handles all of
 * the Xdelta decode/encode/VCDIFF-print functions 4. command-line
 * interpreter 5. an Xdelta application header which stores default
 * filename, external compression settings 6. output/error printing
 * 7. basic file support and OS interface
 */

/* TODO list: 1. do exact gzip-like filename, stdout handling.  make a
 * .vcdiff extension, refuse to encode to stdout without -cf, etc.
 * 2. Allow the user to add a comment string to the app header without
 * disturbing the default behavior.  3. "Source file must be seekable"
 * is not actually true for encoding, given current behavior.  Allow
 * non-seekable sources?  It would in theory let you use a fifo for
 * the source.
 */

/* On error handling and printing:
 *
 * The xdelta library sets stream->msg to indicate what condition
 * caused an internal failure, but many failures originate here and
 * are printed here.  The return convention is 0 for success, as
 * throughout Xdelta code, but special attention is required here for
 * the operating system calls with different error handling.  See the
 * main_file_* routines.  All errors in this file have a message
 * printed at the time of occurance.  Since some of these calls occur
 * within calls to the library, the error may end up being printed
 * again with a more general error message.
 */

/*********************************************************************/

#ifndef XD3_POSIX
#define XD3_POSIX 0
#endif
#ifndef XD3_STDIO
#define XD3_STDIO 0
#endif
#ifndef XD3_WIN32
#define XD3_WIN32 0
#endif
#ifndef NOT_MAIN
#define NOT_MAIN 0
#endif

/* Combines xd3_strerror() and strerror() */
const char* xd3_mainerror(int err_num);

/* XPRINTX (used by main) prefixes an "xdelta3: " to the output. */
#define XPR fprintf
#define NT stderr, "xdelta3: "

/* If none are set, default to posix. */
#if (XD3_POSIX + XD3_STDIO + XD3_WIN32) == 0
#undef XD3_POSIX
#define XD3_POSIX 1
#endif

/* Handle externally-compressed inputs. */
#ifndef EXTERNAL_COMPRESSION
#define EXTERNAL_COMPRESSION 1
#endif

#define PRINTHDR_SPECIAL -4378291

/* The number of soft-config variables.  */
#define XD3_SOFTCFG_VARCNT 7

/* this is used as in XPR(NT XD3_LIB_ERRMSG (stream, ret)) to print an
 * error message from the library. */
#define XD3_LIB_ERRMSG(stream, ret) "%s: %s\n", \
    xd3_errstring (stream), xd3_mainerror (ret)

#include <stdio.h>  /* fprintf */

#if XD3_POSIX
#include <unistd.h> /* close, read, write... */
#include <sys/types.h>
#include <fcntl.h>
#endif

#ifndef _WIN32
#include <unistd.h> /* lots */
#include <sys/time.h> /* gettimeofday() */
#include <sys/stat.h> /* stat() and fstat() */
#else
#if defined(_MSC_VER)
#define strtoll _strtoi64
#endif
#include <sys/types.h>
#include <sys/stat.h>
#ifndef WIFEXITED
#   define WIFEXITED(stat)  (((*((int *) &(stat))) & 0xff) == 0)
#endif
#ifndef WEXITSTATUS
#   define WEXITSTATUS(stat) (((*((int *) &(stat))) >> 8) & 0xff)
#endif
#ifndef S_ISREG
//#   ifdef S_IFREG
//#       define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
//#   else
#       define S_ISREG(m) 1
//#   endif
#endif /* !S_ISREG */

// For standard input/output handles
static STARTUPINFO winStartupInfo;
#endif

/**********************************************************************
 ENUMS and TYPES
 *********************************************************************/

/* These flags (mainly pertaining to main_read() operations) are set
 * in the main_file->flags variable.  All are related to with external
 * decompression support.
 *
 * RD_FIRST causes the external decompression check when the input is
 * first read.
 *
 * RD_NONEXTERNAL disables external decompression for reading a
 * compressed input, in the case of Xdelta inputs.  Note: Xdelta is
 * supported as an external compression type, which makes is the
 * reason for this flag.  An example to justify this is: to create a
 * delta between two files that are VCDIFF-compressed.  Two external
 * Xdelta decoders are run to supply decompressed source and target
 * inputs to the Xdelta encoder. */
typedef enum
{
  RD_FIRST        = (1 << 0),
  RD_NONEXTERNAL  = (1 << 1),
  RD_EXTERNAL_V1  = (1 << 2),
} xd3_read_flags;

/* main_file->mode values */
typedef enum
{
  XO_READ  = 0,
  XO_WRITE = 1,
} main_file_modes;

/* Main commands.  For example, CMD_PRINTHDR is the "xdelta printhdr"
 * command. */
typedef enum
{
  CMD_NONE = 0,
  CMD_PRINTHDR,
  CMD_PRINTHDRS,
  CMD_PRINTDELTA,
  CMD_RECODE,
  CMD_MERGE_ARG,
  CMD_MERGE,
#if XD3_ENCODER
  CMD_ENCODE,
#endif
  CMD_DECODE,
  CMD_TEST,
  CMD_CONFIG,
} xd3_cmd;

#if XD3_ENCODER
#define CMD_DEFAULT CMD_ENCODE
#define IS_ENCODE(cmd) (cmd == CMD_ENCODE)
#else
#define CMD_DEFAULT CMD_DECODE
#define IS_ENCODE(cmd) (0)
#endif

typedef struct _main_file        main_file;
typedef struct _main_extcomp     main_extcomp;
typedef struct _main_blklru      main_blklru;
typedef struct _main_blklru_list main_blklru_list;
typedef struct _main_merge       main_merge;
typedef struct _main_merge_list  main_merge_list;

/* The main_file object supports abstract system calls like open,
 * close, read, write, seek, stat.  The program uses these to
 * represent both seekable files and non-seekable files.  Source files
 * must be seekable, but the target input and any output file do not
 * require seekability.
 */
struct _main_file
{
#if XD3_STDIO
  FILE               *file;
#elif XD3_POSIX
  int                 file;
#elif XD3_WIN32
  HANDLE              file;
#endif

  int                 mode;          /* XO_READ and XO_WRITE */
  const char         *filename;      /* File name or /dev/stdin,
				      * /dev/stdout, /dev/stderr. */
  char               *filename_copy; /* File name or /dev/stdin,
				      * /dev/stdout, /dev/stderr. */
  const char         *realname;      /* File name or /dev/stdin,
				      * /dev/stdout, /dev/stderr. */
  const main_extcomp *compressor;    /* External compression struct. */
  int                 flags;         /* RD_FIRST, RD_NONEXTERNAL, ... */
  xoff_t              nread;         /* for input position */
  xoff_t              nwrite;        /* for output position */
  uint8_t            *snprintf_buf;  /* internal snprintf() use */
};

/* Various strings and magic values used to detect and call external
 * compression.  See below for examples. */
struct _main_extcomp
{
  const char    *recomp_cmdname;
  const char    *recomp_options;

  const char    *decomp_cmdname;
  const char    *decomp_options;

  const char    *ident;
  const char    *magic;
  usize_t        magic_size;
  int            flags;
};

/* This file implements a small LRU of source blocks.  For encoding purposes,
 * we prevent paging in blocks we've already scanned in the source (return
 * XD3_NOTAVAIL). */
struct _main_blklru_list
{
  main_blklru_list  *next;
  main_blklru_list  *prev;
};

struct _main_blklru
{
  uint8_t         *blk;
  xoff_t           blkno;
  main_blklru_list  link;
};

#define LRU_SIZE 32U
#define XD3_MINSRCWINSZ XD3_ALLOCSIZE

/* ... represented as a list (no cache index). */
XD3_MAKELIST(main_blklru_list,main_blklru,link);

/* Merge state: */

struct _main_merge_list
{
  main_merge_list  *next;
  main_merge_list  *prev;
};

struct _main_merge
{
  const char *filename;

  main_merge_list  link;
};

XD3_MAKELIST(main_merge_list,main_merge,link);

// TODO: really need to put options in a struct so that internal
// callers can easily reset state.

/* Program options: various command line flags and options. */
static int         option_stdout             = 0;
static int         option_force              = 0;
static int         option_verbose            = 0;
static int         option_quiet              = 0;
static int         option_use_appheader      = 1;
static uint8_t*    option_appheader          = NULL;
static int         option_use_secondary      = 0;
static char*       option_secondary          = NULL;
static int         option_use_checksum       = 1;
static int         option_use_altcodetable   = 0;
static char*       option_smatch_config      = NULL;
static int         option_no_compress        = 0;
static int         option_no_output          = 0; /* do not write output */
static const char *option_source_filename    = NULL;

static int         option_level              = XD3_DEFAULT_LEVEL;
static usize_t     option_iopt_size          = XD3_DEFAULT_IOPT_SIZE;
static usize_t     option_winsize            = XD3_DEFAULT_WINSIZE;
static usize_t     option_srcwinsz           = XD3_DEFAULT_SRCWINSZ;
static usize_t     option_sprevsz            = XD3_DEFAULT_SPREVSZ;

/* These variables are supressed to avoid their use w/o support.  main() warns
 * appropriately. */
#if EXTERNAL_COMPRESSION
static int         option_decompress_inputs  = 1;
static int         option_recompress_outputs = 1;
#endif

/* This is for comparing "printdelta" output without attention to
 * copy-instruction modes. */
#if VCDIFF_TOOLS
static int option_print_cpymode = 1; /* Note: see reset_defaults(). */ 
#endif

/* Static variables */
IF_DEBUG(static int main_mallocs = 0;)

static char*           program_name = NULL;
static uint8_t*        appheader_used = NULL;
static uint8_t*        main_bdata = NULL;
static usize_t         main_bsize = 0;

/* The LRU: obviously this is shared by all callers. */
static usize_t           lru_size = 0;
static main_blklru      *lru = NULL;  /* array of lru_size elts */
static main_blklru_list  lru_list;
static main_blklru_list  lru_free;
static int               do_not_lru = 0;  /* set to avoid lru */

static int lru_hits   = 0;
static int lru_misses = 0;
static int lru_filled = 0;

/* Hacks for VCDIFF tools */
static int allow_fake_source = 0;

/* recode_stream is used by both recode/merge for reading vcdiff inputs */
static xd3_stream *recode_stream = NULL;

/* merge_stream is used by merge commands for storing the source encoding */
static xd3_stream *merge_stream = NULL;

/* This array of compressor types is compiled even if EXTERNAL_COMPRESSION is
 * false just so the program knows the mapping of IDENT->NAME. */
static main_extcomp extcomp_types[] =
{
  /* The entry for xdelta3 must be 0 because the program_name is set there. */
  { "xdelta3",  "-cfq",  "xdelta3",    "-dcfq",  "X", "\xd6\xc3\xc4", 3,
    RD_NONEXTERNAL },
  { "bzip2",    "-cf",   "bzip2",      "-dcf",   "B", "BZh",          3, 0 },
  { "gzip",     "-cf",   "gzip",       "-dcf",   "G", "\037\213",     2, 0 },
  { "compress", "-cf",   "uncompress", "-cf",    "Z", "\037\235",     2, 0 },

  /* TODO: add commandline support for magic-less formats */
  /*{ "lzma", "-cf",   "lzma", "-dcf",   "M", "]\000", 2, 0 },*/
};

// };

static int main_input (xd3_cmd cmd, main_file *ifile,
                       main_file *ofile, main_file *sfile);
static void main_get_appheader (xd3_stream *stream, main_file *ifile,
				main_file *output, main_file *sfile);

static int main_help (void);

static int
main_version (void)
{
  /* $Format: "  DP(RINT \"Xdelta version $Xdelta3Version$, Copyright (C) 2007, 2008, Joshua MacDonald\n\");" $ */
  DP(RINT "Xdelta version 3.0u, Copyright (C) 2007, 2008, Joshua MacDonald\n");
  DP(RINT "Xdelta comes with ABSOLUTELY NO WARRANTY.\n");
  DP(RINT "This is free software, and you are welcome to redistribute it\n");
  DP(RINT "under certain conditions; see \"COPYING\" for details.\n");
  return EXIT_SUCCESS;
}

static int
main_config (void)
{
  main_version ();

  DP(RINT "EXTERNAL_COMPRESSION=%d\n", EXTERNAL_COMPRESSION);
  DP(RINT "GENERIC_ENCODE_TABLES=%d\n", GENERIC_ENCODE_TABLES);
  DP(RINT "GENERIC_ENCODE_TABLES_COMPUTE=%d\n", GENERIC_ENCODE_TABLES_COMPUTE);
  DP(RINT "REGRESSION_TEST=%d\n", REGRESSION_TEST);
  DP(RINT "SECONDARY_DJW=%d\n", SECONDARY_DJW);
  DP(RINT "SECONDARY_FGK=%d\n", SECONDARY_FGK);
  DP(RINT "UNALIGNED_OK=%d\n", UNALIGNED_OK);
  DP(RINT "VCDIFF_TOOLS=%d\n", VCDIFF_TOOLS);
  DP(RINT "XD3_ALLOCSIZE=%d\n", XD3_ALLOCSIZE);
  DP(RINT "XD3_DEBUG=%d\n", XD3_DEBUG);
  DP(RINT "XD3_ENCODER=%d\n", XD3_ENCODER);
  DP(RINT "XD3_POSIX=%d\n", XD3_POSIX);
  DP(RINT "XD3_STDIO=%d\n", XD3_STDIO);
  DP(RINT "XD3_WIN32=%d\n", XD3_WIN32);
  DP(RINT "XD3_USE_LARGEFILE64=%d\n", XD3_USE_LARGEFILE64);
  DP(RINT "XD3_DEFAULT_LEVEL=%d\n", XD3_DEFAULT_LEVEL);
  DP(RINT "XD3_DEFAULT_IOPT_SIZE=%d\n", XD3_DEFAULT_IOPT_SIZE);
  DP(RINT "XD3_DEFAULT_SPREVSZ=%d\n", XD3_DEFAULT_SPREVSZ);
  DP(RINT "XD3_DEFAULT_SRCWINSZ=%d\n", XD3_DEFAULT_SRCWINSZ);
  DP(RINT "XD3_DEFAULT_WINSIZE=%d\n", XD3_DEFAULT_WINSIZE);
  DP(RINT "XD3_HARDMAXWINSIZE=%d\n", XD3_HARDMAXWINSIZE);
  DP(RINT "sizeof(void*)=%ld\n", sizeof(void*));
  DP(RINT "sizeof(int)=%ld\n", sizeof(int));
  DP(RINT "sizeof(uint32_t)=%ld\n", sizeof(uint32_t));
  DP(RINT "sizeof(uint64_t)=%ld\n", sizeof(uint64_t));
  DP(RINT "sizeof(usize_t)=%ld\n", sizeof(usize_t));
  DP(RINT "sizeof(xoff_t)=%ld\n", sizeof(xoff_t));

  return EXIT_SUCCESS;
}

static void
reset_defaults(void)
{
  option_stdout = 0;
  option_force = 0;
  option_verbose = 0;
  option_quiet = 0;
  option_appheader = NULL;
  option_use_secondary = 0;
  option_secondary = NULL;
  option_use_altcodetable = 0;
  option_smatch_config = NULL;
  option_no_compress = 0;
  option_no_output = 0;
  option_source_filename = NULL;
  program_name = NULL;
  appheader_used = NULL;
  main_bdata = NULL;
  main_bsize = 0;
  lru_size = 0;
  lru = NULL;
  do_not_lru = 0;
  lru_hits   = 0;
  lru_misses = 0;
  lru_filled = 0;
  allow_fake_source = 0;
  option_smatch_config = NULL;

  option_use_appheader = 1;
  option_use_checksum = 1;
#if EXTERNAL_COMPRESSION
  option_decompress_inputs  = 1;
  option_recompress_outputs = 1;
#endif
#if VCDIFF_TOOLS
  option_print_cpymode = 1;
#endif
  option_level = XD3_DEFAULT_LEVEL;
  option_iopt_size = XD3_DEFAULT_IOPT_SIZE;
  option_winsize = XD3_DEFAULT_WINSIZE;
  option_srcwinsz = XD3_DEFAULT_SRCWINSZ;
  option_sprevsz = XD3_DEFAULT_SPREVSZ;
}

static void*
main_malloc1 (usize_t size)
{
  void* r = malloc (size);
  if (r == NULL) { XPR(NT "malloc: %s\n", xd3_mainerror (ENOMEM)); }
  else if (option_verbose > 3) { XPR(NT "malloc: %u: %p\n", size, r); }
  return r;
}

static void*
main_malloc (usize_t size)
{
  void *r = main_malloc1 (size);
  if (r) { IF_DEBUG (main_mallocs += 1); }
  return r;
}

static void*
main_alloc (void   *opaque,
	    usize_t  items,
	    usize_t  size)
{
  return main_malloc1 (items * size);
}

static void
main_free1 (void *opaque, void *ptr)
{
  if (option_verbose > 3) { XPR(NT "free: %p\n", ptr); }
  free (ptr);
}

static void
main_free (void *ptr)
{
  if (ptr)
    {
      IF_DEBUG (main_mallocs -= 1);
      main_free1 (NULL, ptr);
      IF_DEBUG (XD3_ASSERT(main_mallocs >= 0));
    }
}

/* This ensures that (ret = errno) always indicates failure, in case errno was
 * accidentally not set.  If this prints there's a bug somewhere. */
static int