bloomfilter.c

GNUnet是一个安全的点对点网络框架

/*
     This file is part of GNUnet.
     (C) 2001, 2002, 2003, 2004, 2006, 2008 Christian Grothoff (and other contributing authors)

     GNUnet is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published
     by the Free Software Foundation; either version 2, or (at your
     option) any later version.

     GNUnet is distributed in the hope that it will be useful, but
     WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     General Public License for more details.

     You should have received a copy of the GNU General Public License
     along with GNUnet; see the file COPYING.  If not, write to the
     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
     Boston, MA 02111-1307, USA.
*/
/**
 * @file util/containers/bloomfilter.c
 * @brief data structure used to reduce disk accesses.
 *
 * The idea basically: Create a signature for each element in the
 * database. Add those signatures to a bit array. When doing a lookup,
 * check if the bit array matches the signature of the requested
 * element. If yes, address the disk, otherwise return 'not found'.
 *
 * A property of the bloom filter is that sometimes we will have
 * a match even if the element is not on the disk (then we do
 * an unnecessary disk access), but what's most important is that
 * we never get a single "false negative".
 *
 * To be able to delete entries from the bloom filter, we maintain
 * a 4 bit counter in the file on the drive (we still use only one
 * bit in memory).
 *
 * @author Igor Wronsky
 * @author Christian Grothoff
 */

#include "platform.h"
#include "gnunet_util.h"
#include "gnunet_util_containers.h"

typedef struct GNUNET_BloomFilter
{

  /**
   * Concurrency control
   */
  struct GNUNET_Mutex *lock;

  /**
   * The actual bloomfilter bit array
   */
  char *bitArray;

  /**
   * For error handling.
   */
  struct GNUNET_GE_Context *ectx;

  /**
   * Filename of the filter
   */
  char *filename;

  /**
   * The bit counter file on disk
   */
  int fd;

  /**
   * How many bits we set for each stored element
   */
  unsigned int addressesPerElement;

  /**
   * Size of bitArray in bytes
   */
  unsigned int bitArraySize;

} Bloomfilter;


/**
 * Sets a bit active in the bitArray. Increment bit-specific
 * usage counter on disk only if below 4bit max (==15).
 *
 * @param bitArray memory area to set the bit in
 * @param bitIdx which bit to set
 */
static void
setBit (char *bitArray, unsigned int bitIdx)
{
  unsigned int arraySlot;
  unsigned int targetBit;

  arraySlot = bitIdx / 8;
  targetBit = (1L << (bitIdx % 8));
  bitArray[arraySlot] |= targetBit;
}

/**
 * Clears a bit from bitArray. Bit is cleared from the array
 * only if the respective usage counter on the disk hits/is zero.
 *
 * @param bitArray memory area to set the bit in
 * @param bitIdx which bit to unset
 */
static void
clearBit (char *bitArray, unsigned int bitIdx)
{
  unsigned int slot;
  unsigned int targetBit;

  slot = bitIdx / 8;
  targetBit = (1L << (bitIdx % 8));
  bitArray[slot] = bitArray[slot] & (~targetBit);
}

/**
 * Checks if a bit is active in the bitArray
 *
 * @param bitArray memory area to set the bit in
 * @param bitIdx which bit to test
 * @return GNUNET_YES if the bit is set, GNUNET_NO if not.
 */
static int
testBit (char *bitArray, unsigned int bitIdx)
{
  unsigned int slot;
  unsigned int targetBit;

  slot = bitIdx / 8;
  targetBit = (1L << (bitIdx % 8));
  if (bitArray[slot] & targetBit)
    return GNUNET_YES;
  else
    return GNUNET_NO;
}

/**
 * Sets a bit active in the bitArray and increments
 * bit-specific usage counter on disk (but only if
 * the counter was below 4 bit max (==15)).
 *
 * @param bitArray memory area to set the bit in
 * @param bitIdx which bit to test
 * @param fd A file to keep the 4 bit address usage counters in
 */
static void
incrementBit (char *bitArray, unsigned int bitIdx, int fd)
{
  unsigned int fileSlot;
  unsigned char value;
  unsigned int high;
  unsigned int low;
  unsigned int targetLoc;

  setBit (bitArray, bitIdx);
  if (fd == -1)
    return;
  /* Update the counter file on disk */
  fileSlot = bitIdx / 2;
  targetLoc = bitIdx % 2;

  if (fileSlot != (unsigned int) LSEEK (fd, fileSlot, SEEK_SET))
    GNUNET_GE_DIE_STRERROR (NULL,
                            GNUNET_GE_ADMIN | GNUNET_GE_USER | GNUNET_GE_FATAL
                            | GNUNET_GE_IMMEDIATE, "lseek");
  value = 0;
  READ (fd, &value, 1);

  low = value & 0xF;
  high = (value & (~0xF)) >> 4;

  if (targetLoc == 0)
    {
      if (low < 0xF)
        low++;
    }
  else
    {
      if (high < 0xF)
        high++;
    }
  value = ((high << 4) | low);
  if (fileSlot != (unsigned int) LSEEK (fd, fileSlot, SEEK_SET))
    GNUNET_GE_DIE_STRERROR (NULL,
                            GNUNET_GE_ADMIN | GNUNET_GE_USER | GNUNET_GE_FATAL
                            | GNUNET_GE_IMMEDIATE, "lseek");
  if (1 != WRITE (fd, &value, 1))
    GNUNET_GE_DIE_STRERROR (NULL,
                            GNUNET_GE_ADMIN | GNUNET_GE_USER | GNUNET_GE_FATAL
                            | GNUNET_GE_IMMEDIATE, "write");

}

/**
 * Clears a bit from bitArray if the respective usage
 * counter on the disk hits/is zero.
 *
 * @param bitArray memory area to set the bit in
 * @param bitIdx which bit to test
 * @param fd A file to keep the 4bit address usage counters in
 */
static void
decrementBit (char *bitArray, unsigned int bitIdx, int fd)
{
  unsigned int fileSlot;
  unsigned char value;
  unsigned int high;
  unsigned int low;
  unsigned int targetLoc;

  if (fd == -1)
    return;                     /* cannot decrement! */
  GNUNET_GE_ASSERT (NULL, fd != -1);
  /* Each char slot in the counter file holds two 4 bit counters */
  fileSlot = bitIdx / 2;
  targetLoc = bitIdx % 2;
  LSEEK (fd, fileSlot, SEEK_SET);
  value = 0;
  READ (fd, &value, 1);

  low = value & 0xF;
  high = (value & 0xF0) >> 4;

  /* decrement, but once we have reached the max, never go back! */
  if (targetLoc == 0)
    {
      if ((low > 0) && (low < 0xF))
        low--;
      if (low == 0)
        {
          clearBit (bitArray, bitIdx);
        }
    }
  else
    {
      if ((high > 0) && (high < 0xF))
        high--;
      if (high == 0)
        {
          clearBit (bitArray, bitIdx);
        }
    }
  value = ((high << 4) | low);
  LSEEK (fd, fileSlot, SEEK_SET);
  if (1 != WRITE (fd, &value, 1))
    GNUNET_GE_DIE_STRERROR (NULL,
                            GNUNET_GE_ADMIN | GNUNET_GE_USER | GNUNET_GE_FATAL
                            | GNUNET_GE_IMMEDIATE, "write");
}

#define BUFFSIZE 65536

/**
 * Creates a file filled with zeroes
 *
 * @param fd the file handle
 * @param size the size of the file
 * @return GNUNET_OK if created ok, GNUNET_SYSERR otherwise
 */
static int
makeEmptyFile (int fd, unsigned int size)
{
  char *buffer;
  unsigned int bytesleft = size;
  int res = 0;

  if (fd == -1)
    return GNUNET_SYSERR;
  buffer = GNUNET_malloc (BUFFSIZE);
  memset (buffer, 0, BUFFSIZE);
  LSEEK (fd, 0, SEEK_SET);

  while (bytesleft > 0)
    {
      if (bytesleft > BUFFSIZE)
        {
          res = WRITE (fd, buffer, BUFFSIZE);
          bytesleft -= BUFFSIZE;
        }
      else
        {
          res = WRITE (fd, buffer, bytesleft);
          bytesleft = 0;
        }
      if (res == -1)
        {
          GNUNET_GE_DIE_STRERROR (NULL,
                                  GNUNET_GE_ADMIN | GNUNET_GE_USER |
                                  GNUNET_GE_FATAL | GNUNET_GE_IMMEDIATE,
                                  "write");
          GNUNET_free (buffer);
          return GNUNET_SYSERR;
        }
    }
  GNUNET_free (buffer);
  return GNUNET_OK;
}

/* ************** GNUNET_BloomFilter GNUNET_hash iterator ********* */

/**
 * Iterator (callback) method to be called by the
 * bloomfilter iterator on each bit that is to be
 * set or tested for the key.
 *
 * @param bf the filter to manipulate
 * @param bit the current bit
 * @param additional context specific argument
 */
typedef void (*BitIterator) (Bloomfilter * bf, unsigned int bit, void *arg);

/**
 * Call an iterator for each bit that the bloomfilter
 * must test or set for this element.
 *
 * @param bf the filter
 * @param callback the method to call
 * @param arg extra argument to callback
 * @param key the key for which we iterate over the BF bits
 */
static void
iterateBits (Bloomfilter * bf,
             BitIterator callback, void *arg, const GNUNET_HashCode * key)
{
  GNUNET_HashCode tmp[2];
  int bitCount;
  int round;
  unsigned int slot = 0;

  bitCount = bf->addressesPerElement;
  memcpy (&tmp[0], key, sizeof (GNUNET_HashCode));
  round = 0;
  while (bitCount > 0)
    {
      while (slot < (sizeof (GNUNET_HashCode) / sizeof (unsigned int)))
        {
          callback (bf,
                    (((unsigned int *) &tmp[round & 1])[slot]) &
                    ((bf->bitArraySize * 8) - 1), arg);
          slot++;
          bitCount--;
          if (bitCount == 0)
            break;
        }
      if (bitCount > 0)
        {
          GNUNET_hash (&tmp[round & 1], sizeof (GNUNET_HashCode),
                       &tmp[(round + 1) & 1]);