depot.rb

harvest是一个下载html网页得机器人

#=================================================================================================
# Ruby API of Depot, the basic API of QDBM
#                                                       Copyright (C) 2000-2003 Mikio Hirabayashi
# This file is part of QDBM, Quick Database Manager.
# QDBM is free software; you can redistribute it and/or modify it under the terms of the GNU
# Lesser General Public License as published by the Free Software Foundation; either version
# 2.1 of the License or any later version.  QDBM is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
# details.
# You should have received a copy of the GNU Lesser General Public License along with QDBM; if
# not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
# 02111-1307 USA.
#=================================================================================================


require 'mod_depot'
require 'thread'



##
# require 'depot'
# The library `depot' should be included in application codes.
# An instance of the class `Depot' is used as a database handle.
# `Depot' performs Mix-in of `Enumerable'.
# Each method of `Depot' throws an exception of `Depot::EANY' or its sub classes when an error
# occurs: `Depot::ENOERR', `Depot::EFATAL', `Depot::EMODE', `Depot::EBROKEN', `Depot::EKEEP',
# `Depot::ENOITEM', `Depot::EALLOC', `Depot::EMAP', `Depot::EOPEN', `Depot::ECLOSE',
# `Depot::ETRUNC', `Depot::ESYNC', `Depot::ESTAT', `Depot::ESEEK', `Depot::EREAD',
# `Depot::EWRITE', `Depot::ELOCK', `Depot::EUNLINK', `Depot::EMKDIR', `Depot::ERMDIR' and
# `Depot::EMISC'.
#
class Depot
  include Mod_Depot, Enumerable
  #----------------------------------------------------------------
  # class constants
  #----------------------------------------------------------------
  MyMutex = Mutex::new()
  #----------------------------------------------------------------
  # class methods
  #----------------------------------------------------------------
  public
  ##
  # depot = Depot::new(name, omode, bnum)
  # Constructor: Get a database handle.
  # `name' specifies the name of a database file.
  # `omode' specifies the connection mode: `Depot::OWRITER' as a writer, `Depot::OREADER' as a
  # reader.  If the mode is `Depot::OWRITER', the following may be added by bitwise or:
  # `Depot::OCREAT', which means it creates a new database if not exist, `Depot::OTRUNC', which
  # means it creates a new database regardless if one exists.  Both of `Depot::OREADER' and
  # `Depot::OWRITER' can be added to by bitwise or: `Depot::ONOLCK', which means it opens a
  # database file without file locking.  If it is omitted, `Depot::OREADER' is specified.
  # `bnum' specifies the number of elements of the bucket array.  If it is omitted or not more
  # than 0, the default value is specified.  The size of a bucket array is determined on
  # creating, and can not be changed except for by optimization of the database.  Suggested
  # size of a bucket array is about from 0.5 to 4 times of the number of all records to store.
  # The return value is the database handle.
  # An exception of `Depot::EANY' or its sub classes is thrown if an error occures.
  # If a block parameter is given, this method works as an iterator.  A database handle is
  # opened and passed via the first argument of the block.  The database handle is surely
  # closed when the block is over.
  # While connecting as a writer, an exclusive lock is invoked to the database file.
  # While connecting as a reader, a shared lock is invoked to the database file.  The thread
  # blocks until the lock is achieved.  If `Depot::ONOLCK' is used, the application is
  # responsible for exclusion control.
  #
  #@ DEFINED IMPLICITLY
  ##
  # depot = Depot::open(name, omode, bnum)
  # Constructor: An alias of `new'.
  #
  #@ DEFINED OUTSIDE
  #----------------------------------------------------------------
  # private methods
  #----------------------------------------------------------------
  private
  #=
  # initialize(name, omode, bnum)
  # Method: Called implicitly by the constructor.
  #
  def initialize(name, omode = OREADER, bnum = -1)
    MyMutex.synchronize() do
      @index = mod_open(name, omode, bnum)
      @name = name
    end
    if(iterator?)
      begin
        yield(self)
      ensure
        close()
      end
    end
    self
  end
  #=
  # clone()
  # Method: Forbidden to use.
  #
  def clone
    raise(DepotError)
  end
  #=
  # dup()
  # Method: Forbidden to use.
  #
  alias dup clone
  #----------------------------------------------------------------
  # public methods
  #----------------------------------------------------------------
  public
  ##
  # bool = depot.close()
  # Method: Close the database handle.
  # The return value is always `true'.
  # An exception of `Depot::EANY' or its sub classes is thrown if an error occures.
  # Because the region of a closed handle is released, it becomes impossible to use the handle.
  # Updating a database is assured to be written when the handle is closed.  If a writer opens
  # a database but does not close it appropriately, the database will be broken.
  #
  def close()
    MyMutex.synchronize() do
      begin
        mod_close(@index)
      ensure
        @index = -1
      end
    end
  end
  ##
  # bool = depot.put(key, val, dmode)
  # Method: Store a record.
  # `key' specifies a key.  Although it must be an instance of String, binary data is okey.
  # `val' specifies a value.  Although it must be an instance of String, binary data is okey.
  # `dmode' specifies behavior when the key overlaps, by the following values: `Depot::DOVER',
  # which means the specified value overwrites the existing one, `Depot::DKEEP', which means
  # the existing value is kept, `Depot::DCAT', which means the specified value is concatenated
  # at the end of the existing value.  If it is omitted, `Depot::DOVER' is specified.
  # The return value is always `true'.
  # An exception of `Depot::EANY' or its sub classes is thrown if an error occures or replace
  # is cancelled.
  #
  def put(key, val, dmode = DOVER)
    mod_put(@index, key, val, dmode)
  end
  ##
  # bool = depot.store(key, val)
  # Method: An alias of `put'.
  #
  alias store put
  ##
  # str = (depot[key] = val)
  # Method: An alias of `put'.
  #
  alias []= put
  ##
  # bool = depot.out(key)
  # Method: Delete a record.
  # `key' specifies a key.  Although it must be an instance of String, binary data is okey.
  # The return value is always `true'.
  # An exception of `Depot::EANY' or its sub classes is thrown if an error occures or no record
  # corresponds.
  #
  def out(key)
    mod_out(@index, key)
  end
  ##
  # bool = depot.delete(key)
  # Method: An alias of `out'.
  #
  alias delete out
  ##
  # bool = depot.clear()
  # Method: Delete all records.
  # The return value is always `true'.
  # An exception of `Depot::EANY' or its sub classes is thrown if an error occures.
  #
  def clear
    MyMutex.synchronize() do
      iterinit()
      while(rnum() > 0)
        out(iternext())
      end
    end
    true
  end
  ##
  # str = depot.get(key, start, max)
  # Method: Retrieve a record.
  # `key' specifies a key.  Although it must be an instance of String, binary data is okey.
  # `start' specifies the offset address of the beginning of the region of the value to be read.
  # If it is negative or omitted, the offset is specified as 0.
  # `max' specifies the max size to be read.  If it is negative or omitted, the size to read is
  # unlimited.
  # The return value is an instance of the value of the corresponding record.
  # An exception of `Depot::EANY' or its sub classes is thrown if an error occures, no record
  # corresponds, or the size of the value of the corresponding record is less than `max'.
  #
  def get(key, start = 0, max = -1)
    mod_get(@index, key, start, max)
  end
  ##
  # str = depot.fetch(key)
  # Method: An alias of `get'.
  #
  alias fetch get
  ##
  # str = depot[key]
  # Method: An alias of `get'.
  #
  alias [] get
  ##
  # num = depot.vsiz(key)
  # Method: Get the size of the value of a record.
  # `key' specifies a key.  Although it must be an instance of String, binary data is okey.
  # The return value is the size of the value of the corresponding record.
  # An exception of `Depot::EANY' or its sub classes is thrown if an error occures or no record
  # corresponds.
  # Because this method does not read the entity of a record, it is faster than `get'.
  #
  def vsiz(key)
    mod_vsiz(@index, key)
  end
  ##
  # bool = depot.iterinit()
  # Method: Initialize the iterator of the database handle.
  # The return value is always `true'.
  # An exception of `Depot::EANY' or its sub classes is thrown if an error occures.
  # The iterator is used in order to access the key of every record stored in a database.
  #
  def iterinit()
    mod_iterinit(@index)
  end
  ##
  # str = depot.iternext()
  # Method: Get the next key of the iterator.
  # The return value is the value of the next key.
  # An exception of `Depot::EANY' or its sub classes is thrown if an error occures or no record
  # is to be get out of the iterator.
  # It is possible to access every record by iteration of calling this method.  However, it is
  # not assured if updating the database is occurred while the iteration.  Besides, the order
  # of this traversal access method is arbitrary, so it is not assured that the order of
  # storing matches the one of the traversal access.
  #
  def iternext()
    mod_iternext(@index)
  end
  ##
  # bool = depot.setalign(align)
  # Method: Set alignment of the database handle.
  # `align' specifies the basic size of alignment.  If it is omitted, alignment is cleared.
  # The return value is always `true'.
  # An exception of `Depot::EANY' or its sub classes is thrown if an error occures.
  # If alignment is set to a database, the efficiency of overwriting values are improved.
  # The size of alignment is suggested to be average size of the values of the records to be
  # stored.  If alignment is positive, padding whose size is multiple number of the alignment
  # is placed.  If alignment is negative, as `vsiz' is the size of a value, the size of padding
  # is calculated with `(vsiz / pow(2, abs(align) - 1))'.  Because alignment setting is not
  # saved in a database, you should specify alignment every opening a database.
  #
  def setalign(align = 0)
    mod_setalign(@index, align)
  end
  ##
  # bool = depot.sync()
  # Method: Synchronize updating contents with the file and the device.
  # The return value is always `true'.
  # An exception of `Depot::EANY' or its sub classes is thrown if an error occures.
  # This method is useful when another process uses the connected database file.
  #