xcrtest.cc

harvest是一个下载html网页得机器人

    } else {
      rnum = curia->rnum();
    }
    // roop for retrieving
    for(i = 1; i <= rnum; i++){
      // set a buffer for a key
      len = sprintf(buf, "%08d", i);
      // retrieve a record from the database
      if(lob){
        val = curia->getlob(buf, len);
      } else {
        val = curia->get(buf, len);
      }
      free(val);
      // print progression
      if(rnum > 250 && i % (rnum / 250) == 0){
        cout << '.';
        cout.flush();
        if(i == rnum || i % (rnum / 10) == 0){
          cout << " (" << setw(8) << setfill('0') << i << ")" << endl;
          cout.flush();
        }
      }
    }
    // close the database
    curia->close();
    cout << "ok" << endl << endl;
  } catch(Curia_error& e){
    err = true;
    // report an error
    pxdperror(name, e);
  }
  // destroy instance
  if(curia) delete curia;
  return err ? 1 : 0;
}


/* do multi thread test */
int domulti(const char* name, int rnum, int bnum, int dnum, int tnum, bool lob){
  Curia* curia;
  pthread_t* tids;
  bool* tsts;
  Mission* mss;
  Curia_error* ep;
  int i, len, size;
  bool err;
  char buf[RECBUFSIZ], *val;
  cout << "<Multi-thread Test>" << endl;
  cout << "  name=" << name << "  rnum=" << rnum << "  bnum=" << bnum <<
    "  dnum= " << dnum << "  tnum=" << tnum << "  lob=" << lob << endl;
  cout << endl;
  // open the database
  curia = 0;
  try {
    cout << "Creating a database ... ";
    curia = new Curia(name, Curia::OWRITER | Curia::OCREAT | Curia::OTRUNC, bnum, dnum);
    curia->setalign(ALIGNSIZ);
    cout << "ok" << endl;
  } catch(Curia_error& e){
    if(curia) delete curia;
    pxdperror(name, e);
    return 1;
  }
  // prepare threads
  tids = new pthread_t[tnum];
  tsts = new bool[tnum];
  mss = new Mission[tnum];
  err = false;
  // roop for each threads
  cout << "Writing" << endl;
  for(i = 0; i < tnum; i++){
    mss[i].curia = curia;
    mss[i].lob = lob;
    mss[i].rnum = rnum;
    mss[i].id = i;
    mss[i].printer = (i == tnum / 2);
    if(pthread_create(tids + i, 0, mtwriter, (void*)(mss + i)) == 0){
      tsts[i] = true;
    } else {
      tsts[i] = false;
      err = true;
      cerr << progname << ": " << name << ": " << "pthread error" << endl;
    }
  }
  // join every threads
  for(i = 0; i < tnum; i++){
    if(tsts[i]){
      if(pthread_join(tids[i], (void**)&ep) != 0){
        err = true;
        cerr << progname << ": " << name << ": " << "pthread error" << endl;
      }
    }
    if(ep){
      err = true;
      pxdperror(name, *ep);
      delete ep;
    }
  }
  delete[] mss;
  delete[] tsts;
  delete[] tids;
  if(!err) cout << "ok" << endl;
  cout.flush();
  // check every record
  cout << "Validation checking ... ";
  cout.flush();
  try {
    for(i = 1; i <= rnum; i++){
      len = sprintf(buf, "%08d", i);
      if(lob){
        val = curia->getlob(buf, len, 0, -1, &size);
      } else {
        val = curia->get(buf, len, 0, -1, &size);
      }
      free(val);
      if(size != tnum){
        cerr << progname << ": " << name << ": " << "size error: " << size << endl;
        err = true;
      }
    }
  } catch(Curia_error& e){
    pxdperror(name, e);
    err = true;
  }
  if(!err) cout << "ok" << endl;
  // close the database
  cout << "Closing the database ... ";
  try {
    curia->close();
  } catch(Curia_error& e){
    pxdperror(name, e);
    delete curia;
    return 1;
  }
  cout << "ok" << endl;
  if(!err) cout << "all ok" << endl << endl;
  delete curia;
  return err ? 1 : 0;
}


/* writer with each thread */
void* mtwriter(void* arg){
  Mission* mp = (Mission*)arg;
  Curia* curia = mp->curia;
  bool lob = mp->lob;
  int rnum = mp->rnum;
  int id = mp->id;
  bool printer = mp->printer;
  char buf[RECBUFSIZ];
  void* mc;
  int i, len;
  // roop for storing
  for(i = 1; i <= rnum; i++){
    len = sprintf(buf, "%08d", i);
    // store a record
    try {
      if(lob){
        curia->putlob(buf, len, id % 2 ? "*" : "=", 1, Curia::DCAT);
      } else {
        curia->put(buf, len, id % 2 ? "*" : "=", 1, Curia::DCAT);
      }
    } catch(Curia_error& e){
      return new Curia_error(e);
    }
    // print progression
    if(printer && rnum > 250 && i % (rnum / 250) == 0){
      cout << '.';
      cout.flush();
      if(i == rnum || i % (rnum / 10) == 0){
        cout << " (" << setw(8) << setfill('0') << i << ")" << endl;
        cout.flush();
      }
    }
    // malloc reentrant check
    if((mc = (char*)malloc(1)) != 0){
      free(mc);
    } else {
      return new Curia_error(Curia::EALLOC);
    }
  }
  return 0;
}


/* do combination test */
int domisc(const char* name){
  bool err;
  int i, len;
  char buf[RECBUFSIZ];
  cout << "<Miscellaneous Test>" << endl;
  cout << "  name=" << name << endl;
  cout << endl;
  err = false;
  ADBM* adbm = 0;
  try {
    // open the database
    cout << "Creating a database ... ";
    adbm = new Curia(name, Curia::OWRITER | Curia::OCREAT | Curia::OTRUNC);
    cout << "ok" << endl;
    // write with datadase abstraction
    cout << "Writing with database abstraction ... ";
    for(i = 1; i <= 100; i++){
      len = sprintf(buf, "%08d", i);
      Datum key(buf, len);
      Datum val(buf, len);
      adbm->storerec(key, val, true);
    }
    cout << "ok" << endl;
    // read with datadase abstraction
    cout << "Reading with database abstraction ... ";
    for(i = 1; i <= 100; i++){
      len = sprintf(buf, "%08d", i);
      Datum key(buf, len);
      const Datum& val = adbm->fetchrec(key);
      len = val.size();
      if(len != 8) throw DBM_error("size error");
    }
    cout << "ok" << endl;
    // traversal access with database abstraction
    cout << "Traversal access with database abstraction ... ";
    i = 0;
    try {
      for(adbm->firstkey(); true; adbm->nextkey()){
        i++;
      }
    } catch(DBM_error& e){
      if(adbm->error()) throw;
    }
    if(i != 100) throw DBM_error("iteration error");
    cout << "ok" << endl;
    // close the database
    cout << "Closing the database ... ";
    adbm->close();
    cout << "ok" << endl;
  } catch(DBM_error& e){
    err = true;
    pxdperror(name, e);
  }
  if(adbm) delete adbm;
  if(err) return 1;



  Curia* curia = 0;
  try {
    // open the database
    cout << "Opening the database ... ";
    curia = new Curia(name);
    cout << "ok" << endl;
    // traversal access
    cout << "Traversal access ... ";
    try {
      curia->iterinit();
      for(i = 0; true; i++){
        free(curia->iternext(0));
      }
      if(i != 100) throw Curia_error();
    } catch(Curia_error& e){
      if(e != Curia::ENOITEM) throw;
    }
    cout << "ok" << endl;
    // checking information
    cout << "Checking information" << endl;
    cout << "  - fsiz ... " << curia->fsiz() << endl;
    cout << "  - bnum ... " << curia->bnum() << endl;
    cout << "  - busenum ... " << curia->busenum() << endl;
    cout << "  - rnum ... " << curia->rnum() << endl;
    cout << "  - writable ... " << curia->writable() << endl;
    cout << "  - fatalerror ... " << curia->fatalerror() << endl;
    cout << "  - inode ... " << curia->inode() << endl;
    cout << "ok" << endl;
    // close the database
    cout << "Closing the database ... ";
    curia->close();
    cout << "ok" << endl;
  } catch(Curia_error& e){
    err = true;
    pxdperror(name, e);
  }
  if(curia) delete curia;
  if(err) return 1;
  curia = 0;
  try {
    // open the database
    cout << "Opening the database ... ";
    curia = new Curia(name, Curia::OWRITER);
    cout << "ok" << endl;
    // sync the database
    cout << "Syncing the database ... ";
    curia->sync();
    cout << "ok" << endl;
    // optimize the database
    cout << "Optimizing the database ... ";
    curia->optimize();
    cout << "ok" << endl;
    // close the database
    cout << "Closing the database ... ";
    curia->close();
    cout << "ok" << endl;
  } catch(Curia_error& e){
    err = true;
    pxdperror(name, e);
  }
  if(curia) delete curia;
  if(err) return 1;
  cout << "all ok" << endl << endl;
  return 0;
}



/* END OF FILE */