sequencer.cc

来自「Larbin互联网蜘蛛索引系统」· CC 代码 · 共 51 行

// Larbin
// Sebastien Ailleret
// 15-11-99 -> 17-01-00

#include <iostream.h>

#include "global.h"
#include "xutils/url.h"
#include "xutils/Site.h"
#include "xutils/debug.h"

static void getUrl ();

/** start the sequencer
 */
void *startSequencer(void *none) {
  crash("Sequencer on");
  uint space = 0;
  for (;;) {
	if (space == 0) {
	  space += global::inter->putAll();
	}
	space--;
	getUrl();
  }
  return NULL;
}

/* Get the next url
 * here is defined how priorities are handled
 */
static void getUrl () {
  url *u = NULL;
  while (u == NULL) {
	u = global::URLsInput->tryGet();
	if (u != NULL) {
	  // We've got one url (priority)
	  global::siteList[u->hostHashCode() % siteListSize].putPriorityUrl(u);
	  return;
	} else {
	  // Try to get an ordinary url
	  u = global::URLsInternal->tryGet();
	  if (u != NULL) {
		global::siteList[u->hostHashCode() % siteListSize].putUrl(u);
		return;
	  }
	}
	sleep(1);
  }
}