sql_struct.txt

网页抓取程序

create database if not exists `se_hosts`;

USE `se_hosts`;

/*Table structure for table `hostlist` */

DROP TABLE IF EXISTS `hostlist`;

CREATE TABLE `hostlist` (                                    
	`id` int(10) unsigned NOT NULL AUTO_INCREMENT,             
	`hostname` varchar(100) NOT NULL,                          
	`port` int(6) NOT NULL DEFAULT '80',                       
	`status` int(11) NOT NULL DEFAULT '0',                     
	`lastvisit` varchar(10) DEFAULT NULL,                      
	`indexed_pages` int(11) unsigned NOT NULL DEFAULT '0',     
	`time_sec` int(11) unsigned NOT NULL DEFAULT '0',          
	`bytes_downloaded` int(11) unsigned NOT NULL DEFAULT '0',  
	`error_pages` int(10) unsigned NOT NULL DEFAULT '0',       
	`priority` int(11) unsigned NOT NULL DEFAULT '0',          
	PRIMARY KEY (`hostname`),                                  
	UNIQUE KEY `id` (`id`)                                     
	) ENGINE=MyISAM CHARSET=utf8 ;

/*Table structure for table `hostlist_extras` */

DROP TABLE IF EXISTS `hostlist_extras`;

CREATE TABLE `hostlist_extras` (
  `host_id` int(10) unsigned NOT NULL,
  `max_pages` int(10) unsigned NOT NULL default '0',
  `max_level` int(10) unsigned NOT NULL default '0',
  `max_seconds` int(10) unsigned NOT NULL default '0',
  `max_bytes` int(10) unsigned NOT NULL default '0',
  PRIMARY KEY  (`host_id`)
) ENGINE=MyISAM CHARSET=latin1;

/*Table structure for table `rels` */

DROP TABLE IF EXISTS `rels`;

CREATE TABLE `rels` (                                                        
          `host_id` int(10) unsigned NOT NULL,                                       
          `page` varchar(255) NOT NULL,                                              
          `linkedhost_id` int(10) unsigned NOT NULL,                                 
          `linkedpage` varchar(255) NOT NULL,                                        
          `textlink` varchar(255) CHARACTER SET utf8 DEFAULT NULL,                   
          UNIQUE KEY `unique_index` (`host_id`,`page`,`linkedhost_id`,`linkedpage`)  
        ) ENGINE=MyISAM CHARSET=latin1;

/* *********************************************************************** */

create database if not exists `se_spiderdb`;

USE `se_spiderdb`;

/*Table structure for table `pagelist` */

DROP TABLE IF EXISTS `pagelist`;

CREATE TABLE `pagelist` (                                        
  `id` int(11) NOT NULL auto_increment,                          
  `host_id` int(10) unsigned NOT NULL,                           
  `hostname` varchar(100) NOT NULL,                              
  `page` varchar(255) NOT NULL,                                  
  `title` varchar(255) NOT NULL,                                 
  `text` longtext NOT NULL,                                      
  `cache` longblob,                                              
  `version` int(11) unsigned NOT NULL default '0',               
  `level` int(11) unsigned NOT NULL default '0',                 
  `rank` int(11) unsigned NOT NULL default '0',                  
  `date` varchar(10) NOT NULL,                                   
  `time` varchar(10) NOT NULL,                                   
  PRIMARY KEY  (`id`),                                           
  FULLTEXT KEY `description` (`title`,`text`,`hostname`,`page`)  
) ENGINE=MyISAM CHARSET=utf8;

/*Table structure for table `tablelist` */

DROP TABLE IF EXISTS `tablelist`;

CREATE TABLE `tablelist` (
  `idtable` int(11) NOT NULL auto_increment,
  `name` varchar(20) NOT NULL default '',
  `status` int(2) unsigned NOT NULL default '0',
  `strdate` varchar(10) NOT NULL default '',
  PRIMARY KEY  (`idtable`)
) ENGINE=MyISAM CHARSET=utf8;


/*Table structure for table `wordlist` */

DROP TABLE IF EXISTS `wordlist`;

CREATE TABLE `wordlist` (
  `id` int(11) NOT NULL auto_increment,
  `word` varchar(50) NOT NULL,
  PRIMARY KEY  (`id`),
  UNIQUE KEY `word` (`word`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8;


/*Table structure for table `ii` */

DROP TABLE IF EXISTS `ii`;

CREATE TABLE `ii` (
  `wordid` int(11) NOT NULL,
  `pageid` int(11) NOT NULL,
  `position` int(11) NOT NULL default '0'
) ENGINE=MyISAM DEFAULT CHARSET=latin1;

CREATE VIEW `view_ii` AS (select distinct `ii`.`pageid` AS `pageid` from `ii`);