testhadoop.py

HADOOP 0.18.0安装源代码头文件

#Licensed to the Apache Software Foundation (ASF) under one
#or more contributor license agreements.  See the NOTICE file
#distributed with this work for additional information
#regarding copyright ownership.  The ASF licenses this file
#to you under the Apache License, Version 2.0 (the
#"License"); you may not use this file except in compliance
#with the License.  You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

#Unless required by applicable law or agreed to in writing, software
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.
import unittest, os, sys, re, threading, time

myDirectory = os.path.realpath(sys.argv[0])
rootDirectory   = re.sub("/testing/.*", "", myDirectory)

sys.path.append(rootDirectory)

from testing.lib import BaseTestSuite

excludes = []

import tempfile, getpass
from xml.dom import minidom

from hodlib.Hod.hadoop import hadoopConfig

# All test-case classes should have the naming convention test_.*
class test_hadoopConfig(unittest.TestCase):
  def setUp(self):
    self.__hadoopConfig = hadoopConfig()
    self.rootDir = '/tmp/hod-%s' % getpass.getuser()
    if not os.path.exists(self.rootDir):
      os.mkdir(self.rootDir)
    self.testingDir = tempfile.mkdtemp( dir=self.rootDir,
                                  prefix='HadoopTestSuite.test_hadoopConfig')
    self.confDir = tempfile.mkdtemp(dir=self.rootDir,
                                  prefix='HadoopTestSuite.test_hadoopConfig')
    self.tempDir = '/tmp/hod-%s/something' % getpass.getuser()
    self.hadoopSite = os.path.join(self.confDir,'hadoop-site.xml')
    self.numNodes = 4
    self.hdfsAddr = 'nosuchhost1.apache.org:50505'
    self.mapredAddr = 'nosuchhost2.apache.org:50506'
    self.finalServerParams = {
                                'mapred.child.java.opts' : '-Xmx1024m',
                                'mapred.compress.map.output' : 'false',
                             }
    self.serverParams = {
                          'mapred.userlog.limit' : '200',
                          'mapred.userlog.retain.hours' : '10',
                          'mapred.reduce.parallel.copies' : '20',
                        }
    self.clientParams = {
                          'mapred.tasktracker.tasks.maximum' : '2',
                          'io.sort.factor' : '100',
                          'io.sort.mb' : '200',
                          'mapred.userlog.limit.kb' : '1024',
                          'io.file.buffer.size' : '262144',
                        }
    self.clusterFactor = 1.9
    self.mySysDir = '/user/' + getpass.getuser() + '/mapredsystem'
    pass

  def testSuccess(self):
    self.__hadoopConfig.gen_site_conf(
                  confDir = self.confDir,\
                  tempDir = self.tempDir,\
                  numNodes = self.numNodes,\
                  hdfsAddr = self.hdfsAddr,\
                  mrSysDir = self.mySysDir,\
                  mapredAddr = self.mapredAddr,\
                  clientParams = self.clientParams,\
                  serverParams = self.serverParams,\
                  finalServerParams = self.finalServerParams,\
                  clusterFactor = self.clusterFactor

    )
    xmldoc = minidom.parse(self.hadoopSite)
    xmldoc = xmldoc.childNodes[0] # leave out xml spec
    properties = xmldoc.childNodes # children of tag configuration
    keyvals = {}
    for prop in properties:
      if not isinstance(prop,minidom.Comment):
        #      ---------- tag -------------------- -value elem-- data -- 
        name = prop.getElementsByTagName('name')[0].childNodes[0].data
        value = prop.getElementsByTagName('value')[0].childNodes[0].data
        keyvals[name] = value

    # fs.default.name should start with hdfs://
    assert(keyvals['fs.default.name'].startswith('hdfs://'))
    assert(keyvals['hadoop.tmp.dir'] == self.tempDir)
    assert(keyvals['dfs.client.buffer.dir'] == os.path.join(self.tempDir,
                                                            'dfs', 'tmp'))

    # TODO other tests
    pass
    
  def tearDown(self):
    if os.path.exists(self.hadoopSite): os.unlink(self.hadoopSite)
    if os.path.exists(self.confDir) : os.rmdir(self.confDir)
    if os.path.exists(self.testingDir) : os.rmdir(self.testingDir)
    pass

class HadoopTestSuite(BaseTestSuite):
  def __init__(self):
    # suite setup
    BaseTestSuite.__init__(self, __name__, excludes)
    pass
  
  def cleanUp(self):
    # suite tearDown
    pass

def RunHadoopTests():
  suite = HadoopTestSuite()
  testResult = suite.runTests()
  suite.cleanUp()
  return testResult

if __name__ == "__main__":
  RunHadoopTests()