testdistcc.py

distcc编译器的源代码.好像是readhat公司开发的.

#! /usr/bin/env python2.2

# Copyright (C) 2002, 2003, 2004 by Martin Pool <[e-mail]>
# 
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA

"""distcc test suite, using comfychair

This script is called with $PATH pointing to the appropriate location
for the built (or installed) programs to be tested.
"""


# There are pretty strong hierarchies of test cases: ones to do with
# running a daemon, compiling a file and so on.  This nicely maps onto
# a hierarchy of object classes.

# It seems to work best if an instance of the class corresponds to an
# invocation of a test: this means each method runs just once and so
# object state is not very useful, but nevermind.

# Having a complicated pattens of up and down-calls within the class
# methods seems to make things more complicated.  It may be better if
# abstract superclasses just provide methods that can be called,
# rather than establishing default behaviour.

# TODO: Run the server in a different directory from the clients

# TODO: Some kind of direct test of the host selection algorithm.

# TODO: Test host files containing \r.

# TODO: Optionally run all discc tests under Valgrind, ElectricFence
# or something similar.

# TODO: Test that ccache correctly caches compilations through distcc:
# make up a random file so it won't hit, then compile once and compile
# twice and examine the log file to make sure we got a hit.  Also
# check that the binary works properly.

# TODO: Test cpp from stdin

# TODO: Do all this with malloc debugging on.

# TODO: Redirect daemon output to a file so that we can more easily
# check it.  Is there a straightforward way to test that it's also OK
# when send through syslogd?

# TODO: Check behaviour when children are killed off.

# TODO: Test compiling over IPv6

# TODO: Argument scanning tests should be run with various hostspecs,
# because that makes a big difference to how the client handles them.

# TODO: Test that ccache gets hits when calling distcc.  Presumably
# this is skipped if we can't find ccache.  Need to parse `ccache -s`.

# TODO: Set TMPDIR to be inside the working directory, and perhaps
# also set DISTCC_SAVE_TEMPS.  Might help for debugging.

# Check that without DISTCC_SAVE_TEMPS temporary files are cleaned up.

# TODO: Test compiling a really large source file that produces a
# large object file.  Perhaps need to generate it at run time -- just
# one big array?

# TODO: Perhaps redirect stdout, stderr to a temporary file while
# running?  Use os.open(), os.dup2().

# TODO: Test "distcc gcc -c foo.c bar.c".  gcc would actually compile
# both of them.  We could split it into multiple compiler invocations,
# but this is so rare that it's probably not worth the complexity.  So
# at the moment is just handled locally.

# TODO: Test crazy option arguments like "distcc -o -output -c foo.c"

# TODO: Test attempt to compile a nonexistent file.

# TODO: Add test harnesses that just exercise the bulk file transfer
# routines.

# TODO: Test -MD, -MMD, -M, etc.

# TODO: Test using '.include' in an assembly file, and make sure that
# it is resolved on the client, not on the server.

# TODO: Run "sleep" as a compiler, then kill the client and make sure
# that the server and "sleep" promptly terminate.

# TODO: Set umask 0, then check that the files are created with mode
# 0644.

# TODO: Perhaps have a little compiler that crashes.  Check that the
# signal gets properly reported back.

# TODO: Have a little compiler that takes a very long time to run.
# Try interrupting the connection and see if the compiler is cleaned
# up in a reasonable time.

# TODO: Try to build a nonexistent source file.  Check that we only
# get one error message -- if there were two, we would incorrectly
# have tried to build the program both remotely and locally.

# TODO: Test compiling a 0-byte source file.  This should be allowed.

# TODO: Test a compiler that produces 0 byte output.  I don't know an
# easy way to get that out of gcc aside from the Apple port though.

# TODO: Test a compiler that sleeps for a long time; try killing the
# server and make sure it goes away.

# TODO: Set LANG=C before running all tests, to try to make sure that
# localizations don't break attempts to parse error messages.  Is
# setting LANG enough, or do we also need LC_*?  (Thanks to Oscar
# Esteban.)

# TODO: Test scheduler.  Perhaps run really slow jobs to make things
# deterministic, and test that they're dispatched in a reasonable way.

# TODO: Test generating dependencies with -MD.  Possibly can't be
# done.

# TODO: Test a nasty cpp that always writes to stdout regardless of
# -o.

# TODO: Test giving up privilege using --user.  Difficult -- we may
# need root privileges to run meaningful tests.

# TODO: Test that recursion safeguard works.

# TODO: Test masquerade mode.  Requires us to create symlinks in a
# special directory on the path.

# TODO: Test SSH mode.  May need to skip if we can't ssh to this
# machine.  Perhaps provide a little null-ssh.

# TODO: Test path stripping.

# TODO: Test backoff from downed hosts.

# TODO: Check again in --no-prefork mode.

# TODO: Test lzo is parsed properly

# TODO: Test with DISTCC_DIR set, and not set.


import time, sys, string, os, types, re, popen2, pprint
import signal, os.path, string
import comfychair

from stat import *                      # this is safe

EXIT_DISTCC_FAILED           = 100
EXIT_BAD_ARGUMENTS           = 101
EXIT_BIND_FAILED             = 102
EXIT_CONNECT_FAILED          = 103
EXIT_COMPILER_CRASHED        = 104
EXIT_OUT_OF_MEMORY           = 105
EXIT_BAD_HOSTSPEC            = 106
EXIT_COMPILER_MISSING        = 110
EXIT_ACCESS_DENIED           = 113

_gcc                         = None     # full path to gcc

class SimpleDistCC_Case(comfychair.TestCase):
    '''Abstract base class for distcc tests'''
    def setup(self):
        self.stripEnvironment()

    def stripEnvironment(self):
        """Remove all DISTCC variables from the environment, so that
        the test is not affected by the development environment."""
        for key in os.environ.keys():
            if key[:7] == 'DISTCC_':
                # NOTE: This only works properly on Python 2.2: on
                # earlier versions, it does not call unsetenv() and so
                # subprocesses may get confused.
                del os.environ[key]
        os.environ['TMPDIR'] = self.tmpdir
        ddir = os.path.join(self.tmpdir, 'distccdir')
        os.mkdir(ddir)
        os.environ['DISTCC_DIR'] = ddir


class WithDaemon_Case(SimpleDistCC_Case):
    """Start the daemon, and then run a command locally against it.

The daemon doesn't detach until it has bound the network interface, so
as soon as that happens we can go ahead and start the client."""

    def setup(self):
        import random
        SimpleDistCC_Case.setup(self)
        self.daemon_pidfile = os.path.join(os.getcwd(), "daemonpid.tmp")
        self.daemon_logfile = os.path.join(os.getcwd(), "distccd.log")
        self.server_port = 42000 # random.randint(42000, 43000)
        self.startDaemon()
        self.setupEnv()

    def setupEnv(self):
        os.environ['DISTCC_HOSTS'] = '127.0.0.1:%d' % self.server_port
        os.environ['DISTCC_LOG'] = os.path.join(os.getcwd(), 'distcc.log')
        os.environ['DISTCC_VERBOSE'] = '1'


    def teardown(self):
        SimpleDistCC_Case.teardown(self)


    def killDaemon(self):
        import signal, time

        try:
            pid = int(open(self.daemon_pidfile, 'rt').read())
        except IOError:
            # the daemon probably already exited, perhaps because of a timeout
            return
        os.kill(pid, signal.SIGTERM)

        # We can't wait on it, because it detached.  So just keep
        # pinging until it goes away.
        while 1:
            try:
                os.kill(pid, 0)
            except OSError:
                break
            time.sleep(0.2)


    def daemon_command(self):
        """Return command to start the daemon"""
        return ("distccd  --verbose --lifetime=%d --daemon --log-file %s "
                "--pid-file %s --port %d --allow 127.0.0.1"
                % (self.daemon_lifetime(),
                   self.daemon_logfile, self.daemon_pidfile, self.server_port))

    def daemon_lifetime(self):
        # Enough for most tests, even on a fairly loaded machine.
        # Might need more for long-running tests.
        return 60

    def startDaemon(self):
        """Start a daemon in the background, return its pid"""
        # The daemon detaches once it has successfully bound the
        # socket, so if something goes wrong at startup we ought to
        # find out straight away.  If it starts successfully, then we
        # can go ahead and try to connect.
        
        while 1:
            cmd = self.daemon_command()
            result, out, err = self.runcmd_unchecked(cmd)
            if result == 0:
                break
            elif result == EXIT_BIND_FAILED:
                self.server_port += 1
                continue
            else:
                self.fail("failed to start daemon: %d" % result)
        self.add_cleanup(self.killDaemon)




class StartStopDaemon_Case(WithDaemon_Case):
    def runtest(self):
        pass


class VersionOption_Case(SimpleDistCC_Case):
    """Test that --version returns some kind of version string.

    This is also a good test that the programs were built properly and are
    executable."""
    def runtest(self):
        import string
        for prog in 'distcc', 'distccd':
            out, err = self.runcmd("%s --version" % prog)
            assert out[-1] == '\n'
            out = out[:-1]
            line1,trash = string.split(out, '\n', 1)
            self.assert_re_match(r'^%s [\w.-]+ [.\w-]+ \(protocol.*\) \(default port 3632\)$'
                                 % prog, line1)


class HelpOption_Case(SimpleDistCC_Case):
    """Test --help is reasonable."""
    def runtest(self):
        for prog in 'distcc', 'distccd':
            out, err = self.runcmd(prog + " --help")
            self.assert_re_search("Usage:", out)


class BogusOption_Case(SimpleDistCC_Case):
    """Test handling of --bogus-option.

    Now that we support implicit compilers, this is passed to gcc, which returns 1."""
    def runtest(self):
        self.runcmd("distcc " + _gcc + " --bogus-option", 1)
        self.runcmd("distccd " + _gcc + " --bogus-option", EXIT_BAD_ARGUMENTS)


class GccOptionsPassed_Case(SimpleDistCC_Case):
    """Test that options following the compiler name are passed to the compiler."""
    def runtest(self):
        out, err = self.runcmd("DISTCC_HOSTS=localhost distcc " + \
                               _gcc + " --help")
        if re.search('distcc', out):
            raise ("gcc help contains \"distcc\": \"%s\"" % out)
        self.assert_re_match(r"^Usage: gcc", out)


class StripArgs_Case(SimpleDistCC_Case):
    """Test -D and -I arguments are removed"""
    def runtest(self):
        cases = (("gcc -c hello.c", "gcc -c hello.c"),
                 ("cc -Dhello hello.c -c", "cc hello.c -c"),
                 ("gcc -g -O2 -W -Wall -Wshadow -Wpointer-arith -Wcast-align -c -o h_strip.o h_strip.c",
                  "gcc -g -O2 -W -Wall -Wshadow -Wpointer-arith -Wcast-align -c -o h_strip.o h_strip.c"),
                 # invalid but should work
                 ("cc -c hello.c -D", "cc -c hello.c"),
                 ("cc -c hello.c -D -D", "cc -c hello.c"),
                 ("cc -c hello.c -I ../include", "cc -c hello.c"),
                 ("cc -c -I ../include  hello.c", "cc -c hello.c"),
                 ("cc -c -I. -I.. -I../include -I/home/mbp/garnome/include -c -o foo.o foo.c",
                  "cc -c -c -o foo.o foo.c"),
                 ("cc -c -DDEBUG -DFOO=23 -D BAR -c -o foo.o foo.c",
                  "cc -c -c -o foo.o foo.c"),

                 # New options stripped in 0.11
                 ("cc -o nsinstall.o -c -DOSTYPE=\"Linux2.4\" -DOSARCH=\"Linux\" -DOJI -D_BSD_SOURCE -I../dist/include -I../dist/include -I/home/mbp/work/mozilla/mozilla-1.1/dist/include/nspr -I/usr/X11R6/include -fPIC -I/usr/X11R6/include -Wall -W -Wno-unused -Wpointer-arith -Wcast-align -pedantic -Wno-long-long -pthread -pipe -DDEBUG -D_DEBUG -DDEBUG_mbp -DTRACING -g -I/usr/X11R6/include -include ../config-defs.h -DMOZILLA_CLIENT -Wp,-MD,.deps/nsinstall.pp nsinstall.c",
                  "cc -o nsinstall.o -c -fPIC -Wall -W -Wno-unused -Wpointer-arith -Wcast-align -pedantic -Wno-long-long -pthread -pipe -g nsinstall.c"),
                 )
        for cmd, expect in cases:
            o, err = self.runcmd("h_strip %s" % cmd)
            if o[-1] == '\n': o = o[:-1]
            self.assert_equal(o, expect)


class IsSource_Case(SimpleDistCC_Case):
    def runtest(self):
        """Test distcc's method for working out whether a file is source"""
        cases = (( "hello.c",          "source",       "not-preprocessed" ),
                 ( "hello.cpp",        "source",       "not-preprocessed" ),
                 ( "hello.2.4.4.i",    "source",       "preprocessed" ),
                 ( ".foo",             "not-source",   "not-preprocessed" ),
                 ( "gcc",              "not-source",   "not-preprocessed" ),
                 ( "hello.ii",         "source",       "preprocessed" ),
                 ( "hello.c++",        "source",       "not-preprocessed" ),
                 ( "boot.s",           "not-source",   "not-preprocessed" ),
                 ( "boot.S",           "not-source",   "not-preprocessed" ))
        for f, issrc, iscpp in cases:
            o, err = self.runcmd("h_issource '%s'" % f)
            expected = ("%s %s\n" % (issrc, iscpp))
            if o != expected:
                raise AssertionError("issource %s gave %s, expected %s" %
                                     (f, `o`, `expected`))



class ScanArgs_Case(SimpleDistCC_Case):
    '''Test understanding of gcc command lines.'''
    def runtest(self):
        cases = [("gcc -c hello.c", "distribute", "hello.c", "hello.o"),
                 ("gcc hello.c", "local"),
                 ("gcc -o /tmp/hello.o -c ../src/hello.c", "distribute", "../src/hello.c", "/tmp/hello.o"),
                 ("gcc -DMYNAME=quasibar.c bar.c -c -o bar.o", "distribute", "bar.c", "bar.o"),
                 ("gcc -ohello.o -c hello.c", "distribute", "hello.c", "hello.o"),
                 ("ccache gcc -c hello.c", "distribute", "hello.c", "hello.o"),
                 ("gcc hello.o", "local"),
                 ("gcc -o hello.o hello.c", "local"),
                 ("gcc -o hello.o -c hello.s", "local"),
                 ("gcc -o hello.o -c hello.S", "local"),
                 ("gcc -fprofile-arcs -ftest-coverage -c hello.c", "local", "hello.c", "hello.o"),
                 ("gcc -S hello.c", "distribute", "hello.c", "hello.s"),
                 ("gcc -c -S hello.c", "distribute", "hello.c", "hello.s"),
                 ("gcc -S -c hello.c", "distribute", "hello.c", "hello.s"),
                 ("gcc -M hello.c", "local"),
                 ("gcc -ME hello.c", "local"),
                 
                 ("gcc -MD -c hello.c", "distribute", "hello.c", "hello.o"),
                 ("gcc -MMD -c hello.c", "distribute", "hello.c", "hello.o"),

                 # Assemble to stdout (thanks Alexandre).  
                 ("gcc -S foo.c -o -", "local"),
                 ("-S -o - foo.c", "local"),
                 ("-c -S -o - foo.c", "local"),
                 ("-S -c -o - foo.c", "local"),