run-webkit-tests

linux下开源浏览器WebKit的源码,市面上的很多商用浏览器都是移植自WebKit

#!/usr/bin/perl

# Copyright (C) 2005, 2006, 2007, 2008, 2009 Apple Inc. All rights reserved.
# Copyright (C) 2006 Alexey Proskuryakov ([e-mail])
# Copyright (C) 2007 Matt Lilek ([e-mail])
# Copyright (C) 2007 Eric Seidel <[e-mail]>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer. 
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution. 
# 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of
#     its contributors may be used to endorse or promote products derived
#     from this software without specific prior written permission. 
#
# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# Script to run the WebKit Open Source Project layout tests.

# Run all the tests passed in on the command line.
# If no tests are passed, find all the .html, .shtml, .xml, .xhtml, .pl, .php (and svg) files in the test directory.

# Run each text.
# Compare against the existing file xxx-expected.txt.
# If there is a mismatch, generate xxx-actual.txt and xxx-diffs.txt.

# At the end, report:
#   the number of tests that got the expected results
#   the number of tests that ran, but did not get the expected results
#   the number of tests that failed to run
#   the number of tests that were run but had no expected results to compare against

use strict;
use warnings;

use Cwd;
use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
use File::Basename;
use File::Copy;
use File::Find;
use File::Path;
use File::Spec;
use File::Spec::Functions;
use FindBin;
use Getopt::Long;
use IPC::Open2;
use IPC::Open3;
use Time::HiRes qw(time usleep);

use List::Util 'shuffle';

use lib $FindBin::Bin;
use webkitdirs;
use POSIX;

sub openDumpTool();
sub closeDumpTool();
sub dumpToolDidCrash();
sub closeHTTPD();
sub countAndPrintLeaks($$$);
sub fileNameWithNumber($$);
sub numericcmp($$);
sub openHTTPDIfNeeded();
sub pathcmp($$);
sub processIgnoreTests($$);
sub slowestcmp($$);
sub splitpath($);
sub stripExtension($);
sub isTextOnlyTest($);
sub expectedDirectoryForTest($;$;$);
sub countFinishedTest($$$$);
sub testCrashedOrTimedOut($$$$$);
sub sampleDumpTool();
sub printFailureMessageForTest($$);
sub toURL($);
sub toWindowsPath($);
sub closeCygpaths();
sub validateSkippedArg($$;$);
sub htmlForResultsSection(\@$&);
sub deleteExpectedAndActualResults($);
sub recordActualResultsAndDiff($$);
sub buildPlatformHierarchy();
sub epiloguesAndPrologues($$);
sub parseLeaksandPrintUniqueLeaks();
sub readFromDumpToolWithTimer(*;$);
sub setFileHandleNonBlocking(*$);
sub writeToFile($$);

# Argument handling
my $addPlatformExceptions = 0;
my $complexText = 0;
my $configuration = configuration();
my $guardMalloc = '';
my $httpdPort = 8000;
my $httpdSSLPort = 8443;
my $ignoreTests = '';
my $launchSafari = 1;
my $platform;
my $pixelTests = '';
my $quiet = '';
my $report10Slowest = 0;
my $resetResults = 0;
my $shouldCheckLeaks = 0;
my $showHelp = 0;
my $testsPerDumpTool;
my $testHTTP = 1;
my $testMedia = 1;
my $testResultsDirectory = "/tmp/layout-test-results";
my $threaded = 0;
my $tolerance = 0;
my $treatSkipped = "default";
my $verbose = 0;
my $useValgrind = 0;
my $strictTesting = 0;
my $generateNewResults = isAppleMacWebKit() ? 1 : 0;
my $stripEditingCallbacks = isCygwin();
my $runSample = 1;
my $root;
my $reverseTests = 0;
my $randomizeTests = 0;
my $mergeDepth;
my @leaksFilenames;
my $run64Bit;

# Default to --no-http for Qt, Gtk and wx for now.
$testHTTP = 0 if (isQt() || isGtk() || isWx());

my $expectedTag = "expected";
my $actualTag = "actual";
my $diffsTag = "diffs";
my $errorTag = "stderr";

if (isTiger()) {
    $platform = "mac-tiger";
    $tolerance = 1.0;
} elsif (isLeopard()) {
    $platform = "mac-leopard";
    $tolerance = 0.1;
} elsif (isSnowLeopard()) {
    $platform = "mac-snowleopard";
    $tolerance = 0.1;
} elsif (isAppleMacWebKit()) {
    $platform = "mac";
} elsif (isQt()) {
    $platform = "qt";
} elsif (isGtk()) {
    $platform = "gtk";
} elsif (isCygwin()) {
    $platform = "win";
}

if (!defined($platform)) {
    print "WARNING: Your platform is not recognized. Any platform-specific results will be generated in platform/undefined.\n";
    $platform = "undefined";
}

my $programName = basename($0);
my $launchSafariDefault = $launchSafari ? "launch" : "do not launch";
my $httpDefault = $testHTTP ? "run" : "do not run";
my $sampleDefault = $runSample ? "run" : "do not run";

# FIXME: "--strict" should be renamed to qt-mac-comparison, or something along those lines.
my $usage = <<EOF;
Usage: $programName [options] [testdir|testpath ...]
  --add-platform-exceptions       Put new results for non-platform-specific failing tests into the platform-specific results directory
  --complex-text                  Use the complex text code path for all text (Mac OS X and Windows only)
  -c|--configuration config       Set DumpRenderTree build configuration
  -g|--guard-malloc               Enable malloc guard
  --help                          Show this help message
  --[no-]http                     Run (or do not run) http tests (default: $httpDefault)
  -i|--ignore-tests               Comma-separated list of directories or tests to ignore
  --[no-]launch-safari            Launch (or do not launch) Safari to display test results (default: $launchSafariDefault)
  -l|--leaks                      Enable leaks checking
  --[no-]new-test-results         Generate results for new tests
  -p|--pixel-tests                Enable pixel tests
  --tolerance t                   Ignore image differences less than this percentage (default: $tolerance)
  --platform                      Override the detected platform to use for tests and results (default: $platform)
  --port                          Web server port to use with http tests
  -q|--quiet                      Less verbose output
  --reset-results                 Reset ALL results (including pixel tests if --pixel-tests is set)
  -o|--results-directory          Output results directory (default: $testResultsDirectory)
  --random                        Run the tests in a random order
  --reverse                       Run the tests in reverse alphabetical order
  --root                          Path to root tools build
  --[no-]sample-on-timeout        Run sample on timeout (default: $sampleDefault) (Mac OS X only)
  -1|--singly                     Isolate each test case run (implies --verbose)
  --skipped=[default|ignore|only] Specifies how to treat the Skipped file
                                     default: Tests/directories listed in the Skipped file are not tested
                                     ignore:  The Skipped file is ignored
                                     only:    Only those tests/directories listed in the Skipped file will be run
  --slowest                       Report the 10 slowest tests
  --strict                        Do a comparison with the output on Mac (Qt only)
  --[no-]strip-editing-callbacks  Remove editing callbacks from expected results
  -t|--threaded                   Run a concurrent JavaScript thead with each test
  --valgrind                      Run DumpRenderTree inside valgrind (Qt/Linux only)
  -v|--verbose                    More verbose output (overrides --quiet)
  -m|--merge-leak-depth arg       Merges leak callStacks and prints the number of unique leaks beneath a callstack depth of arg.  Defaults to 5.
  --64-bit                        run in 64bit mode
EOF

# Process @ARGV for configuration switches before calling GetOptions()
$configuration = passedConfiguration() || configuration();

my $getOptionsResult = GetOptions(
    'complex-text' => \$complexText,
    'c|configuration=s' => \$configuration,
    'guard-malloc|g' => \$guardMalloc,
    'help' => \$showHelp,
    'http!' => \$testHTTP,
    'ignore-tests|i=s' => \$ignoreTests,
    'launch-safari!' => \$launchSafari,
    'leaks|l' => \$shouldCheckLeaks,
    'pixel-tests|p' => \$pixelTests,
    'platform=s' => \$platform,
    'port=i' => \$httpdPort,
    'quiet|q' => \$quiet,
    'reset-results' => \$resetResults,
    'new-test-results!' => \$generateNewResults,
    'results-directory|o=s' => \$testResultsDirectory,
    'singly|1' => sub { $testsPerDumpTool = 1; },
    'nthly=i' => \$testsPerDumpTool,
    'skipped=s' => \&validateSkippedArg,
    'slowest' => \$report10Slowest,
    'threaded|t' => \$threaded,
    'tolerance=f' => \$tolerance,
    'verbose|v' => \$verbose,
    'valgrind' => \$useValgrind,
    'sample-on-timeout!' => \$runSample,
    'strict' => \$strictTesting,
    'strip-editing-callbacks!' => \$stripEditingCallbacks,
    'random' => \$randomizeTests,
    'reverse' => \$reverseTests,
    'root=s' => \$root,
    'add-platform-exceptions' => \$addPlatformExceptions,
    'merge-leak-depth|m:5' => \$mergeDepth,
    '64-bit!' => \$run64Bit
);

if (!$getOptionsResult || $showHelp) {
    print STDERR $usage;
    exit 1;
}

my $ignoreSkipped = $treatSkipped eq "ignore";
my $skippedOnly = $treatSkipped eq "only";

!$skippedOnly || @ARGV == 0 or die "--skipped=only cannot be used when tests are specified on the command line.";

setConfiguration($configuration);

$testsPerDumpTool = 1000 if !$testsPerDumpTool;

$verbose = 1 if $testsPerDumpTool == 1;

if ($shouldCheckLeaks && $testsPerDumpTool > 1000) {
    print STDERR "\nWARNING: Running more than 1000 tests at a time with MallocStackLogging enabled may cause a crash.\n\n";
}

# Stack logging does not play well with QuickTime on Tiger (rdar://problem/5537157)
$testMedia = 0 if $shouldCheckLeaks && isTiger();

setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root));
my $productDir = productDir();
$productDir .= "/bin" if isQt();
$productDir .= "/Programs" if isGtk();
setRun64Bit($run64Bit);

chdirWebKit();

if (!defined($root)) {
    # Push the parameters to build-dumprendertree as an array
    my @args = argumentsForConfiguration();

    if (isAppleMacWebKit()) {
        my $arch = preferredArchitecture();
        push(@args, "ARCHS=$arch");
    }

    my $buildResult = system "WebKitTools/Scripts/build-dumprendertree", @args;
    if ($buildResult) {
        print STDERR "Compiling DumpRenderTree failed!\n";
        exit exitStatus($buildResult);
    }
}

my $dumpToolName = "DumpRenderTree";
$dumpToolName .= "_debug" if isCygwin() && $configuration ne "Release";
my $dumpTool = "$productDir/$dumpToolName";
die "can't find executable $dumpToolName (looked in $productDir)\n" unless -x $dumpTool;

my $imageDiffTool = "$productDir/ImageDiff";
$imageDiffTool .= "_debug" if isCygwin() && $configuration ne "Release";
die "can't find executable $imageDiffTool (looked in $productDir)\n" if $pixelTests && !-x $imageDiffTool;

checkFrameworks() unless isCygwin();

my $layoutTestsName = "LayoutTests";
my $testDirectory = File::Spec->rel2abs($layoutTestsName);
my $expectedDirectory = $testDirectory;
my $platformBaseDirectory = catdir($testDirectory, "platform");
my $platformTestDirectory = catdir($platformBaseDirectory, $platform);
my @platformHierarchy = buildPlatformHierarchy();

$expectedDirectory = $ENV{"WebKitExpectedTestResultsDirectory"} if $ENV{"WebKitExpectedTestResultsDirectory"};

my $testResults = catfile($testResultsDirectory, "results.html");

print "Running tests from $testDirectory\n";
if ($pixelTests) {
    print "Enabling pixel tests with a tolerance of $tolerance%\n";
    if (isDarwin()) {
        print "WARNING: Temporarily changing the main display color profile:\n";
        print "\tThe colors on your screen will change for the duration of the testing.\n";
        print "\tThis allows the pixel tests to have consistent color values across all machines.\n";
        
        if (isPerianInstalled()) {
            print "WARNING: Perian's QuickTime component is installed and this may affect pixel test results!\n";
            print "\tYou should avoid generating new pixel results in this environment.\n";
            print "\tSee https://bugs.webkit.org/show_bug.cgi?id=22615 for details.\n";
        }
    }
}

my @tests = ();
my %testType = ();

system "ln", "-s", $testDirectory, "/tmp/LayoutTests" unless -x "/tmp/LayoutTests";

my %ignoredFiles = ();
my %ignoredDirectories = map { $_ => 1 } qw(platform);
my %ignoredLocalDirectories = map { $_ => 1 } qw(.svn _svn resources);
my %supportedFileExtensions = map { $_ => 1 } qw(html shtml xml xhtml pl php);

# FIXME: We should fix webkitdirs.pm:hasSVG/WMLSupport() to do the correct feature detection for Cygwin.
if (checkWebCoreSVGSupport(0)) { 
    $supportedFileExtensions{'svg'} = 1;
} elsif (isCygwin()) {
    $supportedFileExtensions{'svg'} = 1;
} else {
    $ignoredLocalDirectories{'svg'} = 1;
}

if (checkWebCoreWMLSupport(0)) { 
    $supportedFileExtensions{'wml'} = 1;
} else {
    $ignoredDirectories{'http/tests/wml'} = 1;
    $ignoredDirectories{'fast/wml'} = 1;
    $ignoredDirectories{'wml'} = 1;
}

if (!$testHTTP) {
    $ignoredDirectories{'http'} = 1;
}

if (!$testMedia) {
    $ignoredDirectories{'media'} = 1;
    $ignoredDirectories{'http/tests/media'} = 1;
}

if (!checkWebCoreAcceleratedCompositingSupport(0)) {
    $ignoredDirectories{'compositing'} = 1;
}

if (!checkWebCore3DTransformsSupport(0)) {
    $ignoredDirectories{'animations/3d'} = 1;
    $ignoredDirectories{'transforms/3d'} = 1;