#!/usr/bin/env python2.7
|
|
# Copyright 2015, VIXL authors
|
# All rights reserved.
|
#
|
# Redistribution and use in source and binary forms, with or without
|
# modification, are permitted provided that the following conditions are met:
|
#
|
# * Redistributions of source code must retain the above copyright notice,
|
# this list of conditions and the following disclaimer.
|
# * Redistributions in binary form must reproduce the above copyright notice,
|
# this list of conditions and the following disclaimer in the documentation
|
# and/or other materials provided with the distribution.
|
# * Neither the name of ARM Limited nor the names of its contributors may be
|
# used to endorse or promote products derived from this software without
|
# specific prior written permission.
|
#
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
|
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
|
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
import argparse
|
import fcntl
|
import git
|
import itertools
|
import multiprocessing
|
import os
|
from os.path import join
|
import platform
|
import re
|
import subprocess
|
import sys
|
import time
|
|
import config
|
import clang_format
|
import lint
|
import printer
|
import test
|
import threaded_tests
|
import util
|
|
|
dir_root = config.dir_root
|
|
def Optionify(name):
|
return '--' + name
|
|
|
# The options that can be tested are abstracted to provide an easy way to add
|
# new ones.
|
# Environment options influence the environment. They can be used for example to
|
# set the compiler used.
|
# Build options are options passed to scons, with a syntax like `scons opt=val`
|
# Runtime options are options passed to the test program.
|
# See the definition of `test_options` below.
|
|
# 'all' is a special value for the options. If specified, all other values of
|
# the option are tested.
|
class TestOption(object):
|
type_environment = 'type_environment'
|
type_build = 'type_build'
|
type_run = 'type_run'
|
|
def __init__(self, option_type, name, help,
|
val_test_choices, val_test_default = None,
|
# If unset, the user can pass any value.
|
strict_choices = True, test_independently = False):
|
self.name = name
|
self.option_type = option_type
|
self.help = help
|
self.val_test_choices = val_test_choices
|
self.strict_choices = strict_choices
|
self.test_independently = test_independently
|
if val_test_default is not None:
|
self.val_test_default = val_test_default
|
else:
|
self.val_test_default = val_test_choices[0]
|
|
def ArgList(self, to_test):
|
res = []
|
if to_test == 'all':
|
for value in self.val_test_choices:
|
if value != 'all':
|
res.append(self.GetOptionString(value))
|
else:
|
for value in to_test:
|
res.append(self.GetOptionString(value))
|
return res
|
|
class EnvironmentOption(TestOption):
|
option_type = TestOption.type_environment
|
def __init__(self, name, environment_variable_name, help,
|
val_test_choices, val_test_default = None,
|
strict_choices = True):
|
super(EnvironmentOption, self).__init__(EnvironmentOption.option_type,
|
name,
|
help,
|
val_test_choices,
|
val_test_default,
|
strict_choices = strict_choices)
|
self.environment_variable_name = environment_variable_name
|
|
def GetOptionString(self, value):
|
return self.environment_variable_name + '=' + value
|
|
|
class BuildOption(TestOption):
|
option_type = TestOption.type_build
|
def __init__(self, name, help,
|
val_test_choices, val_test_default = None,
|
strict_choices = True, test_independently = False):
|
super(BuildOption, self).__init__(BuildOption.option_type,
|
name,
|
help,
|
val_test_choices,
|
val_test_default,
|
strict_choices = strict_choices,
|
test_independently = test_independently)
|
def GetOptionString(self, value):
|
return self.name + '=' + value
|
|
|
class RuntimeOption(TestOption):
|
option_type = TestOption.type_run
|
def __init__(self, name, help,
|
val_test_choices, val_test_default = None):
|
super(RuntimeOption, self).__init__(RuntimeOption.option_type,
|
name,
|
help,
|
val_test_choices,
|
val_test_default)
|
def GetOptionString(self, value):
|
if value == 'on':
|
return Optionify(self.name)
|
else:
|
return None
|
|
|
|
environment_option_compiler = \
|
EnvironmentOption('compiler', 'CXX', 'Test for the specified compilers.',
|
val_test_choices=['all'] + config.tested_compilers,
|
strict_choices = False)
|
test_environment_options = [
|
environment_option_compiler
|
]
|
|
build_option_mode = \
|
BuildOption('mode', 'Test with the specified build modes.',
|
val_test_choices=['all'] + config.build_options_modes)
|
build_option_standard = \
|
BuildOption('std', 'Test with the specified C++ standard.',
|
val_test_choices=['all'] + config.tested_cpp_standards,
|
strict_choices = False)
|
build_option_target = \
|
BuildOption('target', 'Test with the specified isa enabled.',
|
val_test_choices=['all'] + config.build_options_target,
|
strict_choices = False, test_independently = True)
|
build_option_negative_testing = \
|
BuildOption('negative_testing', 'Test with negative testing enabled.',
|
val_test_choices=['all'] + config.build_options_negative_testing,
|
strict_choices = False, test_independently = True)
|
test_build_options = [
|
build_option_mode,
|
build_option_standard,
|
build_option_target,
|
build_option_negative_testing
|
]
|
|
test_runtime_options = []
|
|
test_options = \
|
test_environment_options + test_build_options + test_runtime_options
|
|
|
def BuildOptions():
|
args = argparse.ArgumentParser(
|
description =
|
'''This tool runs all tests matching the specified filters for multiple
|
environment, build options, and runtime options configurations.''',
|
# Print default values.
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
args.add_argument('filters', metavar='filter', nargs='*',
|
help='Run tests matching all of the (regexp) filters.')
|
|
# We automatically build the script options from the options to be tested.
|
test_arguments = args.add_argument_group(
|
'Test options',
|
'These options indicate what should be tested')
|
for option in test_options:
|
choices = option.val_test_choices if option.strict_choices else None
|
help = option.help
|
if not option.strict_choices:
|
help += ' Supported values: {' + ','.join(option.val_test_choices) + '}'
|
test_arguments.add_argument(Optionify(option.name),
|
nargs='+',
|
choices=choices,
|
default=option.val_test_default,
|
help=help,
|
action='store')
|
|
general_arguments = args.add_argument_group('General options')
|
general_arguments.add_argument('--fast', action='store_true',
|
help='''Skip the lint and clang-format tests,
|
and run only with one compiler, in one mode,
|
with one C++ standard, and with an appropriate
|
default for runtime options.''')
|
general_arguments.add_argument('--dry-run', action='store_true',
|
help='''Don't actually build or run anything,
|
but print the configurations that would be
|
tested.''')
|
general_arguments.add_argument(
|
'--jobs', '-j', metavar='N', type=int, nargs='?',
|
default=multiprocessing.cpu_count(),
|
const=multiprocessing.cpu_count(),
|
help='''Runs the tests using N jobs. If the option is set but no value is
|
provided, the script will use as many jobs as it thinks useful.''')
|
general_arguments.add_argument('--clang-format',
|
default=clang_format.DEFAULT_CLANG_FORMAT,
|
help='Path to clang-format.')
|
general_arguments.add_argument('--nobench', action='store_true',
|
help='Do not run benchmarks.')
|
general_arguments.add_argument('--nolint', action='store_true',
|
help='Do not run the linter.')
|
general_arguments.add_argument('--noclang-format', action='store_true',
|
help='Do not run clang-format.')
|
general_arguments.add_argument('--notest', action='store_true',
|
help='Do not run tests.')
|
general_arguments.add_argument('--fail-early', action='store_true',
|
help='Exit as soon as a test fails.')
|
sim_default = 'none' if platform.machine() == 'aarch64' else 'aarch64'
|
general_arguments.add_argument(
|
'--simulator', action='store', choices=['aarch64', 'none'],
|
default=sim_default,
|
help='Explicitly enable or disable the simulator.')
|
general_arguments.add_argument(
|
'--under_valgrind', action='store_true',
|
help='''Run the test-runner commands under Valgrind.
|
Note that a few tests are known to fail because of
|
issues in Valgrind''')
|
return args.parse_args()
|
|
|
def RunCommand(command, environment_options = None):
|
# Create a copy of the environment. We do not want to pollute the environment
|
# of future commands run.
|
environment = os.environ
|
# Configure the environment.
|
# TODO: We currently pass the options as strings, so we need to parse them. We
|
# should instead pass them as a data structure and build the string option
|
# later. `environment_options` looks like `['CXX=compiler', 'OPT=val']`.
|
if environment_options:
|
for option in environment_options:
|
opt, val = option.split('=')
|
environment[opt] = val
|
|
printable_command = ''
|
if environment_options:
|
printable_command += ' '.join(environment_options) + ' '
|
printable_command += ' '.join(command)
|
|
printable_command_orange = \
|
printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR
|
printer.PrintOverwritableLine(printable_command_orange)
|
sys.stdout.flush()
|
|
# Start a process for the command.
|
# Interleave `stderr` and `stdout`.
|
p = subprocess.Popen(command,
|
stdout=subprocess.PIPE,
|
stderr=subprocess.STDOUT,
|
env=environment)
|
|
# We want to be able to display a continuously updated 'work indicator' while
|
# the process is running. Since the process can hang if the `stdout` pipe is
|
# full, we need to pull from it regularly. We cannot do so via the
|
# `readline()` function because it is blocking, and would thus cause the
|
# indicator to not be updated properly. So use file control mechanisms
|
# instead.
|
indicator = ' (still working: %d seconds elapsed)'
|
|
# Mark the process output as non-blocking.
|
flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL)
|
fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
|
|
t_start = time.time()
|
t_last_indication = t_start
|
process_output = ''
|
|
# Keep looping as long as the process is running.
|
while p.poll() is None:
|
# Avoid polling too often.
|
time.sleep(0.1)
|
# Update the progress indicator.
|
t_current = time.time()
|
if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1):
|
printer.PrintOverwritableLine(
|
printable_command_orange + indicator % int(t_current - t_start))
|
sys.stdout.flush()
|
t_last_indication = t_current
|
# Pull from the process output.
|
while True:
|
try:
|
line = os.read(p.stdout.fileno(), 1024)
|
except OSError:
|
line = ''
|
break
|
if line == '': break
|
process_output += line
|
|
# The process has exited. Don't forget to retrieve the rest of its output.
|
out, err = p.communicate()
|
rc = p.poll()
|
process_output += out
|
|
if rc == 0:
|
printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR)
|
else:
|
printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR)
|
printer.Print(process_output)
|
return rc
|
|
|
def RunLinter():
|
rc, default_tracked_files = lint.GetDefaultFilesToLint()
|
if rc:
|
return rc
|
return lint.RunLinter(map(lambda x: join(dir_root, x), default_tracked_files),
|
jobs = args.jobs, progress_prefix = 'cpp lint: ')
|
|
|
def RunClangFormat():
|
return clang_format.ClangFormatFiles(clang_format.GetCppSourceFilesToFormat(),
|
args.clang_format, jobs = args.jobs,
|
progress_prefix = 'clang-format: ')
|
|
|
|
def BuildAll(build_options, jobs):
|
scons_command = ["scons", "-C", dir_root, 'all', '-j', str(jobs)]
|
scons_command += list(build_options)
|
return RunCommand(scons_command, list(environment_options))
|
|
|
# Work out if the given options or args allow to run on the specified arch.
|
# * arches is a list of ISA/architecture (a64, aarch32, etc)
|
# * options are test.py's command line options if any.
|
# * args are the arguments given to the build script.
|
def CanRunOn(arches, options, args):
|
# First we check in the build specific options.
|
for option in options:
|
if 'target' in option:
|
# The option format is 'target=x,y,z'.
|
for target in (option.split('='))[1].split(','):
|
if target in arches:
|
return True
|
|
# There was a target build option but it didn't include the target arch.
|
return False
|
|
# No specific build option, check the script arguments.
|
# The meaning of 'all' will depend on the platform, e.g. 32-bit compilers
|
# cannot handle Aarch64 while 64-bit compiler can handle Aarch32. To avoid
|
# any issues no benchmarks are run for target='all'.
|
if args.target == 'all': return False
|
|
for target in args.target[0].split(','):
|
if target in arches:
|
return True
|
|
return False
|
|
|
def CanRunAarch64(options, args):
|
return CanRunOn(['aarch64', 'a64'], options, args)
|
|
|
def CanRunAarch32(options, args):
|
return CanRunOn(['aarch32', 'a32', 't32'], options, args)
|
|
|
def RunBenchmarks(options, args):
|
rc = 0
|
if CanRunAarch32(options, args):
|
benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch32_benchmarks)
|
for bench in benchmark_names:
|
rc |= RunCommand(
|
[os.path.realpath(
|
join(config.dir_build_latest, 'benchmarks/aarch32', bench)), '10'])
|
if CanRunAarch64(options, args):
|
benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch64_benchmarks)
|
for bench in benchmark_names:
|
rc |= RunCommand(
|
[util.relrealpath(
|
join(config.dir_build_latest,
|
'benchmarks/aarch64', bench)), '10'])
|
return rc
|
|
|
def PrintStatus(success):
|
printer.Print('\n$ ' + ' '.join(sys.argv))
|
if success:
|
printer.Print('SUCCESS')
|
else:
|
printer.Print('FAILURE')
|
|
|
|
if __name__ == '__main__':
|
util.require_program('scons')
|
rc = 0
|
|
args = BuildOptions()
|
|
def MaybeExitEarly(rc):
|
if args.fail_early and rc != 0:
|
PrintStatus(rc == 0)
|
sys.exit(rc)
|
|
if args.under_valgrind:
|
util.require_program('valgrind')
|
|
if args.fast:
|
def SetFast(option, specified, default):
|
option.val_test_choices = \
|
[default if specified == 'all' else specified[0]]
|
# `g++` is very slow to compile a few aarch32 test files.
|
SetFast(environment_option_compiler, args.compiler, 'clang++')
|
SetFast(build_option_standard, args.std, 'c++98')
|
SetFast(build_option_mode, args.mode, 'debug')
|
|
if not args.nolint and not (args.fast or args.dry_run):
|
rc |= RunLinter()
|
MaybeExitEarly(rc)
|
|
if not args.noclang_format and not (args.fast or args.dry_run):
|
rc |= RunClangFormat()
|
MaybeExitEarly(rc)
|
|
# List all combinations of options that will be tested.
|
def ListCombinations(args, options):
|
opts_list = [
|
opt.ArgList(args.__dict__[opt.name])
|
for opt in options
|
if not opt.test_independently
|
]
|
return list(itertools.product(*opts_list))
|
# List combinations of options that should only be tested independently.
|
def ListIndependentCombinations(args, options, base):
|
n = []
|
for opt in options:
|
if opt.test_independently:
|
for o in opt.ArgList(args.__dict__[opt.name]):
|
n.append(base + (o,))
|
return n
|
# TODO: We should refine the configurations we test by default, instead of
|
# always testing all possible combinations.
|
test_env_combinations = ListCombinations(args, test_environment_options)
|
test_build_combinations = ListCombinations(args, test_build_options)
|
if not args.fast:
|
test_build_combinations.extend(
|
ListIndependentCombinations(args,
|
test_build_options,
|
test_build_combinations[0]))
|
test_runtime_combinations = ListCombinations(args, test_runtime_options)
|
|
for environment_options in test_env_combinations:
|
for build_options in test_build_combinations:
|
if (args.dry_run):
|
for runtime_options in test_runtime_combinations:
|
print(' '.join(filter(None, environment_options)) + ', ' +
|
' '.join(filter(None, build_options)) + ', ' +
|
' '.join(filter(None, runtime_options)))
|
continue
|
|
# Avoid going through the build stage if we are not using the build
|
# result.
|
if not (args.notest and args.nobench):
|
build_rc = BuildAll(build_options, args.jobs)
|
# Don't run the tests for this configuration if the build failed.
|
if build_rc != 0:
|
rc |= build_rc
|
MaybeExitEarly(rc)
|
continue
|
|
# Use the realpath of the test executable so that the commands printed
|
# can be copy-pasted and run.
|
test_executable = util.relrealpath(
|
join(config.dir_build_latest, 'test', 'test-runner'))
|
|
if not args.notest:
|
printer.Print(test_executable)
|
|
for runtime_options in test_runtime_combinations:
|
if not args.notest:
|
runtime_options = [x for x in runtime_options if x is not None]
|
prefix = ' ' + ' '.join(runtime_options) + ' '
|
rc |= threaded_tests.RunTests(test_executable,
|
args.filters,
|
list(runtime_options),
|
args.under_valgrind,
|
jobs = args.jobs, prefix = prefix)
|
MaybeExitEarly(rc)
|
|
if not args.nobench:
|
rc |= RunBenchmarks(build_options, args)
|
MaybeExitEarly(rc)
|
|
if not args.dry_run:
|
PrintStatus(rc == 0)
|
|
sys.exit(rc)
|
