#!/usr/bin/env python3
|
#
|
# Copyright (C) 2016 The Android Open Source Project
|
#
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
# you may not use this file except in compliance with the License.
|
# You may obtain a copy of the License at
|
#
|
# http://www.apache.org/licenses/LICENSE-2.0
|
#
|
# Unless required by applicable law or agreed to in writing, software
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# See the License for the specific language governing permissions and
|
# limitations under the License.
|
|
import abc
|
import argparse
|
import filecmp
|
import os
|
import shlex
|
import shutil
|
import subprocess
|
import sys
|
|
from glob import glob
|
from subprocess import DEVNULL
|
from tempfile import mkdtemp
|
|
sys.path.append(os.path.dirname(os.path.dirname(
|
os.path.realpath(__file__))))
|
|
from common.common import RetCode
|
from common.common import CommandListToCommandString
|
from common.common import FatalError
|
from common.common import GetEnvVariableOrError
|
from common.common import RunCommand
|
from common.common import RunCommandForOutput
|
from common.common import DeviceTestEnv
|
|
# Return codes supported by bisection bug search.
|
BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT)
|
|
|
def GetExecutionModeRunner(dexer, debug_info, device, mode):
|
"""Returns a runner for the given execution mode.
|
|
Args:
|
dexer: string, defines dexer
|
debug_info: boolean, if True include debugging info
|
device: string, target device serial number (or None)
|
mode: string, execution mode
|
Returns:
|
TestRunner with given execution mode
|
Raises:
|
FatalError: error for unknown execution mode
|
"""
|
if mode == 'ri':
|
return TestRunnerRIOnHost(debug_info)
|
if mode == 'hint':
|
return TestRunnerArtIntOnHost(dexer, debug_info)
|
if mode == 'hopt':
|
return TestRunnerArtOptOnHost(dexer, debug_info)
|
if mode == 'tint':
|
return TestRunnerArtIntOnTarget(dexer, debug_info, device)
|
if mode == 'topt':
|
return TestRunnerArtOptOnTarget(dexer, debug_info, device)
|
raise FatalError('Unknown execution mode')
|
|
|
#
|
# Execution mode classes.
|
#
|
|
|
class TestRunner(object):
|
"""Abstraction for running a test in a particular execution mode."""
|
__meta_class__ = abc.ABCMeta
|
|
@abc.abstractproperty
|
def description(self):
|
"""Returns a description string of the execution mode."""
|
|
@abc.abstractproperty
|
def id(self):
|
"""Returns a short string that uniquely identifies the execution mode."""
|
|
@property
|
def output_file(self):
|
return self.id + '_out.txt'
|
|
@abc.abstractmethod
|
def GetBisectionSearchArgs(self):
|
"""Get arguments to pass to bisection search tool.
|
|
Returns:
|
list of strings - arguments for bisection search tool, or None if
|
runner is not bisectable
|
"""
|
|
@abc.abstractmethod
|
def CompileAndRunTest(self):
|
"""Compile and run the generated test.
|
|
Ensures that the current Test.java in the temporary directory is compiled
|
and executed under the current execution mode. On success, transfers the
|
generated output to the file self.output_file in the temporary directory.
|
|
Most nonzero return codes are assumed non-divergent, since systems may
|
exit in different ways. This is enforced by normalizing return codes.
|
|
Returns:
|
normalized return code
|
"""
|
|
|
class TestRunnerWithHostCompilation(TestRunner):
|
"""Abstract test runner that supports compilation on host."""
|
|
def __init__(self, dexer, debug_info):
|
"""Constructor for the runner with host compilation.
|
|
Args:
|
dexer: string, defines dexer
|
debug_info: boolean, if True include debugging info
|
"""
|
self._dexer = dexer
|
self._debug_info = debug_info
|
|
def CompileOnHost(self):
|
if self._dexer == 'dx' or self._dexer == 'd8':
|
dbg = '-g' if self._debug_info else '-g:none'
|
if RunCommand(['javac', '--release=8', dbg, 'Test.java'],
|
out=None, err=None, timeout=30) == RetCode.SUCCESS:
|
dx = 'dx' if self._dexer == 'dx' else 'd8-compat-dx'
|
retc = RunCommand([dx, '--dex', '--output=classes.dex'] + glob('*.class'),
|
out=None, err='dxerr.txt', timeout=30)
|
else:
|
retc = RetCode.NOTCOMPILED
|
else:
|
raise FatalError('Unknown dexer: ' + self._dexer)
|
return retc
|
|
|
class TestRunnerRIOnHost(TestRunner):
|
"""Concrete test runner of the reference implementation on host."""
|
|
def __init__(self, debug_info):
|
"""Constructor for the runner with host compilation.
|
|
Args:
|
debug_info: boolean, if True include debugging info
|
"""
|
self._debug_info = debug_info
|
|
@property
|
def description(self):
|
return 'RI on host'
|
|
@property
|
def id(self):
|
return 'RI'
|
|
def CompileAndRunTest(self):
|
dbg = '-g' if self._debug_info else '-g:none'
|
if RunCommand(['javac', '--release=8', dbg, 'Test.java'],
|
out=None, err=None, timeout=30) == RetCode.SUCCESS:
|
retc = RunCommand(['java', 'Test'], self.output_file, err=None)
|
else:
|
retc = RetCode.NOTCOMPILED
|
return retc
|
|
def GetBisectionSearchArgs(self):
|
return None
|
|
|
class TestRunnerArtOnHost(TestRunnerWithHostCompilation):
|
"""Abstract test runner of Art on host."""
|
|
def __init__(self, dexer, debug_info, extra_args=None):
|
"""Constructor for the Art on host tester.
|
|
Args:
|
dexer: string, defines dexer
|
debug_info: boolean, if True include debugging info
|
extra_args: list of strings, extra arguments for dalvikvm
|
"""
|
super().__init__(dexer, debug_info)
|
self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex']
|
if extra_args is not None:
|
self._art_cmd += extra_args
|
self._art_cmd.append('Test')
|
|
def CompileAndRunTest(self):
|
if self.CompileOnHost() == RetCode.SUCCESS:
|
retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt')
|
else:
|
retc = RetCode.NOTCOMPILED
|
return retc
|
|
|
class TestRunnerArtIntOnHost(TestRunnerArtOnHost):
|
"""Concrete test runner of interpreter mode Art on host."""
|
|
def __init__(self, dexer, debug_info):
|
"""Constructor for the Art on host tester (interpreter).
|
|
Args:
|
dexer: string, defines dexer
|
debug_info: boolean, if True include debugging info
|
"""
|
super().__init__(dexer, debug_info, ['-Xint'])
|
|
@property
|
def description(self):
|
return 'Art interpreter on host'
|
|
@property
|
def id(self):
|
return 'HInt'
|
|
def GetBisectionSearchArgs(self):
|
return None
|
|
|
class TestRunnerArtOptOnHost(TestRunnerArtOnHost):
|
"""Concrete test runner of optimizing compiler mode Art on host."""
|
|
def __init__(self, dexer, debug_info):
|
"""Constructor for the Art on host tester (optimizing).
|
|
Args:
|
dexer: string, defines dexer
|
debug_info: boolean, if True include debugging info
|
"""
|
super().__init__(dexer, debug_info, None)
|
|
@property
|
def description(self):
|
return 'Art optimizing on host'
|
|
@property
|
def id(self):
|
return 'HOpt'
|
|
def GetBisectionSearchArgs(self):
|
cmd_str = CommandListToCommandString(
|
self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:])
|
return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
|
|
|
class TestRunnerArtOnTarget(TestRunnerWithHostCompilation):
|
"""Abstract test runner of Art on target."""
|
|
def __init__(self, dexer, debug_info, device, extra_args=None):
|
"""Constructor for the Art on target tester.
|
|
Args:
|
dexer: string, defines dexer
|
debug_info: boolean, if True include debugging info
|
device: string, target device serial number (or None)
|
extra_args: list of strings, extra arguments for dalvikvm
|
"""
|
super().__init__(dexer, debug_info)
|
self._test_env = DeviceTestEnv('jfuzz_', specific_device=device)
|
self._dalvik_cmd = ['dalvikvm']
|
if extra_args is not None:
|
self._dalvik_cmd += extra_args
|
self._device = device
|
self._device_classpath = None
|
|
def CompileAndRunTest(self):
|
if self.CompileOnHost() == RetCode.SUCCESS:
|
self._device_classpath = self._test_env.PushClasspath('classes.dex')
|
cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']
|
(output, retc) = self._test_env.RunCommand(
|
cmd, {'ANDROID_LOG_TAGS': '*:s'})
|
with open(self.output_file, 'w') as run_out:
|
run_out.write(output)
|
else:
|
retc = RetCode.NOTCOMPILED
|
return retc
|
|
def GetBisectionSearchArgs(self):
|
cmd_str = CommandListToCommandString(
|
self._dalvik_cmd + ['-cp',self._device_classpath, 'Test'])
|
cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
|
if self._device:
|
cmd += ['--device-serial', self._device]
|
else:
|
cmd.append('--device')
|
return cmd
|
|
|
class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget):
|
"""Concrete test runner of interpreter mode Art on target."""
|
|
def __init__(self, dexer, debug_info, device):
|
"""Constructor for the Art on target tester (interpreter).
|
|
Args:
|
dexer: string, defines dexer
|
debug_info: boolean, if True include debugging info
|
device: string, target device serial number (or None)
|
"""
|
super().__init__(dexer, debug_info, device, ['-Xint'])
|
|
@property
|
def description(self):
|
return 'Art interpreter on target'
|
|
@property
|
def id(self):
|
return 'TInt'
|
|
def GetBisectionSearchArgs(self):
|
return None
|
|
|
class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget):
|
"""Concrete test runner of optimizing compiler mode Art on target."""
|
|
def __init__(self, dexer, debug_info, device):
|
"""Constructor for the Art on target tester (optimizing).
|
|
Args:
|
dexer: string, defines dexer
|
debug_info: boolean, if True include debugging info
|
device: string, target device serial number (or None)
|
"""
|
super().__init__(dexer, debug_info, device, None)
|
|
@property
|
def description(self):
|
return 'Art optimizing on target'
|
|
@property
|
def id(self):
|
return 'TOpt'
|
|
def GetBisectionSearchArgs(self):
|
cmd_str = CommandListToCommandString(
|
self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'])
|
cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
|
if self._device:
|
cmd += ['--device-serial', self._device]
|
else:
|
cmd.append('--device')
|
return cmd
|
|
|
#
|
# Tester class.
|
#
|
|
|
class JFuzzTester(object):
|
"""Tester that runs JFuzz many times and report divergences."""
|
|
def __init__(self, num_tests, device, mode1, mode2, jfuzz_args,
|
report_script, true_divergence_only, dexer, debug_info):
|
"""Constructor for the tester.
|
|
Args:
|
num_tests: int, number of tests to run
|
device: string, target device serial number (or None)
|
mode1: string, execution mode for first runner
|
mode2: string, execution mode for second runner
|
jfuzz_args: list of strings, additional arguments for jfuzz
|
report_script: string, path to script called for each divergence
|
true_divergence_only: boolean, if True don't bisect timeout divergences
|
dexer: string, defines dexer
|
debug_info: boolean, if True include debugging info
|
"""
|
self._num_tests = num_tests
|
self._device = device
|
self._runner1 = GetExecutionModeRunner(dexer, debug_info, device, mode1)
|
self._runner2 = GetExecutionModeRunner(dexer, debug_info, device, mode2)
|
self._jfuzz_args = jfuzz_args
|
self._report_script = report_script
|
self._true_divergence_only = true_divergence_only
|
self._dexer = dexer
|
self._debug_info = debug_info
|
self._save_dir = None
|
self._results_dir = None
|
self._jfuzz_dir = None
|
# Statistics.
|
self._test = 0
|
self._num_success = 0
|
self._num_not_compiled = 0
|
self._num_not_run = 0
|
self._num_timed_out = 0
|
self._num_divergences = 0
|
|
def __enter__(self):
|
"""On entry, enters new temp directory after saving current directory.
|
|
Raises:
|
FatalError: error when temp directory cannot be constructed
|
"""
|
self._save_dir = os.getcwd()
|
self._results_dir = mkdtemp(dir='/tmp/')
|
self._jfuzz_dir = mkdtemp(dir=self._results_dir)
|
if self._results_dir is None or self._jfuzz_dir is None:
|
raise FatalError('Cannot obtain temp directory')
|
os.chdir(self._jfuzz_dir)
|
return self
|
|
def __exit__(self, etype, evalue, etraceback):
|
"""On exit, re-enters previously saved current directory and cleans up."""
|
os.chdir(self._save_dir)
|
shutil.rmtree(self._jfuzz_dir)
|
if self._num_divergences == 0:
|
shutil.rmtree(self._results_dir)
|
|
def Run(self):
|
"""Runs JFuzz many times and report divergences."""
|
print()
|
print('**\n**** JFuzz Testing\n**')
|
print()
|
print('#Tests :', self._num_tests)
|
print('Device :', self._device)
|
print('Directory :', self._results_dir)
|
print('Exec-mode1:', self._runner1.description)
|
print('Exec-mode2:', self._runner2.description)
|
print('Dexer :', self._dexer)
|
print('Debug-info:', self._debug_info)
|
print()
|
self.ShowStats()
|
for self._test in range(1, self._num_tests + 1):
|
self.RunJFuzzTest()
|
self.ShowStats()
|
if self._num_divergences == 0:
|
print('\n\nsuccess (no divergences)\n')
|
else:
|
print('\n\nfailure (divergences)\n')
|
|
def ShowStats(self):
|
"""Shows current statistics (on same line) while tester is running."""
|
print('\rTests:', self._test,
|
'Success:', self._num_success,
|
'Not-compiled:', self._num_not_compiled,
|
'Not-run:', self._num_not_run,
|
'Timed-out:', self._num_timed_out,
|
'Divergences:', self._num_divergences,
|
end='')
|
sys.stdout.flush()
|
|
def RunJFuzzTest(self):
|
"""Runs a single JFuzz test, comparing two execution modes."""
|
self.ConstructTest()
|
retc1 = self._runner1.CompileAndRunTest()
|
retc2 = self._runner2.CompileAndRunTest()
|
self.CheckForDivergence(retc1, retc2)
|
self.CleanupTest()
|
|
def ConstructTest(self):
|
"""Use JFuzz to generate next Test.java test.
|
|
Raises:
|
FatalError: error when jfuzz fails
|
"""
|
if (RunCommand(['jfuzz'] + self._jfuzz_args, out='Test.java', err=None)
|
!= RetCode.SUCCESS):
|
raise FatalError('Unexpected error while running JFuzz')
|
|
def CheckForDivergence(self, retc1, retc2):
|
"""Checks for divergences and updates statistics.
|
|
Args:
|
retc1: int, normalized return code of first runner
|
retc2: int, normalized return code of second runner
|
"""
|
if retc1 == retc2:
|
# No divergence in return code.
|
if retc1 == RetCode.SUCCESS:
|
# Both compilations and runs were successful, inspect generated output.
|
runner1_out = self._runner1.output_file
|
runner2_out = self._runner2.output_file
|
if not filecmp.cmp(runner1_out, runner2_out, shallow=False):
|
# Divergence in output.
|
self.ReportDivergence(retc1, retc2, is_output_divergence=True)
|
else:
|
# No divergence in output.
|
self._num_success += 1
|
elif retc1 == RetCode.TIMEOUT:
|
self._num_timed_out += 1
|
elif retc1 == RetCode.NOTCOMPILED:
|
self._num_not_compiled += 1
|
else:
|
self._num_not_run += 1
|
else:
|
# Divergence in return code.
|
if self._true_divergence_only:
|
# When only true divergences are requested, any divergence in return
|
# code where one is a time out is treated as a regular time out.
|
if RetCode.TIMEOUT in (retc1, retc2):
|
self._num_timed_out += 1
|
return
|
# When only true divergences are requested, a runtime crash in just
|
# the RI is treated as if not run at all.
|
if retc1 == RetCode.ERROR and retc2 == RetCode.SUCCESS:
|
if self._runner1.GetBisectionSearchArgs() is None:
|
self._num_not_run += 1
|
return
|
self.ReportDivergence(retc1, retc2, is_output_divergence=False)
|
|
def GetCurrentDivergenceDir(self):
|
return self._results_dir + '/divergence' + str(self._num_divergences)
|
|
def ReportDivergence(self, retc1, retc2, is_output_divergence):
|
"""Reports and saves a divergence."""
|
self._num_divergences += 1
|
print('\n' + str(self._num_divergences), end='')
|
if is_output_divergence:
|
print(' divergence in output')
|
else:
|
print(' divergence in return code: ' + retc1.name + ' vs. ' +
|
retc2.name)
|
# Save.
|
ddir = self.GetCurrentDivergenceDir()
|
os.mkdir(ddir)
|
for f in glob('*.txt') + ['Test.java']:
|
shutil.copy(f, ddir)
|
# Maybe run bisection bug search.
|
if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES:
|
self.MaybeBisectDivergence(retc1, retc2, is_output_divergence)
|
# Call reporting script.
|
if self._report_script:
|
self.RunReportScript(retc1, retc2, is_output_divergence)
|
|
def RunReportScript(self, retc1, retc2, is_output_divergence):
|
"""Runs report script."""
|
try:
|
title = "Divergence between {0} and {1} (found with fuzz testing)".format(
|
self._runner1.description, self._runner2.description)
|
# Prepare divergence comment.
|
jfuzz_cmd_and_version = subprocess.check_output(
|
['grep', '-o', 'jfuzz.*', 'Test.java'], universal_newlines=True)
|
(jfuzz_cmd_str, jfuzz_ver) = jfuzz_cmd_and_version.split('(')
|
# Strip right parenthesis and new line.
|
jfuzz_ver = jfuzz_ver[:-2]
|
jfuzz_args = ['\'-{0}\''.format(arg)
|
for arg in jfuzz_cmd_str.strip().split(' -')][1:]
|
wrapped_args = ['--jfuzz_arg={0}'.format(opt) for opt in jfuzz_args]
|
repro_cmd_str = (os.path.basename(__file__) +
|
' --num_tests=1 --dexer=' + self._dexer +
|
(' --debug_info ' if self._debug_info else ' ') +
|
' '.join(wrapped_args))
|
comment = 'jfuzz {0}\nReproduce test:\n{1}\nReproduce divergence:\n{2}\n'.format(
|
jfuzz_ver, jfuzz_cmd_str, repro_cmd_str)
|
if is_output_divergence:
|
(output, _, _) = RunCommandForOutput(
|
['diff', self._runner1.output_file, self._runner2.output_file],
|
None, subprocess.PIPE, subprocess.STDOUT)
|
comment += 'Diff:\n' + output
|
else:
|
comment += '{0} vs {1}\n'.format(retc1, retc2)
|
# Prepare report script command.
|
script_cmd = [self._report_script, title, comment]
|
ddir = self.GetCurrentDivergenceDir()
|
bisection_out_files = glob(ddir + '/*_bisection_out.txt')
|
if bisection_out_files:
|
script_cmd += ['--bisection_out', bisection_out_files[0]]
|
subprocess.check_call(script_cmd, stdout=DEVNULL, stderr=DEVNULL)
|
except subprocess.CalledProcessError as err:
|
print('Failed to run report script.\n', err)
|
|
def RunBisectionSearch(self, args, expected_retcode, expected_output,
|
runner_id):
|
ddir = self.GetCurrentDivergenceDir()
|
outfile_path = ddir + '/' + runner_id + '_bisection_out.txt'
|
logfile_path = ddir + '/' + runner_id + '_bisection_log.txt'
|
errfile_path = ddir + '/' + runner_id + '_bisection_err.txt'
|
args = list(args) + ['--logfile', logfile_path, '--cleanup']
|
args += ['--expected-retcode', expected_retcode.name]
|
if expected_output:
|
args += ['--expected-output', expected_output]
|
bisection_search_path = os.path.join(
|
GetEnvVariableOrError('ANDROID_BUILD_TOP'),
|
'art/tools/bisection_search/bisection_search.py')
|
if RunCommand([bisection_search_path] + args, out=outfile_path,
|
err=errfile_path, timeout=300) == RetCode.TIMEOUT:
|
print('Bisection search TIMEOUT')
|
|
def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence):
|
bisection_args1 = self._runner1.GetBisectionSearchArgs()
|
bisection_args2 = self._runner2.GetBisectionSearchArgs()
|
if is_output_divergence:
|
maybe_output1 = self._runner1.output_file
|
maybe_output2 = self._runner2.output_file
|
else:
|
maybe_output1 = maybe_output2 = None
|
if bisection_args1 is not None:
|
self.RunBisectionSearch(bisection_args1, retc2, maybe_output2,
|
self._runner1.id)
|
if bisection_args2 is not None:
|
self.RunBisectionSearch(bisection_args2, retc1, maybe_output1,
|
self._runner2.id)
|
|
def CleanupTest(self):
|
"""Cleans up after a single test run."""
|
for file_name in os.listdir(self._jfuzz_dir):
|
file_path = os.path.join(self._jfuzz_dir, file_name)
|
if os.path.isfile(file_path):
|
os.unlink(file_path)
|
elif os.path.isdir(file_path):
|
shutil.rmtree(file_path)
|
|
|
def main():
|
# Handle arguments.
|
parser = argparse.ArgumentParser()
|
parser.add_argument('--num_tests', default=10000, type=int,
|
help='number of tests to run')
|
parser.add_argument('--device', help='target device serial number')
|
parser.add_argument('--mode1', default='ri',
|
help='execution mode 1 (default: ri)')
|
parser.add_argument('--mode2', default='hopt',
|
help='execution mode 2 (default: hopt)')
|
parser.add_argument('--report_script',
|
help='script called for each divergence')
|
parser.add_argument('--jfuzz_arg', default=[], dest='jfuzz_args',
|
action='append',
|
help='argument for jfuzz')
|
parser.add_argument('--true_divergence', default=False, action='store_true',
|
help='do not bisect timeout divergences')
|
parser.add_argument('--dexer', default='dx', type=str,
|
help='defines dexer as dx or d8 (default: dx)')
|
parser.add_argument('--debug_info', default=False, action='store_true',
|
help='include debugging info')
|
args = parser.parse_args()
|
if args.mode1 == args.mode2:
|
raise FatalError('Identical execution modes given')
|
# Run the JFuzz tester.
|
with JFuzzTester(args.num_tests,
|
args.device,
|
args.mode1, args.mode2,
|
args.jfuzz_args,
|
args.report_script,
|
args.true_divergence,
|
args.dexer,
|
args.debug_info) as fuzzer:
|
fuzzer.Run()
|
|
if __name__ == '__main__':
|
main()
|
