/*-------------------------------------------------------------------------
|
* drawElements Quality Program Test Executor
|
* ------------------------------------------
|
*
|
* Copyright 2014 The Android Open Source Project
|
*
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
* you may not use this file except in compliance with the License.
|
* You may obtain a copy of the License at
|
*
|
* http://www.apache.org/licenses/LICENSE-2.0
|
*
|
* Unless required by applicable law or agreed to in writing, software
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
* See the License for the specific language governing permissions and
|
* limitations under the License.
|
*
|
*//*!
|
* \file
|
* \brief Command line test executor.
|
*//*--------------------------------------------------------------------*/
|
|
#include "xeBatchExecutor.hpp"
|
#include "xeLocalTcpIpLink.hpp"
|
#include "xeTcpIpLink.hpp"
|
#include "xeTestCaseListParser.hpp"
|
#include "xeTestLogWriter.hpp"
|
#include "xeTestResultParser.hpp"
|
|
#include "deCommandLine.hpp"
|
#include "deDirectoryIterator.hpp"
|
#include "deStringUtil.hpp"
|
#include "deUniquePtr.hpp"
|
|
#include "deString.h"
|
|
#include <algorithm>
|
#include <cstdio>
|
#include <cstdlib>
|
#include <fstream>
|
#include <iostream>
|
#include <memory>
|
#include <sstream>
|
#include <string>
|
#include <vector>
|
|
#if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID) || (DE_OS == DE_OS_WIN32)
|
# include <signal.h>
|
#endif
|
|
using std::vector;
|
using std::string;
|
|
namespace
|
{
|
|
// Command line arguments.
|
namespace opt
|
{
|
|
DE_DECLARE_COMMAND_LINE_OPT(StartServer, string);
|
DE_DECLARE_COMMAND_LINE_OPT(Host, string);
|
DE_DECLARE_COMMAND_LINE_OPT(Port, int);
|
DE_DECLARE_COMMAND_LINE_OPT(CaseListDir, string);
|
DE_DECLARE_COMMAND_LINE_OPT(TestSet, vector<string>);
|
DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet, vector<string>);
|
DE_DECLARE_COMMAND_LINE_OPT(ContinueFile, string);
|
DE_DECLARE_COMMAND_LINE_OPT(TestLogFile, string);
|
DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile, string);
|
DE_DECLARE_COMMAND_LINE_OPT(Summary, bool);
|
|
// TargetConfiguration
|
DE_DECLARE_COMMAND_LINE_OPT(BinaryName, string);
|
DE_DECLARE_COMMAND_LINE_OPT(WorkingDir, string);
|
DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs, string);
|
|
void parseCommaSeparatedList (const char* src, vector<string>* dst)
|
{
|
std::istringstream inStr (src);
|
string comp;
|
|
while (std::getline(inStr, comp, ','))
|
dst->push_back(comp);
|
}
|
|
void registerOptions (de::cmdline::Parser& parser)
|
{
|
using de::cmdline::Option;
|
using de::cmdline::NamedValue;
|
|
static const NamedValue<bool> s_yesNo[] =
|
{
|
{ "yes", true },
|
{ "no", false }
|
};
|
|
parser << Option<StartServer> ("s", "start-server", "Start local execserver. Path to the execserver binary.")
|
<< Option<Host> ("c", "connect", "Connect to host. Address of the execserver.")
|
<< Option<Port> ("p", "port", "TCP port of the execserver.", "50016")
|
<< Option<CaseListDir> ("cd", "caselistdir", "Path to the directory containing test case XML files.", ".")
|
<< Option<TestSet> ("t", "testset", "Comma-separated list of include filters.", parseCommaSeparatedList)
|
<< Option<ExcludeSet> ("e", "exclude", "Comma-separated list of exclude filters.", parseCommaSeparatedList, "")
|
<< Option<ContinueFile> (DE_NULL, "continue", "Continue execution by initializing results from existing test log.")
|
<< Option<TestLogFile> ("o", "out", "Output test log filename.", "TestLog.qpa")
|
<< Option<InfoLogFile> ("i", "info", "Output info log filename.", "InfoLog.txt")
|
<< Option<Summary> (DE_NULL, "summary", "Print summary after running tests.", s_yesNo, "yes")
|
<< Option<BinaryName> ("b", "binaryname", "Test binary path. Relative to working directory.", "<Unused>")
|
<< Option<WorkingDir> ("wd", "workdir", "Working directory for the test execution.", ".")
|
<< Option<CmdLineArgs> (DE_NULL, "cmdline", "Additional command line arguments for the test binary.", "");
|
}
|
|
} // opt
|
|
enum RunMode
|
{
|
RUNMODE_CONNECT,
|
RUNMODE_START_SERVER
|
};
|
|
struct CommandLine
|
{
|
CommandLine (void)
|
: port (0)
|
, summary (false)
|
{
|
}
|
|
xe::TargetConfiguration targetCfg;
|
RunMode runMode;
|
string serverBinOrAddress;
|
int port;
|
string caseListDir;
|
vector<string> testset;
|
vector<string> exclude;
|
string inFile;
|
string outFile;
|
string infoFile;
|
bool summary;
|
};
|
|
bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
|
{
|
de::cmdline::Parser parser;
|
de::cmdline::CommandLine opts;
|
|
XE_CHECK(argc >= 1);
|
|
opt::registerOptions(parser);
|
|
if (!parser.parse(argc-1, argv+1, &opts, std::cerr))
|
{
|
std::cout << argv[0] << " [options]\n";
|
parser.help(std::cout);
|
return false;
|
}
|
|
if (opts.hasOption<opt::StartServer>() && opts.hasOption<opt::Host>())
|
{
|
std::cout << "Invalid command line arguments. Both --start-server and --connect defined." << std::endl;
|
return false;
|
}
|
else if (!opts.hasOption<opt::StartServer>() && !opts.hasOption<opt::Host>())
|
{
|
std::cout << "Invalid command line arguments. Must define either --start-server or --connect." << std::endl;
|
return false;
|
}
|
|
if (!opts.hasOption<opt::TestSet>())
|
{
|
std::cout << "Invalid command line arguments. --testset not defined." << std::endl;
|
return false;
|
}
|
|
if (opts.hasOption<opt::StartServer>())
|
{
|
cmdLine.runMode = RUNMODE_START_SERVER;
|
cmdLine.serverBinOrAddress = opts.getOption<opt::StartServer>();
|
}
|
else
|
{
|
cmdLine.runMode = RUNMODE_CONNECT;
|
cmdLine.serverBinOrAddress = opts.getOption<opt::Host>();
|
}
|
|
if (opts.hasOption<opt::ContinueFile>())
|
{
|
cmdLine.inFile = opts.getOption<opt::ContinueFile>();
|
|
if (cmdLine.inFile.empty())
|
{
|
std::cout << "Invalid command line arguments. --continue argument is empty." << std::endl;
|
return false;
|
}
|
}
|
|
cmdLine.port = opts.getOption<opt::Port>();
|
cmdLine.caseListDir = opts.getOption<opt::CaseListDir>();
|
cmdLine.testset = opts.getOption<opt::TestSet>();
|
cmdLine.exclude = opts.getOption<opt::ExcludeSet>();
|
cmdLine.outFile = opts.getOption<opt::TestLogFile>();
|
cmdLine.infoFile = opts.getOption<opt::InfoLogFile>();
|
cmdLine.summary = opts.getOption<opt::Summary>();
|
cmdLine.targetCfg.binaryName = opts.getOption<opt::BinaryName>();
|
cmdLine.targetCfg.workingDir = opts.getOption<opt::WorkingDir>();
|
cmdLine.targetCfg.cmdLineArgs = opts.getOption<opt::CmdLineArgs>();
|
|
return true;
|
}
|
|
bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup)
|
{
|
int ptrnPos = 0;
|
int casePos = 0;
|
|
for (;;)
|
{
|
char c = casePath[casePos];
|
char p = pattern[ptrnPos];
|
|
if (p == '*')
|
{
|
/* Recurse to rest of positions. */
|
int next = casePos;
|
for (;;)
|
{
|
if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup))
|
return DE_TRUE;
|
|
if (casePath[next] == 0)
|
return DE_FALSE; /* No match found. */
|
else
|
next += 1;
|
}
|
DE_ASSERT(DE_FALSE);
|
}
|
else if (c == 0 && p == 0)
|
return true;
|
else if (c == 0)
|
{
|
/* Incomplete match is ok for test groups. */
|
return isTestGroup;
|
}
|
else if (c != p)
|
return false;
|
|
casePos += 1;
|
ptrnPos += 1;
|
}
|
|
DE_ASSERT(false);
|
return false;
|
}
|
|
void readCaseList (xe::TestGroup* root, const char* filename)
|
{
|
xe::TestCaseListParser caseListParser;
|
std::ifstream in (filename, std::ios_base::binary);
|
deUint8 buf[1024];
|
|
XE_CHECK(in.good());
|
|
caseListParser.init(root);
|
|
for (;;)
|
{
|
in.read((char*)&buf[0], sizeof(buf));
|
int numRead = (int)in.gcount();
|
|
if (numRead > 0)
|
caseListParser.parse(&buf[0], numRead);
|
|
if (numRead < (int)sizeof(buf))
|
break; // EOF
|
}
|
}
|
|
void readCaseLists (xe::TestRoot& root, const char* caseListDir)
|
{
|
int testCaseListCount = 0;
|
de::DirectoryIterator iter (caseListDir);
|
|
for (; iter.hasItem(); iter.next())
|
{
|
de::FilePath item = iter.getItem();
|
|
if (item.getType() == de::FilePath::TYPE_FILE)
|
{
|
string baseName = item.getBaseName();
|
if (baseName.find("-cases.xml") == baseName.length()-10)
|
{
|
string packageName = baseName.substr(0, baseName.length()-10);
|
xe::TestGroup* package = root.createGroup(packageName.c_str(), "");
|
|
readCaseList(package, item.getPath());
|
testCaseListCount++;
|
}
|
}
|
}
|
|
if (testCaseListCount == 0)
|
throw xe::Error("Couldn't find test case lists from test case list directory: '" + string(caseListDir) + "'");
|
}
|
|
void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
|
{
|
for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
|
{
|
const xe::TestNode* child = group.getChild(childNdx);
|
const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP;
|
const string fullPath = child->getFullPath();
|
|
if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
|
{
|
if (isGroup)
|
{
|
// Recurse into group.
|
addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
|
}
|
else
|
{
|
DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
|
testSet.add(child);
|
}
|
}
|
}
|
}
|
|
void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
|
{
|
for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
|
{
|
const xe::TestNode* child = group.getChild(childNdx);
|
const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP;
|
const string fullPath = child->getFullPath();
|
|
if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
|
{
|
if (isGroup)
|
{
|
// Recurse into group.
|
removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
|
}
|
else
|
{
|
DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
|
testSet.remove(child);
|
}
|
}
|
}
|
}
|
|
class BatchResultHandler : public xe::TestLogHandler
|
{
|
public:
|
BatchResultHandler (xe::BatchResult* batchResult)
|
: m_batchResult(batchResult)
|
{
|
}
|
|
void setSessionInfo (const xe::SessionInfo& sessionInfo)
|
{
|
m_batchResult->getSessionInfo() = sessionInfo;
|
}
|
|
xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
|
{
|
// \todo [2012-11-01 pyry] What to do with duplicate results?
|
if (m_batchResult->hasTestCaseResult(casePath))
|
return m_batchResult->getTestCaseResult(casePath);
|
else
|
return m_batchResult->createTestCaseResult(casePath);
|
}
|
|
void testCaseResultUpdated (const xe::TestCaseResultPtr&)
|
{
|
}
|
|
void testCaseResultComplete (const xe::TestCaseResultPtr&)
|
{
|
}
|
|
private:
|
xe::BatchResult* m_batchResult;
|
};
|
|
void readLogFile (xe::BatchResult* batchResult, const char* filename)
|
{
|
std::ifstream in (filename, std::ifstream::binary|std::ifstream::in);
|
BatchResultHandler handler (batchResult);
|
xe::TestLogParser parser (&handler);
|
deUint8 buf [1024];
|
int numRead = 0;
|
|
for (;;)
|
{
|
in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
|
numRead = (int)in.gcount();
|
|
if (numRead <= 0)
|
break;
|
|
parser.parse(&buf[0], numRead);
|
}
|
|
in.close();
|
}
|
|
void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult)
|
{
|
int countByStatusCode[xe::TESTSTATUSCODE_LAST];
|
std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0);
|
|
for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter)
|
{
|
const xe::TestNode* node = *iter;
|
if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node))
|
{
|
const xe::TestCase* testCase = static_cast<const xe::TestCase*>(node);
|
string fullPath;
|
xe::TestStatusCode statusCode = xe::TESTSTATUSCODE_PENDING;
|
testCase->getFullPath(fullPath);
|
|
// Parse result data if such exists.
|
if (batchResult.hasTestCaseResult(fullPath.c_str()))
|
{
|
xe::ConstTestCaseResultPtr resultData = batchResult.getTestCaseResult(fullPath.c_str());
|
xe::TestCaseResult result;
|
xe::TestResultParser parser;
|
|
xe::parseTestCaseResultFromData(&parser, &result, *resultData.get());
|
statusCode = result.statusCode;
|
}
|
|
countByStatusCode[statusCode] += 1;
|
}
|
}
|
|
printf("\nTest run summary:\n");
|
int totalCases = 0;
|
for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++)
|
{
|
if (countByStatusCode[code] > 0)
|
printf(" %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]);
|
|
totalCases += countByStatusCode[code];
|
}
|
printf(" %20s: %5d\n", "Total", totalCases);
|
}
|
|
void writeInfoLog (const xe::InfoLog& log, const char* filename)
|
{
|
std::ofstream out(filename, std::ios_base::binary);
|
XE_CHECK(out.good());
|
out.write((const char*)log.getBytes(), log.getSize());
|
out.close();
|
}
|
|
xe::CommLink* createCommLink (const CommandLine& cmdLine)
|
{
|
if (cmdLine.runMode == RUNMODE_START_SERVER)
|
{
|
xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink();
|
try
|
{
|
link->start(cmdLine.serverBinOrAddress.c_str(), DE_NULL, cmdLine.port);
|
return link;
|
}
|
catch (...)
|
{
|
delete link;
|
throw;
|
}
|
}
|
else if (cmdLine.runMode == RUNMODE_CONNECT)
|
{
|
de::SocketAddress address;
|
|
address.setFamily(DE_SOCKETFAMILY_INET4);
|
address.setProtocol(DE_SOCKETPROTOCOL_TCP);
|
address.setHost(cmdLine.serverBinOrAddress.c_str());
|
address.setPort(cmdLine.port);
|
|
xe::TcpIpLink* link = new xe::TcpIpLink();
|
try
|
{
|
std::string error;
|
|
link->connect(address);
|
return link;
|
}
|
catch (const std::exception& error)
|
{
|
delete link;
|
throw xe::Error("Failed to connect to ExecServer at: " + cmdLine.serverBinOrAddress + ":" + de::toString(cmdLine.port) + ", " + error.what());
|
}
|
catch (...)
|
{
|
delete link;
|
throw;
|
}
|
}
|
else
|
{
|
DE_ASSERT(false);
|
return DE_NULL;
|
}
|
}
|
|
#if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID)
|
|
static xe::BatchExecutor* s_executor = DE_NULL;
|
|
void signalHandler (int, siginfo_t*, void*)
|
{
|
if (s_executor)
|
s_executor->cancel();
|
}
|
|
void setupSignalHandler (xe::BatchExecutor* executor)
|
{
|
s_executor = executor;
|
struct sigaction sa;
|
|
sa.sa_sigaction = signalHandler;
|
sa.sa_flags = SA_SIGINFO | SA_RESTART;
|
sigfillset(&sa.sa_mask);
|
|
sigaction(SIGINT, &sa, DE_NULL);
|
}
|
|
void resetSignalHandler (void)
|
{
|
struct sigaction sa;
|
|
sa.sa_handler = SIG_DFL;
|
sa.sa_flags = SA_RESTART;
|
sigfillset(&sa.sa_mask);
|
|
sigaction(SIGINT, &sa, DE_NULL);
|
s_executor = DE_NULL;
|
}
|
|
#elif (DE_OS == DE_OS_WIN32)
|
|
static xe::BatchExecutor* s_executor = DE_NULL;
|
|
void signalHandler (int)
|
{
|
if (s_executor)
|
s_executor->cancel();
|
}
|
|
void setupSignalHandler (xe::BatchExecutor* executor)
|
{
|
s_executor = executor;
|
signal(SIGINT, signalHandler);
|
}
|
|
void resetSignalHandler (void)
|
{
|
signal(SIGINT, SIG_DFL);
|
s_executor = DE_NULL;
|
}
|
|
#else
|
|
void setupSignalHandler (xe::BatchExecutor*)
|
{
|
}
|
|
void resetSignalHandler (void)
|
{
|
}
|
|
#endif
|
|
void runExecutor (const CommandLine& cmdLine)
|
{
|
xe::TestRoot root;
|
|
// Read case list definitions.
|
readCaseLists(root, cmdLine.caseListDir.c_str());
|
|
// Build test set.
|
xe::TestSet testSet;
|
|
// Build test set.
|
for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter)
|
addMatchingCases(root, testSet, filterIter->c_str());
|
|
if (testSet.empty())
|
throw xe::Error("None of the test case lists contains tests matching any of the test sets.");
|
|
// Remove excluded cases.
|
for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter)
|
removeMatchingCases(root, testSet, filterIter->c_str());
|
|
// Initialize batch result.
|
xe::BatchResult batchResult;
|
xe::InfoLog infoLog;
|
|
// Read existing results from input file (if supplied).
|
if (!cmdLine.inFile.empty())
|
readLogFile(&batchResult, cmdLine.inFile.c_str());
|
|
// Initialize commLink.
|
de::UniquePtr<xe::CommLink> commLink(createCommLink(cmdLine));
|
|
xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog);
|
|
try
|
{
|
setupSignalHandler(&executor);
|
executor.run();
|
resetSignalHandler();
|
}
|
catch (...)
|
{
|
resetSignalHandler();
|
|
if (!cmdLine.outFile.empty())
|
{
|
xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
|
printf("Test log written to %s\n", cmdLine.outFile.c_str());
|
}
|
|
if (!cmdLine.infoFile.empty())
|
{
|
writeInfoLog(infoLog, cmdLine.infoFile.c_str());
|
printf("Info log written to %s\n", cmdLine.infoFile.c_str());
|
}
|
|
if (cmdLine.summary)
|
printBatchResultSummary(&root, testSet, batchResult);
|
|
throw;
|
}
|
|
if (!cmdLine.outFile.empty())
|
{
|
xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
|
printf("Test log written to %s\n", cmdLine.outFile.c_str());
|
}
|
|
if (!cmdLine.infoFile.empty())
|
{
|
writeInfoLog(infoLog, cmdLine.infoFile.c_str());
|
printf("Info log written to %s\n", cmdLine.infoFile.c_str());
|
}
|
|
if (cmdLine.summary)
|
printBatchResultSummary(&root, testSet, batchResult);
|
|
{
|
string err;
|
|
if (commLink->getState(err) == xe::COMMLINKSTATE_ERROR)
|
throw xe::Error(err);
|
}
|
}
|
|
} // anonymous
|
|
int main (int argc, const char* const* argv)
|
{
|
CommandLine cmdLine;
|
|
if (!parseCommandLine(cmdLine, argc, argv))
|
return -1;
|
|
try
|
{
|
runExecutor(cmdLine);
|
}
|
catch (const std::exception& e)
|
{
|
printf("%s\n", e.what());
|
return -1;
|
}
|
|
return 0;
|
}
|
