# -*- coding: utf-8 -*-
|
|
#-------------------------------------------------------------------------
|
# drawElements Quality Program utilities
|
# --------------------------------------
|
#
|
# Copyright 2015 The Android Open Source Project
|
#
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
# you may not use this file except in compliance with the License.
|
# You may obtain a copy of the License at
|
#
|
# http://www.apache.org/licenses/LICENSE-2.0
|
#
|
# Unless required by applicable law or agreed to in writing, software
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# See the License for the specific language governing permissions and
|
# limitations under the License.
|
#
|
#-------------------------------------------------------------------------
|
|
import sys
|
import random
|
import string
|
import subprocess
|
from optparse import OptionParser
|
|
def all (results, predicate):
|
for result in results:
|
if not predicate(result):
|
return False
|
return True
|
|
def any (results, predicate):
|
for result in results:
|
if predicate(result):
|
return True
|
return False
|
|
class FilterRule:
|
def __init__ (self, name, description, filters):
|
self.name = name
|
self.description = description
|
self.filters = filters
|
|
class TestCaseResult:
|
def __init__ (self, name, results):
|
self.name = name
|
self.results = results
|
|
class Group:
|
def __init__ (self, name):
|
self.name = name
|
self.cases = []
|
|
def readCaseList (filename):
|
f = open(filename, 'rb')
|
cases = []
|
for line in f:
|
if line[:6] == "TEST: ":
|
case = line[6:].strip()
|
if len(case) > 0:
|
cases.append(case)
|
return cases
|
|
def toResultList (caselist):
|
results = []
|
for case in caselist:
|
results.append(TestCaseResult(case, []))
|
return results
|
|
def addResultsToCaseList (caselist, results):
|
resultMap = {}
|
caseListRes = toResultList(caselist)
|
|
for res in caseListRes:
|
resultMap[res.name] = res
|
|
for result in results:
|
if result.name in resultMap:
|
resultMap[result.name].results += result.results
|
|
return caseListRes
|
|
def readTestResults (filename):
|
f = open(filename, 'rb')
|
csvData = f.read()
|
csvLines = csvData.splitlines()
|
results = []
|
|
f.close()
|
|
for line in csvLines[1:]:
|
args = line.split(',')
|
if len(args) == 1:
|
continue # Ignore
|
|
results.append(TestCaseResult(args[0], args[1:]))
|
|
if len(results) == 0:
|
raise Exception("Empty result list")
|
|
# Sanity check for results
|
numResultItems = len(results[0].results)
|
seenResults = set()
|
for result in results:
|
if result.name in seenResults:
|
raise Exception("Duplicate result row for test case '%s'" % result.name)
|
if len(result.results) != numResultItems:
|
raise Exception("Found %d results for test case '%s', expected %d" % (len(result.results), result.name, numResultItems))
|
seenResults.add(result.name)
|
|
return results
|
|
def readGroupList (filename):
|
f = open(filename, 'rb')
|
groups = []
|
for line in f:
|
group = line.strip()
|
if group != "":
|
groups.append(group)
|
return groups
|
|
def createGroups (results, groupNames):
|
groups = []
|
matched = set()
|
|
for groupName in groupNames:
|
group = Group(groupName)
|
groups.append(group)
|
|
prefix = groupName + "."
|
prefixLen = len(prefix)
|
for case in results:
|
if case.name[:prefixLen] == prefix:
|
if case in matched:
|
die("Case '%s' matched by multiple groups (when processing '%s')" % (case.name, group.name))
|
group.cases.append(case)
|
matched.add(case)
|
|
return groups
|
|
def createLeafGroups (results):
|
groups = []
|
groupMap = {}
|
|
for case in results:
|
parts = case.name.split('.')
|
groupName = string.join(parts[:-1], ".")
|
|
if not groupName in groupMap:
|
group = Group(groupName)
|
groups.append(group)
|
groupMap[groupName] = group
|
else:
|
group = groupMap[groupName]
|
|
group.cases.append(case)
|
|
return groups
|
|
def filterList (results, condition):
|
filtered = []
|
for case in results:
|
if condition(case.results):
|
filtered.append(case)
|
return filtered
|
|
def getFilter (list, name):
|
for filter in list:
|
if filter.name == name:
|
return filter
|
return None
|
|
def getNumCasesInGroups (groups):
|
numCases = 0
|
for group in groups:
|
numCases += len(group.cases)
|
return numCases
|
|
def getCasesInSet (results, caseSet):
|
filtered = []
|
for case in results:
|
if case in caseSet:
|
filtered.append(case)
|
return filtered
|
|
def selectCasesInGroups (results, groups):
|
casesInGroups = set()
|
for group in groups:
|
for case in group.cases:
|
casesInGroups.add(case)
|
return getCasesInSet(results, casesInGroups)
|
|
def selectRandomSubset (results, groups, limit, seed):
|
selectedCases = set()
|
numSelect = min(limit, getNumCasesInGroups(groups))
|
|
random.seed(seed)
|
random.shuffle(groups)
|
|
groupNdx = 0
|
while len(selectedCases) < numSelect:
|
group = groups[groupNdx]
|
if len(group.cases) == 0:
|
del groups[groupNdx]
|
if groupNdx == len(groups):
|
groupNdx -= 1
|
continue # Try next
|
|
selected = random.choice(group.cases)
|
selectedCases.add(selected)
|
group.cases.remove(selected)
|
|
groupNdx = (groupNdx + 1) % len(groups)
|
|
return getCasesInSet(results, selectedCases)
|
|
def die (msg):
|
print msg
|
sys.exit(-1)
|
|
# Named filter lists
|
FILTER_RULES = [
|
FilterRule("all", "No filtering", []),
|
FilterRule("all-pass", "All results must be 'Pass'", [lambda l: all(l, lambda r: r == 'Pass')]),
|
FilterRule("any-pass", "Any of results is 'Pass'", [lambda l: any(l, lambda r: r == 'Pass')]),
|
FilterRule("any-fail", "Any of results is not 'Pass' or 'NotSupported'", [lambda l: not all(l, lambda r: r == 'Pass' or r == 'NotSupported')]),
|
FilterRule("prev-failing", "Any except last result is failure", [lambda l: l[-1] == 'Pass' and not all(l[:-1], lambda r: r == 'Pass')]),
|
FilterRule("prev-passing", "Any except last result is 'Pass'", [lambda l: l[-1] != 'Pass' and any(l[:-1], lambda r: r == 'Pass')])
|
]
|
|
if __name__ == "__main__":
|
parser = OptionParser(usage = "usage: %prog [options] [caselist] [result csv file]")
|
parser.add_option("-f", "--filter", dest="filter", default="all", help="filter rule name")
|
parser.add_option("-l", "--list", action="store_true", dest="list", default=False, help="list available rules")
|
parser.add_option("-n", "--num", dest="limit", default=0, help="limit number of cases")
|
parser.add_option("-s", "--seed", dest="seed", default=0, help="use selected seed for random selection")
|
parser.add_option("-g", "--groups", dest="groups_file", default=None, help="select cases based on group list file")
|
|
(options, args) = parser.parse_args()
|
|
if options.list:
|
print "Available filter rules:"
|
for filter in FILTER_RULES:
|
print " %s: %s" % (filter.name, filter.description)
|
sys.exit(0)
|
|
if len(args) == 0:
|
die("No input files specified")
|
elif len(args) > 2:
|
die("Too many arguments")
|
|
# Fetch filter
|
filter = getFilter(FILTER_RULES, options.filter)
|
if filter == None:
|
die("Unknown filter '%s'" % options.filter)
|
|
# Read case list
|
caselist = readCaseList(args[0])
|
if len(args) > 1:
|
results = readTestResults(args[1])
|
results = addResultsToCaseList(caselist, results)
|
else:
|
results = toResultList(caselist)
|
|
# Execute filters for results
|
for rule in filter.filters:
|
results = filterList(results, rule)
|
|
if options.limit != 0:
|
if options.groups_file != None:
|
groups = createGroups(results, readGroupList(options.groups_file))
|
else:
|
groups = createLeafGroups(results)
|
results = selectRandomSubset(results, groups, int(options.limit), int(options.seed))
|
elif options.groups_file != None:
|
groups = createGroups(results, readGroupList(options.groups_file))
|
results = selectCasesInGroups(results, groups)
|
|
# Print test set
|
for result in results:
|
print result.name
|
