# -*- coding: utf-8 -*-
|
|
#-------------------------------------------------------------------------
|
# drawElements Quality Program utilities
|
# --------------------------------------
|
#
|
# Copyright 2015 The Android Open Source Project
|
#
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
# you may not use this file except in compliance with the License.
|
# You may obtain a copy of the License at
|
#
|
# http://www.apache.org/licenses/LICENSE-2.0
|
#
|
# Unless required by applicable law or agreed to in writing, software
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# See the License for the specific language governing permissions and
|
# limitations under the License.
|
#
|
#-------------------------------------------------------------------------
|
|
import os
|
import re
|
import sys
|
import copy
|
import zlib
|
import time
|
import shlex
|
import shutil
|
import fnmatch
|
import tarfile
|
import argparse
|
import platform
|
import datetime
|
import tempfile
|
import posixpath
|
import subprocess
|
|
from build.common import *
|
from build.config import *
|
from build.build import *
|
|
pythonExecutable = sys.executable or "python"
|
|
def die (msg):
|
print msg
|
sys.exit(-1)
|
|
def removeLeadingPath (path, basePath):
|
# Both inputs must be normalized already
|
assert os.path.normpath(path) == path
|
assert os.path.normpath(basePath) == basePath
|
return path[len(basePath) + 1:]
|
|
def findFile (candidates):
|
for file in candidates:
|
if os.path.exists(file):
|
return file
|
return None
|
|
def getFileList (basePath):
|
allFiles = []
|
basePath = os.path.normpath(basePath)
|
for root, dirs, files in os.walk(basePath):
|
for file in files:
|
relPath = removeLeadingPath(os.path.normpath(os.path.join(root, file)), basePath)
|
allFiles.append(relPath)
|
return allFiles
|
|
def toDatetime (dateTuple):
|
Y, M, D = dateTuple
|
return datetime.datetime(Y, M, D)
|
|
class PackageBuildInfo:
|
def __init__ (self, releaseConfig, srcBasePath, dstBasePath, tmpBasePath):
|
self.releaseConfig = releaseConfig
|
self.srcBasePath = srcBasePath
|
self.dstBasePath = dstBasePath
|
self.tmpBasePath = tmpBasePath
|
|
def getReleaseConfig (self):
|
return self.releaseConfig
|
|
def getReleaseVersion (self):
|
return self.releaseConfig.getVersion()
|
|
def getReleaseId (self):
|
# Release id is crc32(releaseConfig + release)
|
return zlib.crc32(self.releaseConfig.getName() + self.releaseConfig.getVersion()) & 0xffffffff
|
|
def getSrcBasePath (self):
|
return self.srcBasePath
|
|
def getTmpBasePath (self):
|
return self.tmpBasePath
|
|
class DstFile (object):
|
def __init__ (self, dstFile):
|
self.dstFile = dstFile
|
|
def makeDir (self):
|
dirName = os.path.dirname(self.dstFile)
|
if not os.path.exists(dirName):
|
os.makedirs(dirName)
|
|
def make (self, packageBuildInfo):
|
assert False # Should not be called
|
|
class CopyFile (DstFile):
|
def __init__ (self, srcFile, dstFile):
|
super(CopyFile, self).__init__(dstFile)
|
self.srcFile = srcFile
|
|
def make (self, packageBuildInfo):
|
self.makeDir()
|
if os.path.exists(self.dstFile):
|
die("%s already exists" % self.dstFile)
|
shutil.copyfile(self.srcFile, self.dstFile)
|
|
class GenReleaseInfoFileTarget (DstFile):
|
def __init__ (self, dstFile):
|
super(GenReleaseInfoFileTarget, self).__init__(dstFile)
|
|
def make (self, packageBuildInfo):
|
self.makeDir()
|
|
scriptPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, "framework", "qphelper", "gen_release_info.py"))
|
execute([
|
pythonExecutable,
|
"-B", # no .py[co]
|
scriptPath,
|
"--name=%s" % packageBuildInfo.getReleaseVersion(),
|
"--id=0x%08x" % packageBuildInfo.getReleaseId(),
|
"--out=%s" % self.dstFile
|
])
|
|
class GenCMake (DstFile):
|
def __init__ (self, srcFile, dstFile, replaceVars):
|
super(GenCMake, self).__init__(dstFile)
|
self.srcFile = srcFile
|
self.replaceVars = replaceVars
|
|
def make (self, packageBuildInfo):
|
self.makeDir()
|
print " GenCMake: %s" % removeLeadingPath(self.dstFile, packageBuildInfo.dstBasePath)
|
src = readFile(self.srcFile)
|
for var, value in self.replaceVars:
|
src = re.sub('set\(%s\s+"[^"]*"' % re.escape(var),
|
'set(%s "%s"' % (var, value), src)
|
writeFile(self.dstFile, src)
|
|
def createFileTargets (srcBasePath, dstBasePath, files, filters):
|
usedFiles = set() # Files that are already included by other filters
|
targets = []
|
|
for isMatch, createFileObj in filters:
|
# Build list of files that match filter
|
matchingFiles = []
|
for file in files:
|
if not file in usedFiles and isMatch(file):
|
matchingFiles.append(file)
|
|
# Build file objects, add to used set
|
for file in matchingFiles:
|
usedFiles.add(file)
|
targets.append(createFileObj(os.path.join(srcBasePath, file), os.path.join(dstBasePath, file)))
|
|
return targets
|
|
# Generates multiple file targets based on filters
|
class FileTargetGroup:
|
def __init__ (self, srcBasePath, dstBasePath, filters, srcBasePathFunc=PackageBuildInfo.getSrcBasePath):
|
self.srcBasePath = srcBasePath
|
self.dstBasePath = dstBasePath
|
self.filters = filters
|
self.getSrcBasePath = srcBasePathFunc
|
|
def make (self, packageBuildInfo):
|
fullSrcPath = os.path.normpath(os.path.join(self.getSrcBasePath(packageBuildInfo), self.srcBasePath))
|
fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstBasePath))
|
|
allFiles = getFileList(fullSrcPath)
|
targets = createFileTargets(fullSrcPath, fullDstPath, allFiles, self.filters)
|
|
# Make all file targets
|
for file in targets:
|
file.make(packageBuildInfo)
|
|
# Single file target
|
class SingleFileTarget:
|
def __init__ (self, srcFile, dstFile, makeTarget):
|
self.srcFile = srcFile
|
self.dstFile = dstFile
|
self.makeTarget = makeTarget
|
|
def make (self, packageBuildInfo):
|
fullSrcPath = os.path.normpath(os.path.join(packageBuildInfo.srcBasePath, self.srcFile))
|
fullDstPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
|
|
target = self.makeTarget(fullSrcPath, fullDstPath)
|
target.make(packageBuildInfo)
|
|
class BuildTarget:
|
def __init__ (self, baseConfig, generator, targets = None):
|
self.baseConfig = baseConfig
|
self.generator = generator
|
self.targets = targets
|
|
def make (self, packageBuildInfo):
|
print " Building %s" % self.baseConfig.getBuildDir()
|
|
# Create config with full build dir path
|
config = BuildConfig(os.path.join(packageBuildInfo.getTmpBasePath(), self.baseConfig.getBuildDir()),
|
self.baseConfig.getBuildType(),
|
self.baseConfig.getArgs(),
|
srcPath = os.path.join(packageBuildInfo.dstBasePath, "src"))
|
|
assert not os.path.exists(config.getBuildDir())
|
build(config, self.generator, self.targets)
|
|
class BuildAndroidTarget:
|
def __init__ (self, dstFile):
|
self.dstFile = dstFile
|
|
def make (self, packageBuildInfo):
|
print " Building Android binary"
|
|
buildRoot = os.path.join(packageBuildInfo.tmpBasePath, "android-build")
|
|
assert not os.path.exists(buildRoot)
|
os.makedirs(buildRoot)
|
|
# Execute build script
|
scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "android", "scripts", "build.py"))
|
execute([
|
pythonExecutable,
|
"-B", # no .py[co]
|
scriptPath,
|
"--build-root=%s" % buildRoot,
|
])
|
|
srcFile = os.path.normpath(os.path.join(buildRoot, "package", "bin", "dEQP-debug.apk"))
|
dstFile = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, self.dstFile))
|
|
CopyFile(srcFile, dstFile).make(packageBuildInfo)
|
|
class FetchExternalSourcesTarget:
|
def __init__ (self):
|
pass
|
|
def make (self, packageBuildInfo):
|
scriptPath = os.path.normpath(os.path.join(packageBuildInfo.dstBasePath, "src", "external", "fetch_sources.py"))
|
execute([
|
pythonExecutable,
|
"-B", # no .py[co]
|
scriptPath,
|
])
|
|
class RemoveSourcesTarget:
|
def __init__ (self):
|
pass
|
|
def make (self, packageBuildInfo):
|
shutil.rmtree(os.path.join(packageBuildInfo.dstBasePath, "src"), ignore_errors=False)
|
|
class Module:
|
def __init__ (self, name, targets):
|
self.name = name
|
self.targets = targets
|
|
def make (self, packageBuildInfo):
|
for target in self.targets:
|
target.make(packageBuildInfo)
|
|
class ReleaseConfig:
|
def __init__ (self, name, version, modules, sources = True):
|
self.name = name
|
self.version = version
|
self.modules = modules
|
self.sources = sources
|
|
def getName (self):
|
return self.name
|
|
def getVersion (self):
|
return self.version
|
|
def getModules (self):
|
return self.modules
|
|
def packageWithSources (self):
|
return self.sources
|
|
def matchIncludeExclude (includePatterns, excludePatterns, filename):
|
components = os.path.normpath(filename).split(os.sep)
|
for pattern in excludePatterns:
|
for component in components:
|
if fnmatch.fnmatch(component, pattern):
|
return False
|
|
for pattern in includePatterns:
|
for component in components:
|
if fnmatch.fnmatch(component, pattern):
|
return True
|
|
return False
|
|
def copyFileFilter (includePatterns, excludePatterns=[]):
|
return (lambda f: matchIncludeExclude(includePatterns, excludePatterns, f),
|
lambda s, d: CopyFile(s, d))
|
|
def makeFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
|
return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)])
|
|
def makeTmpFileCopyGroup (srcDir, dstDir, includePatterns, excludePatterns=[]):
|
return FileTargetGroup(srcDir, dstDir, [copyFileFilter(includePatterns, excludePatterns)], PackageBuildInfo.getTmpBasePath)
|
|
def makeFileCopy (srcFile, dstFile):
|
return SingleFileTarget(srcFile, dstFile, lambda s, d: CopyFile(s, d))
|
|
def getReleaseFileName (configName, releaseName):
|
today = datetime.date.today()
|
return "dEQP-%s-%04d-%02d-%02d-%s" % (releaseName, today.year, today.month, today.day, configName)
|
|
def getTempDir ():
|
dirName = os.path.join(tempfile.gettempdir(), "dEQP-Releases")
|
if not os.path.exists(dirName):
|
os.makedirs(dirName)
|
return dirName
|
|
def makeRelease (releaseConfig):
|
releaseName = getReleaseFileName(releaseConfig.getName(), releaseConfig.getVersion())
|
tmpPath = getTempDir()
|
srcBasePath = DEQP_DIR
|
dstBasePath = os.path.join(tmpPath, releaseName)
|
tmpBasePath = os.path.join(tmpPath, releaseName + "-tmp")
|
packageBuildInfo = PackageBuildInfo(releaseConfig, srcBasePath, dstBasePath, tmpBasePath)
|
dstArchiveName = releaseName + ".tar.bz2"
|
|
print "Creating release %s to %s" % (releaseName, tmpPath)
|
|
# Remove old temporary dirs
|
for path in [dstBasePath, tmpBasePath]:
|
if os.path.exists(path):
|
shutil.rmtree(path, ignore_errors=False)
|
|
# Make all modules
|
for module in releaseConfig.getModules():
|
print " Processing module %s" % module.name
|
module.make(packageBuildInfo)
|
|
# Remove sources?
|
if not releaseConfig.packageWithSources():
|
shutil.rmtree(os.path.join(dstBasePath, "src"), ignore_errors=False)
|
|
# Create archive
|
print "Creating %s" % dstArchiveName
|
archive = tarfile.open(dstArchiveName, 'w:bz2')
|
archive.add(dstBasePath, arcname=releaseName)
|
archive.close()
|
|
# Remove tmp dirs
|
for path in [dstBasePath, tmpBasePath]:
|
if os.path.exists(path):
|
shutil.rmtree(path, ignore_errors=False)
|
|
print "Done!"
|
|
# Module declarations
|
|
SRC_FILE_PATTERNS = ["*.h", "*.hpp", "*.c", "*.cpp", "*.m", "*.mm", "*.inl", "*.java", "*.aidl", "CMakeLists.txt", "LICENSE.txt", "*.cmake"]
|
TARGET_PATTERNS = ["*.cmake", "*.h", "*.lib", "*.dll", "*.so", "*.txt"]
|
|
BASE = Module("Base", [
|
makeFileCopy ("LICENSE", "src/LICENSE"),
|
makeFileCopy ("CMakeLists.txt", "src/CMakeLists.txt"),
|
makeFileCopyGroup ("targets", "src/targets", TARGET_PATTERNS),
|
makeFileCopyGroup ("execserver", "src/execserver", SRC_FILE_PATTERNS),
|
makeFileCopyGroup ("executor", "src/executor", SRC_FILE_PATTERNS),
|
makeFileCopy ("modules/CMakeLists.txt", "src/modules/CMakeLists.txt"),
|
makeFileCopyGroup ("external", "src/external", ["CMakeLists.txt", "*.py"]),
|
|
# Stylesheet for displaying test logs on browser
|
makeFileCopyGroup ("doc/testlog-stylesheet", "doc/testlog-stylesheet", ["*"]),
|
|
# Non-optional parts of framework
|
makeFileCopy ("framework/CMakeLists.txt", "src/framework/CMakeLists.txt"),
|
makeFileCopyGroup ("framework/delibs", "src/framework/delibs", SRC_FILE_PATTERNS),
|
makeFileCopyGroup ("framework/common", "src/framework/common", SRC_FILE_PATTERNS),
|
makeFileCopyGroup ("framework/qphelper", "src/framework/qphelper", SRC_FILE_PATTERNS),
|
makeFileCopyGroup ("framework/platform", "src/framework/platform", SRC_FILE_PATTERNS),
|
makeFileCopyGroup ("framework/opengl", "src/framework/opengl", SRC_FILE_PATTERNS, ["simplereference"]),
|
makeFileCopyGroup ("framework/egl", "src/framework/egl", SRC_FILE_PATTERNS),
|
|
# android sources
|
makeFileCopyGroup ("android/package/src", "src/android/package/src", SRC_FILE_PATTERNS),
|
makeFileCopy ("android/package/AndroidManifest.xml", "src/android/package/AndroidManifest.xml"),
|
makeFileCopyGroup ("android/package/res", "src/android/package/res", ["*.png", "*.xml"]),
|
makeFileCopyGroup ("android/scripts", "src/android/scripts", [
|
"common.py",
|
"build.py",
|
"resources.py",
|
"install.py",
|
"launch.py",
|
"debug.py"
|
]),
|
|
# Release info
|
GenReleaseInfoFileTarget("src/framework/qphelper/qpReleaseInfo.inl")
|
])
|
|
DOCUMENTATION = Module("Documentation", [
|
makeFileCopyGroup ("doc/pdf", "doc", ["*.pdf"]),
|
makeFileCopyGroup ("doc", "doc", ["porting_layer_changes_*.txt"]),
|
])
|
|
GLSHARED = Module("Shared GL Tests", [
|
# Optional framework components
|
makeFileCopyGroup ("framework/randomshaders", "src/framework/randomshaders", SRC_FILE_PATTERNS),
|
makeFileCopyGroup ("framework/opengl/simplereference", "src/framework/opengl/simplereference", SRC_FILE_PATTERNS),
|
makeFileCopyGroup ("framework/referencerenderer", "src/framework/referencerenderer", SRC_FILE_PATTERNS),
|
|
makeFileCopyGroup ("modules/glshared", "src/modules/glshared", SRC_FILE_PATTERNS),
|
])
|
|
GLES2 = Module("GLES2", [
|
makeFileCopyGroup ("modules/gles2", "src/modules/gles2", SRC_FILE_PATTERNS),
|
makeFileCopyGroup ("data/gles2", "src/data/gles2", ["*.*"]),
|
makeFileCopyGroup ("doc/testspecs/GLES2", "doc/testspecs/GLES2", ["*.txt"])
|
])
|
|
GLES3 = Module("GLES3", [
|
makeFileCopyGroup ("modules/gles3", "src/modules/gles3", SRC_FILE_PATTERNS),
|
makeFileCopyGroup ("data/gles3", "src/data/gles3", ["*.*"]),
|
makeFileCopyGroup ("doc/testspecs/GLES3", "doc/testspecs/GLES3", ["*.txt"])
|
])
|
|
GLES31 = Module("GLES31", [
|
makeFileCopyGroup ("modules/gles31", "src/modules/gles31", SRC_FILE_PATTERNS),
|
makeFileCopyGroup ("data/gles31", "src/data/gles31", ["*.*"]),
|
makeFileCopyGroup ("doc/testspecs/GLES31", "doc/testspecs/GLES31", ["*.txt"])
|
])
|
|
EGL = Module("EGL", [
|
makeFileCopyGroup ("modules/egl", "src/modules/egl", SRC_FILE_PATTERNS)
|
])
|
|
INTERNAL = Module("Internal", [
|
makeFileCopyGroup ("modules/internal", "src/modules/internal", SRC_FILE_PATTERNS),
|
makeFileCopyGroup ("data/internal", "src/data/internal", ["*.*"]),
|
])
|
|
EXTERNAL_SRCS = Module("External sources", [
|
FetchExternalSourcesTarget()
|
])
|
|
ANDROID_BINARIES = Module("Android Binaries", [
|
BuildAndroidTarget ("bin/android/dEQP.apk"),
|
makeFileCopyGroup ("targets/android", "bin/android", ["*.bat", "*.sh"]),
|
])
|
|
COMMON_BUILD_ARGS = ['-DPNG_SRC_PATH=%s' % os.path.realpath(os.path.join(DEQP_DIR, '..', 'libpng'))]
|
NULL_X32_CONFIG = BuildConfig('null-x32', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
|
NULL_X64_CONFIG = BuildConfig('null-x64', 'Release', ['-DDEQP_TARGET=null', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
|
GLX_X32_CONFIG = BuildConfig('glx-x32', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m32', '-DCMAKE_CXX_FLAGS=-m32'] + COMMON_BUILD_ARGS)
|
GLX_X64_CONFIG = BuildConfig('glx-x64', 'Release', ['-DDEQP_TARGET=x11_glx', '-DCMAKE_C_FLAGS=-m64', '-DCMAKE_CXX_FLAGS=-m64'] + COMMON_BUILD_ARGS)
|
|
EXCLUDE_BUILD_FILES = ["CMakeFiles", "*.a", "*.cmake"]
|
|
LINUX_X32_COMMON_BINARIES = Module("Linux x32 Common Binaries", [
|
BuildTarget (NULL_X32_CONFIG, ANY_UNIX_GENERATOR),
|
makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/execserver", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES),
|
makeTmpFileCopyGroup(NULL_X32_CONFIG.getBuildDir() + "/executor", "bin/linux32", ["*"], EXCLUDE_BUILD_FILES),
|
])
|
|
LINUX_X64_COMMON_BINARIES = Module("Linux x64 Common Binaries", [
|
BuildTarget (NULL_X64_CONFIG, ANY_UNIX_GENERATOR),
|
makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/execserver", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES),
|
makeTmpFileCopyGroup(NULL_X64_CONFIG.getBuildDir() + "/executor", "bin/linux64", ["*"], EXCLUDE_BUILD_FILES),
|
])
|
|
# Special module to remove src dir, for example after binary build
|
REMOVE_SOURCES = Module("Remove sources from package", [
|
RemoveSourcesTarget()
|
])
|
|
# Release configuration
|
|
ALL_MODULES = [
|
BASE,
|
DOCUMENTATION,
|
GLSHARED,
|
GLES2,
|
GLES3,
|
GLES31,
|
EGL,
|
INTERNAL,
|
EXTERNAL_SRCS,
|
]
|
|
ALL_BINARIES = [
|
LINUX_X64_COMMON_BINARIES,
|
ANDROID_BINARIES,
|
]
|
|
RELEASE_CONFIGS = {
|
"src": ALL_MODULES,
|
"src-bin": ALL_MODULES + ALL_BINARIES,
|
"bin": ALL_MODULES + ALL_BINARIES + [REMOVE_SOURCES],
|
}
|
|
def parseArgs ():
|
parser = argparse.ArgumentParser(description = "Build release package")
|
parser.add_argument("-c",
|
"--config",
|
dest="config",
|
choices=RELEASE_CONFIGS.keys(),
|
required=True,
|
help="Release configuration")
|
parser.add_argument("-n",
|
"--name",
|
dest="name",
|
required=True,
|
help="Package-specific name")
|
parser.add_argument("-v",
|
"--version",
|
dest="version",
|
required=True,
|
help="Version code")
|
return parser.parse_args()
|
|
if __name__ == "__main__":
|
args = parseArgs()
|
config = ReleaseConfig(args.name, args.version, RELEASE_CONFIGS[args.config])
|
makeRelease(config)
|
