#!/usr/bin/python -u
|
|
"""
|
Utility to upload or remove the packages from the packages repository.
|
"""
|
|
import logging, optparse, os, shutil, sys, tempfile
|
import common
|
from autotest_lib.client.common_lib import utils as client_utils
|
from autotest_lib.client.common_lib import global_config, error
|
from autotest_lib.client.common_lib import packages
|
from autotest_lib.server import utils as server_utils
|
|
c = global_config.global_config
|
logging.basicConfig(level=logging.DEBUG)
|
|
ACTION_REMOVE = 'remove'
|
ACTION_UPLOAD = 'upload'
|
ACTION_TAR_ONLY = 'tar_only'
|
|
def get_exclude_string(client_dir):
|
'''
|
Get the exclude string for the tar command to exclude specific
|
subdirectories inside client_dir.
|
For profilers we need to exclude everything except the __init__.py
|
file so that the profilers can be imported.
|
'''
|
exclude_string = ('--exclude="deps/*" --exclude="tests/*" '
|
'--exclude="site_tests/*" --exclude="**.pyc"')
|
|
# Get the profilers directory
|
prof_dir = os.path.join(client_dir, 'profilers')
|
|
# Include the __init__.py file for the profilers and exclude all its
|
# subdirectories
|
for f in os.listdir(prof_dir):
|
if os.path.isdir(os.path.join(prof_dir, f)):
|
exclude_string += ' --exclude="profilers/%s"' % f
|
|
# The '.' here is needed to zip the files in the current
|
# directory. We use '-C' for tar to change to the required
|
# directory i.e. src_dir and then zip up the files in that
|
# directory(which is '.') excluding the ones in the exclude_dirs
|
exclude_string += " ."
|
|
# TODO(milleral): This is sad and ugly. http://crbug.com/258161
|
# Surprisingly, |exclude_string| actually means argument list, and
|
# we'd like to package up the current global_config.ini also, so let's
|
# just tack it on here.
|
# Also note that this only works because tar prevents us from un-tarring
|
# files into parent directories.
|
exclude_string += " ../global_config.ini"
|
|
return exclude_string
|
|
|
def parse_args():
|
parser = optparse.OptionParser()
|
parser.add_option("-d", "--dependency", help="package the dependency"
|
" from client/deps directory and upload to the repo",
|
dest="dep")
|
parser.add_option("-p", "--profiler", help="package the profiler "
|
"from client/profilers directory and upload to the repo",
|
dest="prof")
|
parser.add_option("-t", "--test", help="package the test from client/tests"
|
" or client/site_tests and upload to the repo.",
|
dest="test")
|
parser.add_option("-c", "--client", help="package the client "
|
"directory alone without the tests, deps and profilers",
|
dest="client", action="store_true", default=False)
|
parser.add_option("-f", "--file", help="simply uploads the specified"
|
"file on to the repo", dest="file")
|
parser.add_option("-r", "--repository", help="the URL of the packages"
|
"repository location to upload the packages to.",
|
dest="repo", default=None)
|
parser.add_option("-o", "--output_dir", help="the output directory"
|
"to place tarballs and md5sum files in.",
|
dest="output_dir", default=None)
|
parser.add_option("-a", "--action", help="the action to perform",
|
dest="action", choices=(ACTION_UPLOAD, ACTION_REMOVE,
|
ACTION_TAR_ONLY), default=None)
|
parser.add_option("--all", help="Upload all the files locally "
|
"to all the repos specified in global_config.ini. "
|
"(includes the client, tests, deps and profilers)",
|
dest="all", action="store_true", default=False)
|
|
options, args = parser.parse_args()
|
return options, args
|
|
def get_build_dir(name, dest_dir, pkg_type):
|
"""Method to generate the build directory where the tarball and checksum
|
is stored. The following package types are handled: test, dep, profiler.
|
Package type 'client' is not handled.
|
"""
|
if pkg_type == 'client':
|
# NOTE: The "tar_only" action for pkg_type "client" has no use
|
# case yet. No known invocations of packager.py with
|
# --action=tar_only send in clients in the command line. Please
|
# confirm the behaviour is expected before this type is enabled for
|
# "tar_only" actions.
|
print ('Tar action not supported for pkg_type= %s, name = %s' %
|
pkg_type, name)
|
return None
|
# For all packages, the work-dir should have 'client' appended to it.
|
base_build_dir = os.path.join(dest_dir, 'client')
|
if pkg_type == 'test':
|
build_dir = os.path.join(get_test_dir(name, base_build_dir), name)
|
else:
|
# For profiler and dep, we append 's', and then append the name.
|
# TODO(pmalani): Make this less fiddly?
|
build_dir = os.path.join(base_build_dir, pkg_type + 's', name)
|
return build_dir
|
|
def process_packages(pkgmgr, pkg_type, pkg_names, src_dir,
|
action, dest_dir=None):
|
"""Method to upload or remove package depending on the flag passed to it.
|
|
If tar_only is set to True, this routine is solely used to generate a
|
tarball and compute the md5sum from that tarball.
|
If the tar_only flag is True, then the remove flag is ignored.
|
"""
|
exclude_string = ' .'
|
names = [p.strip() for p in pkg_names.split(',')]
|
for name in names:
|
print "process_packages: Processing %s ... " % name
|
if pkg_type == 'client':
|
pkg_dir = src_dir
|
exclude_string = get_exclude_string(pkg_dir)
|
elif pkg_type == 'test':
|
# if the package is a test then look whether it is in client/tests
|
# or client/site_tests
|
pkg_dir = os.path.join(get_test_dir(name, src_dir), name)
|
else:
|
# for the profilers and deps
|
pkg_dir = os.path.join(src_dir, name)
|
|
pkg_name = pkgmgr.get_tarball_name(name, pkg_type)
|
|
exclude_string_tar = ((
|
' --exclude="**%s" --exclude="**%s.checksum" ' %
|
(pkg_name, pkg_name)) + exclude_string)
|
if action == ACTION_TAR_ONLY:
|
# We don't want any pre-existing tarballs and checksums to
|
# be repackaged, so we should purge these.
|
build_dir = get_build_dir(name, dest_dir, pkg_type)
|
try:
|
packages.check_diskspace(build_dir)
|
except error.RepoDiskFullError as e:
|
msg = ("Work_dir directory for packages %s does not have "
|
"enough space available: %s" % (build_dir, e))
|
raise error.RepoDiskFullError(msg)
|
tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
|
build_dir, exclude_string_tar)
|
|
# Create the md5 hash too.
|
md5sum = pkgmgr.compute_checksum(tarball_path)
|
md5sum_filepath = os.path.join(build_dir, pkg_name + '.checksum')
|
with open(md5sum_filepath, "w") as f:
|
f.write(md5sum)
|
|
elif action == ACTION_UPLOAD:
|
# Tar the source and upload
|
temp_dir = tempfile.mkdtemp()
|
try:
|
try:
|
packages.check_diskspace(temp_dir)
|
except error.RepoDiskFullError, e:
|
msg = ("Temporary directory for packages %s does not have "
|
"enough space available: %s" % (temp_dir, e))
|
raise error.RepoDiskFullError(msg)
|
|
# Check if tarball already exists. If it does, then don't
|
# create a tarball again.
|
tarball_path = os.path.join(pkg_dir, pkg_name);
|
if os.path.exists(tarball_path):
|
print("process_packages: Tarball %s already exists" %
|
tarball_path)
|
else:
|
tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
|
temp_dir,
|
exclude_string_tar)
|
# Compare the checksum with what packages.checksum has. If they
|
# match then we don't need to perform the upload.
|
if not pkgmgr.compare_checksum(tarball_path):
|
pkgmgr.upload_pkg(tarball_path, update_checksum=True)
|
else:
|
logging.warning('Checksum not changed for %s, not copied '
|
'in packages/ directory.', tarball_path)
|
finally:
|
# remove the temporary directory
|
shutil.rmtree(temp_dir)
|
elif action == ACTION_REMOVE:
|
pkgmgr.remove_pkg(pkg_name, remove_checksum=True)
|
print "Done."
|
|
|
def tar_packages(pkgmgr, pkg_type, pkg_names, src_dir, temp_dir):
|
"""Tar all packages up and return a list of each tar created"""
|
tarballs = []
|
exclude_string = ' .'
|
names = [p.strip() for p in pkg_names.split(',')]
|
for name in names:
|
print "tar_packages: Processing %s ... " % name
|
if pkg_type == 'client':
|
pkg_dir = src_dir
|
exclude_string = get_exclude_string(pkg_dir)
|
elif pkg_type == 'test':
|
# if the package is a test then look whether it is in client/tests
|
# or client/site_tests
|
pkg_dir = os.path.join(get_test_dir(name, src_dir), name)
|
else:
|
# for the profilers and deps
|
pkg_dir = os.path.join(src_dir, name)
|
|
pkg_name = pkgmgr.get_tarball_name(name, pkg_type)
|
|
# We don't want any pre-existing tarballs and checksums to
|
# be repackaged, so we should purge these.
|
exclude_string_tar = ((
|
' --exclude="**%s" --exclude="**%s.checksum" ' %
|
(pkg_name, pkg_name)) + exclude_string)
|
# Check if tarball already exists. If it does, don't duplicate
|
# the effort.
|
tarball_path = os.path.join(pkg_dir, pkg_name);
|
if os.path.exists(tarball_path):
|
print("tar_packages: Tarball %s already exists" % tarball_path);
|
else:
|
tarball_path = pkgmgr.tar_package(pkg_name, pkg_dir,
|
temp_dir, exclude_string_tar)
|
tarballs.append(tarball_path)
|
return tarballs
|
|
|
def process_all_packages(pkgmgr, client_dir, action):
|
"""Process a full upload of packages as a directory upload."""
|
dep_dir = os.path.join(client_dir, "deps")
|
prof_dir = os.path.join(client_dir, "profilers")
|
# Directory where all are kept
|
temp_dir = tempfile.mkdtemp()
|
try:
|
packages.check_diskspace(temp_dir)
|
except error.RepoDiskFullError, e:
|
print ("Temp destination for packages is full %s, aborting upload: %s"
|
% (temp_dir, e))
|
os.rmdir(temp_dir)
|
sys.exit(1)
|
|
# process tests
|
tests_list = get_subdir_list('tests', client_dir)
|
tests = ','.join(tests_list)
|
|
# process site_tests
|
site_tests_list = get_subdir_list('site_tests', client_dir)
|
site_tests = ','.join(site_tests_list)
|
|
# process deps
|
deps_list = get_subdir_list('deps', client_dir)
|
deps = ','.join(deps_list)
|
|
# process profilers
|
profilers_list = get_subdir_list('profilers', client_dir)
|
profilers = ','.join(profilers_list)
|
|
# Update md5sum
|
if action == ACTION_UPLOAD:
|
all_packages = []
|
all_packages.extend(tar_packages(pkgmgr, 'profiler', profilers,
|
prof_dir, temp_dir))
|
all_packages.extend(tar_packages(pkgmgr, 'dep', deps, dep_dir,
|
temp_dir))
|
all_packages.extend(tar_packages(pkgmgr, 'test', site_tests,
|
client_dir, temp_dir))
|
all_packages.extend(tar_packages(pkgmgr, 'test', tests, client_dir,
|
temp_dir))
|
all_packages.extend(tar_packages(pkgmgr, 'client', 'autotest',
|
client_dir, temp_dir))
|
for package in all_packages:
|
pkgmgr.upload_pkg(package, update_checksum=True)
|
client_utils.run('rm -rf ' + temp_dir)
|
elif action == ACTION_REMOVE:
|
process_packages(pkgmgr, 'test', tests, client_dir, action=action)
|
process_packages(pkgmgr, 'test', site_tests, client_dir, action=action)
|
process_packages(pkgmgr, 'client', 'autotest', client_dir,
|
action=action)
|
process_packages(pkgmgr, 'dep', deps, dep_dir, action=action)
|
process_packages(pkgmgr, 'profiler', profilers, prof_dir,
|
action=action)
|
|
|
# Get the list of sub directories present in a directory
|
def get_subdir_list(name, client_dir):
|
dir_name = os.path.join(client_dir, name)
|
return [f for f in
|
os.listdir(dir_name)
|
if os.path.isdir(os.path.join(dir_name, f)) ]
|
|
|
# Look whether the test is present in client/tests and client/site_tests dirs
|
def get_test_dir(name, client_dir):
|
names_test = os.listdir(os.path.join(client_dir, 'tests'))
|
names_site_test = os.listdir(os.path.join(client_dir, 'site_tests'))
|
if name in names_test:
|
src_dir = os.path.join(client_dir, 'tests')
|
elif name in names_site_test:
|
src_dir = os.path.join(client_dir, 'site_tests')
|
else:
|
print "Test %s not found" % name
|
sys.exit(0)
|
return src_dir
|
|
|
def main():
|
# get options and args
|
options, args = parse_args()
|
|
server_dir = server_utils.get_server_dir()
|
autotest_dir = os.path.abspath(os.path.join(server_dir, '..'))
|
|
# extract the pkg locations from global config
|
repo_urls = c.get_config_value('PACKAGES', 'fetch_location',
|
type=list, default=[])
|
upload_paths = c.get_config_value('PACKAGES', 'upload_location',
|
type=list, default=[])
|
|
if options.repo:
|
upload_paths.append(options.repo)
|
|
# Having no upload paths basically means you're not using packaging.
|
if not upload_paths:
|
print("No upload locations found. Please set upload_location under"
|
" PACKAGES in the global_config.ini or provide a location using"
|
" the --repository option.")
|
return
|
|
client_dir = os.path.join(autotest_dir, "client")
|
|
# Bail out if the client directory does not exist
|
if not os.path.exists(client_dir):
|
sys.exit(0)
|
|
dep_dir = os.path.join(client_dir, "deps")
|
prof_dir = os.path.join(client_dir, "profilers")
|
|
# Due to the delayed uprev-ing of certain ebuilds, we need to support
|
# both the legacy command line and the new one.
|
# So, if the new "action" option isn't specified, try looking for the
|
# old style remove/upload argument
|
if options.action is None:
|
if len(args) == 0 or args[0] not in ['upload', 'remove']:
|
print("Either 'upload' or 'remove' needs to be specified "
|
"for the package")
|
sys.exit(0)
|
cur_action = args[0]
|
else:
|
cur_action = options.action
|
|
if cur_action == ACTION_TAR_ONLY and options.output_dir is None:
|
print("An output dir has to be specified")
|
sys.exit(0)
|
|
pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls,
|
upload_paths=upload_paths,
|
run_function_dargs={'timeout':600})
|
|
if options.all:
|
process_all_packages(pkgmgr, client_dir, action=cur_action)
|
|
if options.client:
|
process_packages(pkgmgr, 'client', 'autotest', client_dir,
|
action=cur_action)
|
|
if options.dep:
|
process_packages(pkgmgr, 'dep', options.dep, dep_dir,
|
action=cur_action, dest_dir=options.output_dir)
|
|
if options.test:
|
process_packages(pkgmgr, 'test', options.test, client_dir,
|
action=cur_action, dest_dir=options.output_dir)
|
|
if options.prof:
|
process_packages(pkgmgr, 'profiler', options.prof, prof_dir,
|
action=cur_action, dest_dir=options.output_dir)
|
|
if options.file:
|
if cur_action == ACTION_REMOVE:
|
pkgmgr.remove_pkg(options.file, remove_checksum=True)
|
elif cur_action == ACTION_UPLOAD:
|
pkgmgr.upload_pkg(options.file, update_checksum=True)
|
|
|
if __name__ == "__main__":
|
main()
|
