#!/usr/bin/python
|
"""
|
This script will scan an autotest server results directory for job result
|
directories that have completed and that have not yet been published on
|
a remote dashboard server matching given filtering options and for those it
|
finds it will rsync them to the tko server and mark them as published (it uses
|
a <jobdir>/.tko_published flag file to determine if a jobdir results directory
|
has been published yet).
|
"""
|
|
import sys, os, re, optparse
|
|
import common
|
from autotest_lib.client.common_lib import utils
|
from autotest_lib.server import frontend
|
|
options = optparse.Values()
|
|
USAGE="""tko-publish [options] <resultsdir> <rsync-destination-path>
|
|
Where:
|
<resultsdir> A path to the directory having the job results
|
directories to publish.
|
|
<rsync-destination-path> A valid rsync destination path where to upload the
|
job result directories.
|
Example: user@machine.org:/home/autotest/results"""
|
PUBLISH_FLAGFILE = '.tko_published'
|
RSYNC_COMMAND = 'rsync -aqz "%s" "%s"'
|
|
|
def get_job_dirs(path):
|
regex = re.compile('[1-9][0-9]*-')
|
jobdirs = []
|
|
for dir in os.listdir(path):
|
# skip directories not matching the job result dir pattern
|
if not regex.match(dir):
|
continue
|
|
dir = os.path.join(options.resultsdir, dir)
|
if (os.path.isdir(dir)
|
and not os.path.exists(os.path.join(dir, PUBLISH_FLAGFILE))):
|
jobdirs.append(dir)
|
|
return jobdirs
|
|
|
def publish_job(jobdir):
|
cmd = RSYNC_COMMAND % (jobdir, options.dest)
|
utils.system(cmd)
|
|
# mark the jobdir as published
|
fd = open(os.path.join(jobdir, PUBLISH_FLAGFILE), 'w')
|
fd.close()
|
print 'Published', jobdir
|
|
|
def main():
|
jobdirs = get_job_dirs(options.resultsdir)
|
|
afe = frontend.AFE()
|
# the way AFE API is right now is to give a whole list of jobs and can't
|
# get specific jobs so minimize the queries caching the result
|
finished_jobs = afe.get_jobs(finished=True)
|
|
if options.jobname_pattern:
|
jobname_pattern = re.compile(options.jobname_pattern)
|
else:
|
jobname_pattern = None
|
|
# for each unpublished possible jobdir find it in the database and see
|
# if it is completed
|
for jobdir in jobdirs:
|
job_id = int(os.path.basename(jobdir).split('-')[0])
|
job = [job for job in finished_jobs if job.id == job_id]
|
|
if len(job) != 1:
|
continue
|
|
if jobname_pattern:
|
# does it match the jobname pattern?
|
if not jobname_pattern.match(job[0].name):
|
continue
|
|
# does it match the wanted job owner
|
if options.job_owner and options.job_owner != job[0].owner:
|
continue
|
|
publish_job(jobdir)
|
|
|
if __name__ == '__main__':
|
parser = optparse.OptionParser(usage=USAGE)
|
parser.add_option('--jobname-pattern', dest='jobname_pattern',
|
help='Regexp pattern to match against job names, by '
|
"default there won't be any matching done",
|
default=None)
|
parser.add_option('--job-owner', dest='job_owner', default=None,
|
help='Job owner username to match against for the '
|
'published jobs, by default no matching is done.')
|
options, args = parser.parse_args()
|
|
if len(args) < 2:
|
print USAGE
|
sys.exit(-1)
|
|
options.resultsdir = args[0]
|
options.dest = args[1]
|
main()
|
