#!/usr/bin/env python2
|
"""Generate summary report for ChromeOS toolchain waterfalls."""
|
|
from __future__ import print_function
|
|
import argparse
|
import datetime
|
import getpass
|
import json
|
import os
|
import re
|
import shutil
|
import sys
|
import time
|
|
from cros_utils import command_executer
|
|
# All the test suites whose data we might want for the reports.
|
TESTS = (('bvt-inline', 'HWTest [bvt-inline]'), ('bvt-cq', 'HWTest [bvt-cq]'),
|
('security', 'HWTest [security]'))
|
|
# The main waterfall builders, IN THE ORDER IN WHICH WE WANT THEM
|
# LISTED IN THE REPORT.
|
WATERFALL_BUILDERS = [
|
'amd64-llvm-next-toolchain',
|
'arm-llvm-next-toolchain',
|
'arm64-llvm-next-toolchain',
|
]
|
|
DATA_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-report-data/'
|
ARCHIVE_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-reports/'
|
DOWNLOAD_DIR = '/tmp/waterfall-logs'
|
MAX_SAVE_RECORDS = 7
|
BUILD_DATA_FILE = '%s/build-data.txt' % DATA_DIR
|
LLVM_ROTATING_BUILDER = 'llvm_next_toolchain'
|
ROTATING_BUILDERS = [LLVM_ROTATING_BUILDER]
|
|
# For int-to-string date conversion. Note, the index of the month in this
|
# list needs to correspond to the month's integer value. i.e. 'Sep' must
|
# be as MONTHS[9].
|
MONTHS = [
|
'', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct',
|
'Nov', 'Dec'
|
]
|
|
DAYS_PER_MONTH = {
|
1: 31,
|
2: 28,
|
3: 31,
|
4: 30,
|
5: 31,
|
6: 30,
|
7: 31,
|
8: 31,
|
9: 30,
|
10: 31,
|
11: 31,
|
12: 31
|
}
|
|
|
def format_date(int_date, use_int_month=False):
|
"""Convert an integer date to a string date. YYYYMMDD -> YYYY-MMM-DD"""
|
|
if int_date == 0:
|
return 'today'
|
|
tmp_date = int_date
|
day = tmp_date % 100
|
tmp_date = tmp_date / 100
|
month = tmp_date % 100
|
year = tmp_date / 100
|
|
if use_int_month:
|
date_str = '%d-%02d-%02d' % (year, month, day)
|
else:
|
month_str = MONTHS[month]
|
date_str = '%d-%s-%d' % (year, month_str, day)
|
return date_str
|
|
|
def EmailReport(report_file, report_type, date, email_to):
|
"""Emails the report to the approprite address."""
|
subject = '%s Waterfall Summary report, %s' % (report_type, date)
|
sendgmr_path = '/google/data/ro/projects/gws-sre/sendgmr'
|
command = ('%s --to=%s --subject="%s" --body_file=%s' %
|
(sendgmr_path, email_to, subject, report_file))
|
command_executer.GetCommandExecuter().RunCommand(command)
|
|
|
def GetColor(status):
|
"""Given a job status string, returns appropriate color string."""
|
if status.strip() == 'pass':
|
color = 'green '
|
elif status.strip() == 'fail':
|
color = ' red '
|
elif status.strip() == 'warning':
|
color = 'orange'
|
else:
|
color = ' '
|
return color
|
|
|
def GenerateWaterfallReport(report_dict, waterfall_type, date):
|
"""Write out the actual formatted report."""
|
|
filename = 'waterfall_report.%s_waterfall.%s.txt' % (waterfall_type, date)
|
|
date_string = ''
|
report_list = report_dict.keys()
|
|
with open(filename, 'w') as out_file:
|
# Write Report Header
|
out_file.write('\nStatus of %s Waterfall Builds from %s\n\n' %
|
(waterfall_type, date_string))
|
out_file.write(' \n')
|
out_file.write(
|
' Build bvt- '
|
' bvt-cq '
|
' security \n')
|
out_file.write(
|
' status inline '
|
' \n')
|
|
# Write daily waterfall status section.
|
for builder in report_list:
|
build_dict = report_dict[builder]
|
buildbucket_id = build_dict['buildbucket_id']
|
overall_status = build_dict['status']
|
if 'bvt-inline' in build_dict.keys():
|
inline_status = build_dict['bvt-inline']
|
else:
|
inline_status = ' '
|
if 'bvt-cq' in build_dict.keys():
|
cq_status = build_dict['bvt-cq']
|
else:
|
cq_status = ' '
|
if 'security' in build_dict.keys():
|
security_status = build_dict['security']
|
else:
|
security_status = ' '
|
inline_color = GetColor(inline_status)
|
cq_color = GetColor(cq_status)
|
security_color = GetColor(security_status)
|
|
out_file.write(
|
'%26s %4s %6s %6s %6s\n' %
|
(builder, overall_status, inline_color, cq_color, security_color))
|
if waterfall_type == 'main':
|
out_file.write(' build url: https://cros-goldeneye.corp.google.com/'
|
'chromeos/healthmonitoring/buildDetails?buildbucketId=%s'
|
'\n' % buildbucket_id)
|
else:
|
out_file.write(' build url: https://ci.chromium.org/p/chromeos/'
|
'builds/b%s \n' % buildbucket_id)
|
report_url = ('https://logs.chromium.org/v/?s=chromeos%2Fbuildbucket%2F'
|
'cr-buildbucket.appspot.com%2F' + buildbucket_id +
|
'%2F%2B%2Fsteps%2FReport%2F0%2Fstdout')
|
out_file.write('\n report status url: %s\n' % report_url)
|
out_file.write('\n')
|
|
print('Report generated in %s.' % filename)
|
return filename
|
|
|
def GetTryjobData(date, rotating_builds_dict):
|
"""Read buildbucket id and board from stored file.
|
|
buildbot_test_llvm.py, when it launches the rotating builders,
|
records the buildbucket_id and board for each launch in a file.
|
This reads that data out of the file so we can find the right
|
tryjob data.
|
"""
|
|
date_str = format_date(date, use_int_month=True)
|
fname = '%s.builds' % date_str
|
filename = os.path.join(DATA_DIR, 'rotating-builders', fname)
|
|
if not os.path.exists(filename):
|
print('Cannot find file: %s' % filename)
|
print('Unable to generate rotating builder report for date %d.' % date)
|
return
|
|
with open(filename, 'r') as in_file:
|
lines = in_file.readlines()
|
|
for line in lines:
|
l = line.strip()
|
parts = l.split(',')
|
if len(parts) != 2:
|
print('Warning: Illegal line in data file.')
|
print('File: %s' % filename)
|
print('Line: %s' % l)
|
continue
|
buildbucket_id = parts[0]
|
board = parts[1]
|
rotating_builds_dict[board] = buildbucket_id
|
|
return
|
|
|
def GetRotatingBuildData(date, report_dict, chromeos_root, board,
|
buildbucket_id, ce):
|
"""Gets rotating builder job results via 'cros buildresult'."""
|
path = os.path.join(chromeos_root, 'chromite')
|
save_dir = os.getcwd()
|
date_str = format_date(date, use_int_month=True)
|
os.chdir(path)
|
|
command = (
|
'cros buildresult --buildbucket-id %s --report json' % buildbucket_id)
|
_, out, _ = ce.RunCommandWOutput(command)
|
tmp_dict = json.loads(out)
|
results = tmp_dict[buildbucket_id]
|
|
board_dict = dict()
|
board_dict['buildbucket_id'] = buildbucket_id
|
stages_results = results['stages']
|
for test in TESTS:
|
key1 = test[0]
|
key2 = test[1]
|
if key2 in stages_results:
|
board_dict[key1] = stages_results[key2]
|
board_dict['status'] = results['status']
|
report_dict[board] = board_dict
|
os.chdir(save_dir)
|
return
|
|
|
def GetMainWaterfallData(date, report_dict, chromeos_root, ce):
|
"""Gets main waterfall job results via 'cros buildresult'."""
|
path = os.path.join(chromeos_root, 'chromite')
|
save_dir = os.getcwd()
|
date_str = format_date(date, use_int_month=True)
|
os.chdir(path)
|
for builder in WATERFALL_BUILDERS:
|
command = ('cros buildresult --build-config %s --date %s --report json' %
|
(builder, date_str))
|
_, out, _ = ce.RunCommandWOutput(command)
|
tmp_dict = json.loads(out)
|
builder_dict = dict()
|
for k in tmp_dict.keys():
|
buildbucket_id = k
|
results = tmp_dict[k]
|
|
builder_dict['buildbucket_id'] = buildbucket_id
|
builder_dict['status'] = results['status']
|
stages_results = results['stages']
|
for test in TESTS:
|
key1 = test[0]
|
key2 = test[1]
|
builder_dict[key1] = stages_results[key2]
|
report_dict[builder] = builder_dict
|
os.chdir(save_dir)
|
return
|
|
|
# Check for prodaccess.
|
def CheckProdAccess():
|
"""Verifies prodaccess is current."""
|
status, output, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
|
'prodcertstatus')
|
if status != 0:
|
return False
|
# Verify that status is not expired
|
if 'expires' in output:
|
return True
|
return False
|
|
|
def ValidDate(date):
|
"""Ensures 'date' is a valid date."""
|
min_year = 2018
|
|
tmp_date = date
|
day = tmp_date % 100
|
tmp_date = tmp_date / 100
|
month = tmp_date % 100
|
year = tmp_date / 100
|
|
if day < 1 or month < 1 or year < min_year:
|
return False
|
|
cur_year = datetime.datetime.now().year
|
if year > cur_year:
|
return False
|
|
if month > 12:
|
return False
|
|
if month == 2 and cur_year % 4 == 0 and cur_year % 100 != 0:
|
max_day = 29
|
else:
|
max_day = DAYS_PER_MONTH[month]
|
|
if day > max_day:
|
return False
|
|
return True
|
|
|
def ValidOptions(parser, options):
|
"""Error-check the options passed to this script."""
|
too_many_options = False
|
if options.main:
|
if options.rotating:
|
too_many_options = True
|
|
if too_many_options:
|
parser.error('Can only specify one of --main, --rotating.')
|
|
if not os.path.exists(options.chromeos_root):
|
parser.error(
|
'Invalid chromeos root. Cannot find: %s' % options.chromeos_root)
|
|
email_ok = True
|
if options.email and options.email.find('@') == -1:
|
email_ok = False
|
parser.error('"%s" is not a valid email address; it must contain "@..."' %
|
options.email)
|
|
valid_date = ValidDate(options.date)
|
|
return not too_many_options and valid_date and email_ok
|
|
|
def Main(argv):
|
"""Main function for this script."""
|
parser = argparse.ArgumentParser()
|
parser.add_argument(
|
'--main',
|
dest='main',
|
default=False,
|
action='store_true',
|
help='Generate report only for main waterfall '
|
'builders.')
|
parser.add_argument(
|
'--rotating',
|
dest='rotating',
|
default=False,
|
action='store_true',
|
help='Generate report only for rotating builders.')
|
parser.add_argument(
|
'--date',
|
dest='date',
|
required=True,
|
type=int,
|
help='The date YYYYMMDD of waterfall report.')
|
parser.add_argument(
|
'--email',
|
dest='email',
|
default='',
|
help='Email address to use for sending the report.')
|
parser.add_argument(
|
'--chromeos_root',
|
dest='chromeos_root',
|
required=True,
|
help='Chrome OS root in which to run chroot commands.')
|
|
options = parser.parse_args(argv)
|
|
if not ValidOptions(parser, options):
|
return 1
|
|
main_only = options.main
|
rotating_only = options.rotating
|
date = options.date
|
|
prod_access = CheckProdAccess()
|
if not prod_access:
|
print('ERROR: Please run prodaccess first.')
|
return
|
|
waterfall_report_dict = dict()
|
rotating_report_dict = dict()
|
|
ce = command_executer.GetCommandExecuter()
|
if not rotating_only:
|
GetMainWaterfallData(date, waterfall_report_dict, options.chromeos_root, ce)
|
|
if not main_only:
|
rotating_builds_dict = dict()
|
GetTryjobData(date, rotating_builds_dict)
|
if len(rotating_builds_dict.keys()) > 0:
|
for board in rotating_builds_dict.keys():
|
buildbucket_id = rotating_builds_dict[board]
|
GetRotatingBuildData(date, rotating_report_dict, options.chromeos_root,
|
board, buildbucket_id, ce)
|
|
if options.email:
|
email_to = options.email
|
else:
|
email_to = getpass.getuser()
|
|
if waterfall_report_dict and not rotating_only:
|
main_report = GenerateWaterfallReport(waterfall_report_dict, 'main', date)
|
|
EmailReport(main_report, 'Main', format_date(date), email_to)
|
shutil.copy(main_report, ARCHIVE_DIR)
|
if rotating_report_dict and not main_only:
|
rotating_report = GenerateWaterfallReport(rotating_report_dict, 'rotating',
|
date)
|
|
EmailReport(rotating_report, 'Rotating', format_date(date), email_to)
|
shutil.copy(rotating_report, ARCHIVE_DIR)
|
|
|
if __name__ == '__main__':
|
Main(sys.argv[1:])
|
sys.exit(0)
|
