#!/usr/bin/python
|
# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
|
# Use of this source code is governed by a BSD-style license that can be
|
# found in the LICENSE file.
|
|
"""Runs on autotest servers from a cron job to self update them.
|
|
This script is designed to run on all autotest servers to allow them to
|
automatically self-update based on the manifests used to create their (existing)
|
repos.
|
"""
|
|
from __future__ import print_function
|
|
import ConfigParser
|
import argparse
|
import os
|
import re
|
import subprocess
|
import socket
|
import sys
|
import time
|
|
import common
|
|
from autotest_lib.client.common_lib import global_config
|
from autotest_lib.server import utils as server_utils
|
from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
|
|
|
# How long after restarting a service do we watch it to see if it's stable.
|
SERVICE_STABILITY_TIMER = 60
|
|
# A dict to map update_commands defined in config file to repos or files that
|
# decide whether need to update these commands. E.g. if no changes under
|
# frontend repo, no need to update afe.
|
COMMANDS_TO_REPOS_DICT = {'afe': 'frontend/client/',
|
'tko': 'frontend/client/'}
|
BUILD_EXTERNALS_COMMAND = 'build_externals'
|
|
_RESTART_SERVICES_FILE = os.path.join(os.environ['HOME'],
|
'push_restart_services')
|
|
AFE = frontend_wrappers.RetryingAFE(
|
server=server_utils.get_global_afe_hostname(), timeout_min=5,
|
delay_sec=10)
|
HOSTNAME = socket.gethostname()
|
|
class DirtyTreeException(Exception):
|
"""Raised when the tree has been modified in an unexpected way."""
|
|
|
class UnknownCommandException(Exception):
|
"""Raised when we try to run a command name with no associated command."""
|
|
|
class UnstableServices(Exception):
|
"""Raised if a service appears unstable after restart."""
|
|
|
def strip_terminal_codes(text):
|
"""This function removes all terminal formatting codes from a string.
|
|
@param text: String of text to cleanup.
|
@returns String with format codes removed.
|
"""
|
ESC = '\x1b'
|
return re.sub(ESC+r'\[[^m]*m', '', text)
|
|
|
def _clean_pyc_files():
|
print('Removing .pyc files')
|
try:
|
subprocess.check_output([
|
'find', '.',
|
'(',
|
# These are ignored to reduce IO load (crbug.com/759780).
|
'-path', './site-packages',
|
'-o', '-path', './containers',
|
'-o', '-path', './logs',
|
'-o', '-path', './results',
|
')',
|
'-prune',
|
'-o', '-name', '*.pyc',
|
'-exec', 'rm', '-f', '{}', '+'])
|
except Exception as e:
|
print('Warning: fail to remove .pyc! %s' % e)
|
|
|
def verify_repo_clean():
|
"""This function cleans the current repo then verifies that it is valid.
|
|
@raises DirtyTreeException if the repo is still not clean.
|
@raises subprocess.CalledProcessError on a repo command failure.
|
"""
|
subprocess.check_output(['git', 'stash', '-u'])
|
subprocess.check_output(['git', 'stash', 'clear'])
|
out = subprocess.check_output(['repo', 'status'], stderr=subprocess.STDOUT)
|
out = strip_terminal_codes(out).strip()
|
|
if not 'working directory clean' in out and not 'working tree clean' in out:
|
raise DirtyTreeException('%s repo not clean: %s' % (HOSTNAME, out))
|
|
|
def _clean_externals():
|
"""Clean untracked files within ExternalSource and site-packages/
|
|
@raises subprocess.CalledProcessError on a git command failure.
|
"""
|
dirs_to_clean = ['site-packages/', 'ExternalSource/']
|
cmd = ['git', 'clean', '-fxd'] + dirs_to_clean
|
subprocess.check_output(cmd)
|
|
|
def repo_versions():
|
"""This function collects the versions of all git repos in the general repo.
|
|
@returns A dictionary mapping project names to git hashes for HEAD.
|
@raises subprocess.CalledProcessError on a repo command failure.
|
"""
|
cmd = ['repo', 'forall', '-p', '-c', 'pwd && git log -1 --format=%h']
|
output = strip_terminal_codes(subprocess.check_output(cmd))
|
|
# The expected output format is:
|
|
# project chrome_build/
|
# /dir/holding/chrome_build
|
# 73dee9d
|
#
|
# project chrome_release/
|
# /dir/holding/chrome_release
|
# 9f3a5d8
|
|
lines = output.splitlines()
|
|
PROJECT_PREFIX = 'project '
|
|
project_heads = {}
|
for n in range(0, len(lines), 4):
|
project_line = lines[n]
|
project_dir = lines[n+1]
|
project_hash = lines[n+2]
|
# lines[n+3] is a blank line, but doesn't exist for the final block.
|
|
# Convert 'project chrome_build/' -> 'chrome_build'
|
assert project_line.startswith(PROJECT_PREFIX)
|
name = project_line[len(PROJECT_PREFIX):].rstrip('/')
|
|
project_heads[name] = (project_dir, project_hash)
|
|
return project_heads
|
|
|
def repo_versions_to_decide_whether_run_cmd_update():
|
"""Collect versions of repos/files defined in COMMANDS_TO_REPOS_DICT.
|
|
For the update_commands defined in config files, no need to run the command
|
every time. Only run it when the repos/files related to the commands have
|
been changed.
|
|
@returns A set of tuples: {(cmd, repo_version), ()...}
|
"""
|
results = set()
|
for cmd, repo in COMMANDS_TO_REPOS_DICT.iteritems():
|
version = subprocess.check_output(
|
['git', 'log', '-1', '--pretty=tformat:%h',
|
'%s/%s' % (common.autotest_dir, repo)])
|
results.add((cmd, version.strip()))
|
return results
|
|
|
def repo_sync(update_push_servers=False):
|
"""Perform a repo sync.
|
|
@param update_push_servers: If True, then update test_push servers to ToT.
|
Otherwise, update server to prod branch.
|
@raises subprocess.CalledProcessError on a repo command failure.
|
"""
|
subprocess.check_output(['repo', 'sync', '--force-sync'])
|
if update_push_servers:
|
print('Updating push servers, checkout cros/master')
|
subprocess.check_output(['git', 'checkout', 'cros/master'],
|
stderr=subprocess.STDOUT)
|
else:
|
print('Updating server to prod branch')
|
subprocess.check_output(['git', 'checkout', 'cros/prod'],
|
stderr=subprocess.STDOUT)
|
_clean_pyc_files()
|
|
|
def discover_update_commands():
|
"""Lookup the commands to run on this server.
|
|
These commonly come from shadow_config.ini, since they vary by server type.
|
|
@returns List of command names in string format.
|
"""
|
try:
|
return global_config.global_config.get_config_value(
|
'UPDATE', 'commands', type=list)
|
|
except (ConfigParser.NoSectionError, global_config.ConfigError):
|
return []
|
|
|
def get_restart_services():
|
"""Find the services that need restarting on the current server.
|
|
These commonly come from shadow_config.ini, since they vary by server type.
|
|
@returns Iterable of service names in string format.
|
"""
|
with open(_RESTART_SERVICES_FILE) as f:
|
for line in f:
|
yield line.rstrip()
|
|
|
def update_command(cmd_tag, dryrun=False, use_chromite_master=False):
|
"""Restart a command.
|
|
The command name is looked up in global_config.ini to find the full command
|
to run, then it's executed.
|
|
@param cmd_tag: Which command to restart.
|
@param dryrun: If true print the command that would have been run.
|
@param use_chromite_master: True if updating chromite to master, rather
|
than prod.
|
|
@raises UnknownCommandException If cmd_tag can't be looked up.
|
@raises subprocess.CalledProcessError on a command failure.
|
"""
|
# Lookup the list of commands to consider. They are intended to be
|
# in global_config.ini so that they can be shared everywhere.
|
cmds = dict(global_config.global_config.config.items(
|
'UPDATE_COMMANDS'))
|
|
if cmd_tag not in cmds:
|
raise UnknownCommandException(cmd_tag, cmds)
|
|
command = cmds[cmd_tag]
|
# When updating push servers, pass an arg to build_externals to update
|
# chromite to master branch for testing
|
if use_chromite_master and cmd_tag == BUILD_EXTERNALS_COMMAND:
|
command += ' --use_chromite_master'
|
|
print('Running: %s: %s' % (cmd_tag, command))
|
if dryrun:
|
print('Skip: %s' % command)
|
else:
|
try:
|
subprocess.check_output(command, shell=True,
|
cwd=common.autotest_dir,
|
stderr=subprocess.STDOUT)
|
except subprocess.CalledProcessError as e:
|
print('FAILED %s :' % HOSTNAME)
|
print(e.output)
|
raise
|
|
|
def restart_service(service_name, dryrun=False):
|
"""Restart a service.
|
|
Restarts the standard service with "service <name> restart".
|
|
@param service_name: The name of the service to restart.
|
@param dryrun: Don't really run anything, just print out the command.
|
|
@raises subprocess.CalledProcessError on a command failure.
|
"""
|
cmd = ['sudo', 'service', service_name, 'restart']
|
print('Restarting: %s' % service_name)
|
if dryrun:
|
print('Skip: %s' % ' '.join(cmd))
|
else:
|
subprocess.check_call(cmd, stderr=subprocess.STDOUT)
|
|
|
def service_status(service_name):
|
"""Return the results "status <name>" for a given service.
|
|
This string is expected to contain the pid, and so to change is the service
|
is shutdown or restarted for any reason.
|
|
@param service_name: The name of the service to check on.
|
|
@returns The output of the external command.
|
Ex: autofs start/running, process 1931
|
|
@raises subprocess.CalledProcessError on a command failure.
|
"""
|
return subprocess.check_output(['sudo', 'service', service_name, 'status'])
|
|
|
def restart_services(service_names, dryrun=False, skip_service_status=False):
|
"""Restart services as needed for the current server type.
|
|
Restart the listed set of services, and watch to see if they are stable for
|
at least SERVICE_STABILITY_TIMER. It restarts all services quickly,
|
waits for that delay, then verifies the status of all of them.
|
|
@param service_names: The list of service to restart and monitor.
|
@param dryrun: Don't really restart the service, just print out the command.
|
@param skip_service_status: Set to True to skip service status check.
|
Default is False.
|
|
@raises subprocess.CalledProcessError on a command failure.
|
@raises UnstableServices if any services are unstable after restart.
|
"""
|
service_statuses = {}
|
|
if dryrun:
|
for name in service_names:
|
restart_service(name, dryrun=True)
|
return
|
|
# Restart each, and record the status (including pid).
|
for name in service_names:
|
restart_service(name)
|
|
# Skip service status check if --skip-service-status is specified. Used for
|
# servers in backup status.
|
if skip_service_status:
|
print('--skip-service-status is specified, skip checking services.')
|
return
|
|
# Wait for a while to let the services settle.
|
time.sleep(SERVICE_STABILITY_TIMER)
|
service_statuses = {name: service_status(name) for name in service_names}
|
time.sleep(SERVICE_STABILITY_TIMER)
|
# Look for any services that changed status.
|
unstable_services = [n for n in service_names
|
if service_status(n) != service_statuses[n]]
|
|
# Report any services having issues.
|
if unstable_services:
|
raise UnstableServices('%s service restart failed: %s' %
|
(HOSTNAME, unstable_services))
|
|
|
def run_deploy_actions(cmds_skip=set(), dryrun=False,
|
skip_service_status=False, use_chromite_master=False):
|
"""Run arbitrary update commands specified in global.ini.
|
|
@param cmds_skip: cmds no need to run since the corresponding repo/file
|
does not change.
|
@param dryrun: Don't really restart the service, just print out the command.
|
@param skip_service_status: Set to True to skip service status check.
|
Default is False.
|
@param use_chromite_master: True if updating chromite to master, rather
|
than prod.
|
|
@raises subprocess.CalledProcessError on a command failure.
|
@raises UnstableServices if any services are unstable after restart.
|
"""
|
defined_cmds = set(discover_update_commands())
|
cmds = defined_cmds - cmds_skip
|
if cmds:
|
print('Running update commands:', ', '.join(cmds))
|
for cmd in cmds:
|
update_command(cmd, dryrun=dryrun,
|
use_chromite_master=use_chromite_master)
|
|
services = list(get_restart_services())
|
if services:
|
print('Restarting Services:', ', '.join(services))
|
restart_services(services, dryrun=dryrun,
|
skip_service_status=skip_service_status)
|
|
|
def report_changes(versions_before, versions_after):
|
"""Produce a report describing what changed in all repos.
|
|
@param versions_before: Results of repo_versions() from before the update.
|
@param versions_after: Results of repo_versions() from after the update.
|
|
@returns string containing a human friendly changes report.
|
"""
|
result = []
|
|
if versions_after:
|
for project in sorted(set(versions_before.keys() + versions_after.keys())):
|
result.append('%s:' % project)
|
|
_, before_hash = versions_before.get(project, (None, None))
|
after_dir, after_hash = versions_after.get(project, (None, None))
|
|
if project not in versions_before:
|
result.append('Added.')
|
|
elif project not in versions_after:
|
result.append('Removed.')
|
|
elif before_hash == after_hash:
|
result.append('No Change.')
|
|
else:
|
hashes = '%s..%s' % (before_hash, after_hash)
|
cmd = ['git', 'log', hashes, '--oneline']
|
out = subprocess.check_output(cmd, cwd=after_dir,
|
stderr=subprocess.STDOUT)
|
result.append(out.strip())
|
|
result.append('')
|
else:
|
for project in sorted(versions_before.keys()):
|
_, before_hash = versions_before[project]
|
result.append('%s: %s' % (project, before_hash))
|
result.append('')
|
|
return '\n'.join(result)
|
|
|
def parse_arguments(args):
|
"""Parse command line arguments.
|
|
@param args: The command line arguments to parse. (ususally sys.argsv[1:])
|
|
@returns An argparse.Namespace populated with argument values.
|
"""
|
parser = argparse.ArgumentParser(
|
description='Command to update an autotest server.')
|
parser.add_argument('--skip-verify', action='store_false',
|
dest='verify', default=True,
|
help='Disable verification of a clean repository.')
|
parser.add_argument('--skip-update', action='store_false',
|
dest='update', default=True,
|
help='Skip the repository source code update.')
|
parser.add_argument('--skip-actions', action='store_false',
|
dest='actions', default=True,
|
help='Skip the post update actions.')
|
parser.add_argument('--skip-report', action='store_false',
|
dest='report', default=True,
|
help='Skip the git version report.')
|
parser.add_argument('--actions-only', action='store_true',
|
help='Run the post update actions (restart services).')
|
parser.add_argument('--dryrun', action='store_true',
|
help='Don\'t actually run any commands, just log.')
|
parser.add_argument('--skip-service-status', action='store_true',
|
help='Skip checking the service status.')
|
parser.add_argument('--update_push_servers', action='store_true',
|
help='Indicate to update test_push server. If not '
|
'specify, then update server to production.')
|
parser.add_argument('--force-clean-externals', action='store_true',
|
default=False,
|
help='Force a cleanup of all untracked files within '
|
'site-packages/ and ExternalSource/, so that '
|
'build_externals will build from scratch.')
|
parser.add_argument('--force_update', action='store_true',
|
help='Force to run the update commands for afe, tko '
|
'and build_externals')
|
|
results = parser.parse_args(args)
|
|
if results.actions_only:
|
results.verify = False
|
results.update = False
|
results.report = False
|
|
# TODO(dgarrett): Make these behaviors support dryrun.
|
if results.dryrun:
|
results.verify = False
|
results.update = False
|
results.force_clean_externals = False
|
|
if not results.update_push_servers:
|
print('Will skip service check for pushing servers in prod.')
|
results.skip_service_status = True
|
return results
|
|
|
class ChangeDir(object):
|
|
"""Context manager for changing to a directory temporarily."""
|
|
def __init__(self, dir):
|
self.new_dir = dir
|
self.old_dir = None
|
|
def __enter__(self):
|
self.old_dir = os.getcwd()
|
os.chdir(self.new_dir)
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
os.chdir(self.old_dir)
|
|
|
def _sync_chromiumos_repo():
|
"""Update ~chromeos-test/chromiumos repo."""
|
print('Updating ~chromeos-test/chromiumos')
|
with ChangeDir(os.path.expanduser('~chromeos-test/chromiumos')):
|
ret = subprocess.call(['repo', 'sync', '--force-sync'],
|
stderr=subprocess.STDOUT)
|
_clean_pyc_files()
|
if ret != 0:
|
print('Update failed, exited with status: %d' % ret)
|
|
|
def main(args):
|
"""Main method."""
|
# Be careful before you change this call to `os.chdir()`:
|
# We make several calls to `subprocess.check_output()` and
|
# friends that depend on this directory, most notably calls to
|
# the 'repo' command from `verify_repo_clean()`.
|
os.chdir(common.autotest_dir)
|
global_config.global_config.parse_config_file()
|
|
behaviors = parse_arguments(args)
|
print('Updating server: %s' % HOSTNAME)
|
if behaviors.verify:
|
print('Checking tree status:')
|
verify_repo_clean()
|
print('Tree status: clean')
|
|
if behaviors.force_clean_externals:
|
print('Cleaning all external packages and their cache...')
|
_clean_externals()
|
print('...done.')
|
|
versions_before = repo_versions()
|
versions_after = set()
|
cmd_versions_before = repo_versions_to_decide_whether_run_cmd_update()
|
cmd_versions_after = set()
|
|
if behaviors.update:
|
print('Updating Repo.')
|
repo_sync(behaviors.update_push_servers)
|
versions_after = repo_versions()
|
cmd_versions_after = repo_versions_to_decide_whether_run_cmd_update()
|
_sync_chromiumos_repo()
|
|
if behaviors.actions:
|
# If the corresponding repo/file not change, no need to run the cmd.
|
cmds_skip = (set() if behaviors.force_update else
|
{t[0] for t in cmd_versions_before & cmd_versions_after})
|
run_deploy_actions(
|
cmds_skip, behaviors.dryrun, behaviors.skip_service_status,
|
use_chromite_master=behaviors.update_push_servers)
|
|
if behaviors.report:
|
print('Changes:')
|
print(report_changes(versions_before, versions_after))
|
|
|
if __name__ == '__main__':
|
sys.exit(main(sys.argv[1:]))
|
