#pylint: disable=C0111
|
|
"""
|
Prejob tasks.
|
|
Prejob tasks _usually_ run before a job and verify the state of a machine.
|
Cleanup and repair are exceptions, cleanup can run after a job too, while
|
repair will run anytime the host needs a repair, which could be pre or post
|
job. Most of the work specific to this module is achieved through the prolog
|
and epilog of each task.
|
|
All prejob tasks must have a host, though they may not have an HQE. If a
|
prejob task has a hqe, it will activate the hqe through its on_pending
|
method on successful completion. A row in afe_special_tasks with values:
|
host=C1, unlocked, is_active=0, is_complete=0, type=Verify
|
will indicate to the scheduler that it needs to schedule a new special task
|
of type=Verify, against the C1 host. While the special task is running
|
the scheduler only monitors it through the Agent, and its is_active bit=1.
|
Once a special task finishes, we set its is_active=0, is_complete=1 and
|
success bits, so the scheduler ignores it.
|
HQE.on_pending:
|
Host, HQE -> Pending, Starting
|
This status is acted upon in the scheduler, to assign an AgentTask.
|
PreJobTask:
|
epilog:
|
failure:
|
requeue hqe
|
repair the host
|
Children PreJobTasks:
|
prolog:
|
set Host, HQE status
|
epilog:
|
success:
|
on_pending
|
failure:
|
repair throgh PreJobTask
|
set Host, HQE status
|
|
Failing a prejob task effects both the Host and the HQE, as follows:
|
|
- Host: PreJob failure will result in a Repair job getting queued against
|
the host, is we haven't already tried repairing it more than the
|
max_repair_limit. When this happens, the host will remain in whatever status
|
the prejob task left it in, till the Repair job puts it into 'Repairing'. This
|
way the host_scheduler won't pick bad hosts and assign them to jobs.
|
|
If we have already tried repairing the host too many times, the PreJobTask
|
will flip the host to 'RepairFailed' in its epilog, and it will remain in this
|
state till it is recovered and reverified.
|
|
- HQE: Is either requeued or failed. Requeuing the HQE involves putting it
|
in the Queued state and setting its host_id to None, so it gets a new host
|
in the next scheduler tick. Failing the HQE results in either a Parsing
|
or Archiving postjob task, and an eventual Failed status for the HQE.
|
"""
|
|
import logging
|
import re
|
|
from autotest_lib.client.common_lib import host_protections
|
from autotest_lib.frontend.afe import models
|
from autotest_lib.scheduler import agent_task
|
from autotest_lib.scheduler import drone_manager
|
from autotest_lib.scheduler import scheduler_config
|
from autotest_lib.server import autoserv_utils
|
from autotest_lib.server.cros import provision
|
|
|
class PreJobTask(agent_task.SpecialAgentTask):
|
def epilog(self):
|
super(PreJobTask, self).epilog()
|
|
if self.host.protection == host_protections.Protection.DO_NOT_VERIFY:
|
# effectively ignore failure for these hosts
|
self.success = True
|
|
if self.success:
|
self.host.record_working_state(True,
|
self.task.time_finished)
|
return
|
|
if self.queue_entry:
|
# If we requeue a HQE, we should cancel any remaining pre-job
|
# tasks against this host, otherwise we'll be left in a state
|
# where a queued HQE has special tasks to run against a host.
|
models.SpecialTask.objects.filter(
|
queue_entry__id=self.queue_entry.id,
|
host__id=self.host.id,
|
is_complete=0).update(is_complete=1, success=0)
|
|
previous_provisions = models.SpecialTask.objects.filter(
|
task=models.SpecialTask.Task.PROVISION,
|
queue_entry_id=self.queue_entry.id).count()
|
if (previous_provisions >
|
scheduler_config.config.max_provision_retries):
|
self._actually_fail_queue_entry()
|
# This abort will mark the aborted bit on the HQE itself, to
|
# signify that we're killing it. Technically it also will do
|
# the recursive aborting of all child jobs, but that shouldn't
|
# matter here, as only suites have children, and those are
|
# hostless and thus don't have provisioning.
|
# TODO(milleral) http://crbug.com/188217
|
# However, we can't actually do this yet, as if we set the
|
# abort bit the FinalReparseTask will set the status of the HQE
|
# to ABORTED, which then means that we don't show the status in
|
# run_suite. So in the meantime, don't mark the HQE as
|
# aborted.
|
# queue_entry.abort()
|
else:
|
# requeue() must come after handling provision retries, since
|
# _actually_fail_queue_entry needs an execution subdir.
|
# We also don't want to requeue if we hit the provision retry
|
# limit, since then we overwrite the PARSING state of the HQE.
|
self.queue_entry.requeue()
|
|
# Limit the repair on a host when a prejob task fails, e.g., reset,
|
# verify etc. The number of repair jobs is limited to the specific
|
# HQE and host.
|
previous_repairs = models.SpecialTask.objects.filter(
|
task=models.SpecialTask.Task.REPAIR,
|
queue_entry_id=self.queue_entry.id,
|
host_id=self.queue_entry.host_id).count()
|
if previous_repairs >= scheduler_config.config.max_repair_limit:
|
self.host.set_status(models.Host.Status.REPAIR_FAILED)
|
self._fail_queue_entry()
|
return
|
|
queue_entry = models.HostQueueEntry.objects.get(
|
id=self.queue_entry.id)
|
else:
|
queue_entry = None
|
|
models.SpecialTask.objects.create(
|
host=models.Host.objects.get(id=self.host.id),
|
task=models.SpecialTask.Task.REPAIR,
|
queue_entry=queue_entry,
|
requested_by=self.task.requested_by)
|
|
|
def _should_pending(self):
|
"""
|
Decide if we should call the host queue entry's on_pending method.
|
We should if:
|
1) There exists an associated host queue entry.
|
2) The current special task completed successfully.
|
3) There do not exist any more special tasks to be run before the
|
host queue entry starts.
|
|
@returns: True if we should call pending, false if not.
|
|
"""
|
if not self.queue_entry or not self.success:
|
return False
|
|
# We know if this is the last one when we create it, so we could add
|
# another column to the database to keep track of this information, but
|
# I expect the overhead of querying here to be minimal.
|
queue_entry = models.HostQueueEntry.objects.get(id=self.queue_entry.id)
|
queued = models.SpecialTask.objects.filter(
|
host__id=self.host.id, is_active=False,
|
is_complete=False, queue_entry=queue_entry)
|
queued = queued.exclude(id=self.task.id)
|
return queued.count() == 0
|
|
|
class VerifyTask(PreJobTask):
|
TASK_TYPE = models.SpecialTask.Task.VERIFY
|
|
|
def __init__(self, task):
|
args = ['-v']
|
if task.queue_entry:
|
args.extend(self._generate_autoserv_label_args(task))
|
super(VerifyTask, self).__init__(task, args)
|
self._set_ids(host=self.host, queue_entries=[self.queue_entry])
|
|
|
def prolog(self):
|
super(VerifyTask, self).prolog()
|
|
logging.info("starting verify on %s", self.host.hostname)
|
if self.queue_entry:
|
self.queue_entry.set_status(models.HostQueueEntry.Status.VERIFYING)
|
self.host.set_status(models.Host.Status.VERIFYING)
|
|
# Delete any queued manual reverifies for this host. One verify will do
|
# and there's no need to keep records of other requests.
|
self.remove_special_tasks(models.SpecialTask.Task.VERIFY,
|
keep_last_one=True)
|
|
|
def epilog(self):
|
super(VerifyTask, self).epilog()
|
if self.success:
|
if self._should_pending():
|
self.queue_entry.on_pending()
|
else:
|
self.host.set_status(models.Host.Status.READY)
|
|
|
class CleanupTask(PreJobTask):
|
# note this can also run post-job, but when it does, it's running standalone
|
# against the host (not related to the job), so it's not considered a
|
# PostJobTask
|
|
TASK_TYPE = models.SpecialTask.Task.CLEANUP
|
|
|
def __init__(self, task, recover_run_monitor=None):
|
args = ['--cleanup']
|
if task.queue_entry:
|
args.extend(self._generate_autoserv_label_args(task))
|
super(CleanupTask, self).__init__(task, args)
|
self._set_ids(host=self.host, queue_entries=[self.queue_entry])
|
|
|
def prolog(self):
|
super(CleanupTask, self).prolog()
|
logging.info("starting cleanup task for host: %s", self.host.hostname)
|
self.host.set_status(models.Host.Status.CLEANING)
|
if self.queue_entry:
|
self.queue_entry.set_status(models.HostQueueEntry.Status.CLEANING)
|
|
|
def _finish_epilog(self):
|
if not self.queue_entry or not self.success:
|
return
|
|
do_not_verify_protection = host_protections.Protection.DO_NOT_VERIFY
|
should_run_verify = (
|
self.queue_entry.job.run_verify
|
and self.host.protection != do_not_verify_protection)
|
if should_run_verify:
|
entry = models.HostQueueEntry.objects.get(id=self.queue_entry.id)
|
models.SpecialTask.objects.create(
|
host=models.Host.objects.get(id=self.host.id),
|
queue_entry=entry,
|
task=models.SpecialTask.Task.VERIFY)
|
else:
|
if self._should_pending():
|
self.queue_entry.on_pending()
|
|
|
def epilog(self):
|
super(CleanupTask, self).epilog()
|
|
if self.success:
|
self.host.update_field('dirty', 0)
|
self.host.set_status(models.Host.Status.READY)
|
|
self._finish_epilog()
|
|
|
class ResetTask(PreJobTask):
|
"""Task to reset a DUT, including cleanup and verify."""
|
# note this can also run post-job, but when it does, it's running standalone
|
# against the host (not related to the job), so it's not considered a
|
# PostJobTask
|
|
TASK_TYPE = models.SpecialTask.Task.RESET
|
|
|
def __init__(self, task, recover_run_monitor=None):
|
args = ['--reset']
|
if task.queue_entry:
|
args.extend(self._generate_autoserv_label_args(task))
|
super(ResetTask, self).__init__(task, args)
|
self._set_ids(host=self.host, queue_entries=[self.queue_entry])
|
|
|
def prolog(self):
|
super(ResetTask, self).prolog()
|
logging.info('starting reset task for host: %s',
|
self.host.hostname)
|
self.host.set_status(models.Host.Status.RESETTING)
|
if self.queue_entry:
|
self.queue_entry.set_status(models.HostQueueEntry.Status.RESETTING)
|
|
# Delete any queued cleanups for this host.
|
self.remove_special_tasks(models.SpecialTask.Task.CLEANUP,
|
keep_last_one=False)
|
|
# Delete any queued reverifies for this host.
|
self.remove_special_tasks(models.SpecialTask.Task.VERIFY,
|
keep_last_one=False)
|
|
# Only one reset is needed.
|
self.remove_special_tasks(models.SpecialTask.Task.RESET,
|
keep_last_one=True)
|
|
|
def epilog(self):
|
super(ResetTask, self).epilog()
|
|
if self.success:
|
self.host.update_field('dirty', 0)
|
|
if self._should_pending():
|
self.queue_entry.on_pending()
|
else:
|
self.host.set_status(models.Host.Status.READY)
|
|
|
# TODO (ayatane): Refactor using server/cros/provision
|
def _is_cros_version(label):
|
"""Return whether the label is a cros-version: label."""
|
return label.startswith('cros-version:')
|
|
|
# TODO (ayatane): Refactor using server/cros/provision
|
def _get_cros_version(label):
|
"""Return cros-version from cros-version label."""
|
return label[len('cros-version:'):]
|
|
|
# TODO (ayatane): Refactor into server/cros/provision
|
class _CrosImage(object):
|
"""The name of a CrOS image."""
|
|
_name_pattern = re.compile(
|
r'^'
|
r'(?P<group>[a-z0-9-]+)'
|
r'/'
|
r'(?P<milestone>LATEST|R[0-9]+)'
|
r'-'
|
r'(?P<version>[0-9.]+)'
|
r'(-(?P<rc>rc[0-9]+))?'
|
r'$'
|
)
|
|
def __init__(self, name):
|
"""Initialize instance.
|
|
@param name: Image name string (lumpy-release/R27-3773.0.0)
|
"""
|
self._name = name
|
match = self._name_pattern.search(name)
|
if match is None:
|
raise ValueError('Invalid CrOS image name: %r' % name)
|
self.group = match.group('group')
|
self.milestone = match.group('milestone')
|
self.version = match.group('version')
|
self.rc = match.group('rc')
|
|
def __repr__(self):
|
return '{cls}({name!r})'.format(cls=type(self).__name__,
|
name=self._name)
|
|
def __str__(self):
|
return self._name
|
|
|
class ProvisionTask(PreJobTask):
|
TASK_TYPE = models.SpecialTask.Task.PROVISION
|
|
def __init__(self, task):
|
# Provisioning requires that we be associated with a job/queue entry
|
assert task.queue_entry, "No HQE associated with provision task!"
|
# task.queue_entry is an afe model HostQueueEntry object.
|
# self.queue_entry is a scheduler models HostQueueEntry object, but
|
# it gets constructed and assigned in __init__, so it's not available
|
# yet. Therefore, we're stuck pulling labels off of the afe model
|
# so that we can pass the --provision args into the __init__ call.
|
labels = {x.name for x in task.queue_entry.job.labels}
|
_, provisionable = provision.Provision.partition(labels)
|
extra_command_args = ['--provision',
|
'--job-labels', ','.join(provisionable)]
|
super(ProvisionTask, self).__init__(task, extra_command_args)
|
self._set_milestone(labels)
|
self._set_ids(host=self.host, queue_entries=[self.queue_entry])
|
|
|
def _set_milestone(self, labels):
|
"""Set build milestone from the labels.
|
|
@param labels: iterable of labels.
|
"""
|
labels = (label
|
for label in labels
|
if _is_cros_version(label))
|
for label in labels:
|
try:
|
cros_image = _CrosImage(_get_cros_version(label))
|
except ValueError as e:
|
logging.warning('Could not parse cros-version. Error msg: %s', e)
|
self._milestone = 'N/A'
|
else:
|
self._milestone = cros_image.milestone
|
break
|
|
|
def _command_line(self):
|
# If we give queue_entry to autoserv_run_job_command, then it will
|
# append -c for this invocation if the queue_entry is a client side
|
# test. We don't want that, as it messes with provisioning, so we just
|
# drop it from the arguments here.
|
# Note that we also don't verify job_repo_url as provisioining tasks are
|
# required to stage whatever content we need, and the job itself will
|
# force autotest to be staged if it isn't already.
|
return autoserv_utils.autoserv_run_job_command(
|
autoserv_utils.autoserv_directory,
|
self.host.hostname,
|
results_directory=drone_manager.WORKING_DIRECTORY,
|
extra_args=self._extra_command_args,
|
in_lab=True,
|
)
|
|
def prolog(self):
|
super(ProvisionTask, self).prolog()
|
# add check for previous provision task and abort if exist.
|
logging.info("starting provision task for host: %s", self.host.hostname)
|
self.queue_entry.set_status(
|
models.HostQueueEntry.Status.PROVISIONING)
|
self.host.set_status(models.Host.Status.PROVISIONING)
|
|
|
def epilog(self):
|
super(ProvisionTask, self).epilog()
|
|
# If we were not successful in provisioning the machine
|
# leave the DUT in whatever status was set in the PreJobTask's
|
# epilog. If this task was successful the host status will get
|
# set appropriately as a fallout of the hqe's on_pending. If
|
# we don't call on_pending, it can only be because:
|
# 1. This task was not successful:
|
# a. Another repair is queued: this repair job will set the host
|
# status, and it will remain in 'Provisioning' till then.
|
# b. We have hit the max_repair_limit: in which case the host
|
# status is set to 'RepairFailed' in the epilog of PreJobTask.
|
# 2. The task was successful, but there are other special tasks:
|
# Those special tasks will set the host status appropriately.
|
if self._should_pending():
|
self.queue_entry.on_pending()
|
|
|
class RepairTask(agent_task.SpecialAgentTask):
|
TASK_TYPE = models.SpecialTask.Task.REPAIR
|
|
|
def __init__(self, task):
|
"""\
|
queue_entry: queue entry to mark failed if this repair fails.
|
"""
|
protection = host_protections.Protection.get_string(
|
task.host.protection)
|
# normalize the protection name
|
protection = host_protections.Protection.get_attr_name(protection)
|
|
args = ['-R', '--host-protection', protection]
|
if task.queue_entry:
|
args.extend(self._generate_autoserv_label_args(task))
|
|
super(RepairTask, self).__init__(task, args)
|
|
# *don't* include the queue entry in IDs -- if the queue entry is
|
# aborted, we want to leave the repair task running
|
self._set_ids(host=self.host)
|
|
|
def prolog(self):
|
super(RepairTask, self).prolog()
|
logging.info("repair_task starting")
|
self.host.set_status(models.Host.Status.REPAIRING)
|
|
|
def epilog(self):
|
super(RepairTask, self).epilog()
|
|
if self.success:
|
self.host.set_status(models.Host.Status.READY)
|
else:
|
self.host.set_status(models.Host.Status.REPAIR_FAILED)
|
if self.queue_entry:
|
self._fail_queue_entry()
|
self.host.record_working_state(bool(self.success),
|
self.task.time_finished)
|
