#!/usr/bin/env python
|
# Copyright 2015 The Chromium OS Authors. All rights reserved.
|
# Use of this source code is governed by a BSD-style license that can be
|
# found in the LICENSE file.
|
|
"""Create e-mail reports of the Lab's DUT inventory.
|
|
Gathers a list of all DUTs of interest in the Lab, segregated by
|
model and pool, and determines whether each DUT is working or
|
broken. Then, send one or more e-mail reports summarizing the
|
status to e-mail addresses provided on the command line.
|
|
usage: lab_inventory.py [ options ] [ model ... ]
|
|
Options:
|
--duration / -d <hours>
|
How far back in time to search job history to determine DUT
|
status.
|
|
--model-notify <address>[,<address>]
|
Send the "model status" e-mail to all the specified e-mail
|
addresses.
|
|
--pool-notify <address>[,<address>]
|
Send the "pool status" e-mail to all the specified e-mail
|
addresses.
|
|
--recommend <number>
|
When generating the "model status" e-mail, include a list of
|
<number> specific DUTs to be recommended for repair.
|
|
--report-untestable
|
Scan the inventory for DUTs that can't test because they're stuck in
|
repair loops, or because the scheduler can't give them work.
|
|
--logdir <directory>
|
Log progress and actions in a file under this directory. Text
|
of any e-mail sent will also be logged in a timestamped file in
|
this directory.
|
|
--debug
|
Suppress all logging, metrics reporting, and sending e-mail.
|
Instead, write the output that would be generated onto stdout.
|
|
<model> arguments:
|
With no arguments, gathers the status for all models in the lab.
|
With one or more named models on the command line, restricts
|
reporting to just those models.
|
"""
|
|
|
import argparse
|
import collections
|
import logging
|
import logging.handlers
|
import os
|
import re
|
import sys
|
import time
|
|
import common
|
from autotest_lib.client.bin import utils
|
from autotest_lib.client.common_lib import time_utils
|
from autotest_lib.frontend.afe.json_rpc import proxy
|
from autotest_lib.server import constants
|
from autotest_lib.server import site_utils
|
from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
|
from autotest_lib.server.hosts import servo_host
|
from autotest_lib.server.lib import status_history
|
from autotest_lib.site_utils import gmail_lib
|
from chromite.lib import metrics
|
|
|
CRITICAL_POOLS = constants.Pools.CRITICAL_POOLS
|
SPARE_POOL = constants.Pools.SPARE_POOL
|
MANAGED_POOLS = constants.Pools.MANAGED_POOLS
|
|
# _EXCLUDED_LABELS - A set of labels that disqualify a DUT from
|
# monitoring by this script. Currently, we're excluding these:
|
# + 'adb' - We're not ready to monitor Android or Brillo hosts.
|
# + 'board:guado_moblab' - These are maintained by a separate
|
# process that doesn't use this script.
|
# + 'board:veyron_rialto' due to crbug.com/854404
|
|
_EXCLUDED_LABELS = {'adb', 'board:guado_moblab',
|
'board:veyron_rialto'}
|
|
# _DEFAULT_DURATION:
|
# Default value used for the --duration command line option.
|
# Specifies how far back in time to search in order to determine
|
# DUT status.
|
|
_DEFAULT_DURATION = 24
|
|
# _LOGDIR:
|
# Relative path used in the calculation of the default setting for
|
# the --logdir option. The full path is relative to the root of the
|
# autotest directory, as determined from sys.argv[0].
|
# _LOGFILE:
|
# Basename of a file to which general log information will be
|
# written.
|
# _LOG_FORMAT:
|
# Format string for log messages.
|
|
_LOGDIR = os.path.join('logs', 'dut-data')
|
_LOGFILE = 'lab-inventory.log'
|
_LOG_FORMAT = '%(asctime)s | %(levelname)-10s | %(message)s'
|
|
# Pattern describing location-based host names in the Chrome OS test
|
# labs. Each DUT hostname designates the DUT's location:
|
# * A lab (room) that's physically separated from other labs
|
# (i.e. there's a door).
|
# * A row (or aisle) of DUTs within the lab.
|
# * A vertical rack of shelves on the row.
|
# * A specific host on one shelf of the rack.
|
|
_HOSTNAME_PATTERN = re.compile(
|
r'(chromeos\d+)-row(\d+)-rack(\d+)-host(\d+)')
|
|
# _REPAIR_LOOP_THRESHOLD:
|
# The number of repeated Repair tasks that must be seen to declare
|
# that a DUT is stuck in a repair loop.
|
|
_REPAIR_LOOP_THRESHOLD = 4
|
|
|
_METRICS_PREFIX = 'chromeos/autotest/inventory'
|
_UNTESTABLE_PRESENCE_METRIC = metrics.BooleanMetric(
|
_METRICS_PREFIX + '/untestable',
|
'DUTs that cannot be scheduled for testing')
|
|
_MISSING_DUT_METRIC = metrics.Counter(
|
_METRICS_PREFIX + '/missing', 'DUTs which cannot be found by lookup queries'
|
' because they are invalid or deleted')
|
|
# _Diagnosis - namedtuple corresponding to the return value from
|
# `HostHistory.last_diagnosis()`
|
_Diagnosis = collections.namedtuple('_Diagnosis', ['status', 'task'])
|
|
def _get_diagnosis(history):
|
dut_present = True
|
try:
|
diagnosis = _Diagnosis(*history.last_diagnosis())
|
if (diagnosis.status == status_history.BROKEN
|
and diagnosis.task.end_time < history.start_time):
|
return _Diagnosis(status_history.UNUSED, diagnosis.task)
|
else:
|
return diagnosis
|
except proxy.JSONRPCException as e:
|
logging.warn(e)
|
dut_present = False
|
finally:
|
_MISSING_DUT_METRIC.increment(
|
fields={'host': history.hostname, 'presence': dut_present})
|
return _Diagnosis(None, None)
|
|
|
def _host_is_working(history):
|
return _get_diagnosis(history).status == status_history.WORKING
|
|
|
def _host_is_broken(history):
|
return _get_diagnosis(history).status == status_history.BROKEN
|
|
|
def _host_is_idle(history):
|
idle_statuses = {status_history.UNUSED, status_history.UNKNOWN}
|
return _get_diagnosis(history).status in idle_statuses
|
|
|
class _HostSetInventory(object):
|
"""Maintains a set of related `HostJobHistory` objects.
|
|
Current usage of this class is that all DUTs are part of a single
|
scheduling pool of DUTs for a single model; however, this class make
|
no assumptions about the actual relationship among the DUTs.
|
|
The collection is segregated into disjoint categories of "working",
|
"broken", and "idle" DUTs. Accessor methods allow finding both the
|
list of DUTs in each category, as well as counts of each category.
|
|
Performance note: Certain methods in this class are potentially
|
expensive:
|
* `get_working()`
|
* `get_working_list()`
|
* `get_broken()`
|
* `get_broken_list()`
|
* `get_idle()`
|
* `get_idle_list()`
|
The first time any one of these methods is called, it causes
|
multiple RPC calls with a relatively expensive set of database
|
queries. However, the results of the queries are cached in the
|
individual `HostJobHistory` objects, so only the first call
|
actually pays the full cost.
|
|
Additionally, `get_working_list()`, `get_broken_list()` and
|
`get_idle_list()` cache their return values to avoid recalculating
|
lists at every call; this caching is separate from the caching of
|
RPC results described above.
|
|
This class is deliberately constructed to delay the RPC cost until
|
the accessor methods are called (rather than to query in
|
`record_host()`) so that it's possible to construct a complete
|
`_LabInventory` without making the expensive queries at creation
|
time. `_populate_model_counts()`, below, assumes this behavior.
|
"""
|
|
def __init__(self):
|
self._histories = []
|
self._working_list = None
|
self._broken_list = None
|
self._idle_list = None
|
|
def record_host(self, host_history):
|
"""Add one `HostJobHistory` object to the collection.
|
|
@param host_history The `HostJobHistory` object to be
|
remembered.
|
"""
|
self._working_list = None
|
self._broken_list = None
|
self._idle_list = None
|
self._histories.append(host_history)
|
|
def get_working_list(self):
|
"""Return a list of all working DUTs in the pool.
|
|
Filter `self._histories` for histories where the DUT is
|
diagnosed as working.
|
|
Cache the result so that we only cacluate it once.
|
|
@return A list of HostJobHistory objects.
|
"""
|
if self._working_list is None:
|
self._working_list = [h for h in self._histories
|
if _host_is_working(h)]
|
return self._working_list
|
|
def get_working(self):
|
"""Return the number of working DUTs in the pool."""
|
return len(self.get_working_list())
|
|
def get_broken_list(self):
|
"""Return a list of all broken DUTs in the pool.
|
|
Filter `self._histories` for histories where the DUT is
|
diagnosed as broken.
|
|
Cache the result so that we only cacluate it once.
|
|
@return A list of HostJobHistory objects.
|
"""
|
if self._broken_list is None:
|
self._broken_list = [h for h in self._histories
|
if _host_is_broken(h)]
|
return self._broken_list
|
|
def get_broken(self):
|
"""Return the number of broken DUTs in the pool."""
|
return len(self.get_broken_list())
|
|
def get_idle_list(self):
|
"""Return a list of all idle DUTs in the pool.
|
|
Filter `self._histories` for histories where the DUT is
|
diagnosed as idle.
|
|
Cache the result so that we only cacluate it once.
|
|
@return A list of HostJobHistory objects.
|
"""
|
if self._idle_list is None:
|
self._idle_list = [h for h in self._histories
|
if _host_is_idle(h)]
|
return self._idle_list
|
|
def get_idle(self):
|
"""Return the number of idle DUTs in the pool."""
|
return len(self.get_idle_list())
|
|
def get_total(self):
|
"""Return the total number of DUTs in the pool."""
|
return len(self._histories)
|
|
def get_all_histories(self):
|
return self._histories
|
|
|
class _PoolSetInventory(object):
|
"""Maintains a set of `HostJobHistory`s for a set of pools.
|
|
The collection is segregated into disjoint categories of "working",
|
"broken", and "idle" DUTs. Accessor methods allow finding both the
|
list of DUTs in each category, as well as counts of each category.
|
Accessor queries can be for an individual pool, or against all
|
pools.
|
|
Performance note: This class relies on `_HostSetInventory`. Public
|
methods in this class generally rely on methods of the same name in
|
the underlying class, and so will have the same underlying
|
performance characteristics.
|
"""
|
|
def __init__(self, pools):
|
self._histories_by_pool = {
|
pool: _HostSetInventory() for pool in pools
|
}
|
|
def record_host(self, host_history):
|
"""Add one `HostJobHistory` object to the collection.
|
|
@param host_history The `HostJobHistory` object to be
|
remembered.
|
"""
|
pool = host_history.host_pool
|
self._histories_by_pool[pool].record_host(host_history)
|
|
def _count_pool(self, get_pool_count, pool=None):
|
"""Internal helper to count hosts in a given pool.
|
|
The `get_pool_count` parameter is a function to calculate
|
the exact count of interest for the pool.
|
|
@param get_pool_count Function to return a count from a
|
_PoolCount object.
|
@param pool The pool to be counted. If `None`,
|
return the total across all pools.
|
"""
|
if pool is None:
|
return sum([get_pool_count(cached_history) for cached_history in
|
self._histories_by_pool.values()])
|
else:
|
return get_pool_count(self._histories_by_pool[pool])
|
|
def get_working_list(self):
|
"""Return a list of all working DUTs (across all pools).
|
|
Go through all HostJobHistory objects across all pools,
|
selecting all DUTs identified as working.
|
|
@return A list of HostJobHistory objects.
|
"""
|
l = []
|
for p in self._histories_by_pool.values():
|
l.extend(p.get_working_list())
|
return l
|
|
def get_working(self, pool=None):
|
"""Return the number of working DUTs in a pool.
|
|
@param pool The pool to be counted. If `None`, return the
|
total across all pools.
|
|
@return The total number of working DUTs in the selected
|
pool(s).
|
"""
|
return self._count_pool(_HostSetInventory.get_working, pool)
|
|
def get_broken_list(self):
|
"""Return a list of all broken DUTs (across all pools).
|
|
Go through all HostJobHistory objects across all pools,
|
selecting all DUTs identified as broken.
|
|
@return A list of HostJobHistory objects.
|
"""
|
l = []
|
for p in self._histories_by_pool.values():
|
l.extend(p.get_broken_list())
|
return l
|
|
def get_broken(self, pool=None):
|
"""Return the number of broken DUTs in a pool.
|
|
@param pool The pool to be counted. If `None`, return the
|
total across all pools.
|
|
@return The total number of broken DUTs in the selected pool(s).
|
"""
|
return self._count_pool(_HostSetInventory.get_broken, pool)
|
|
def get_idle_list(self, pool=None):
|
"""Return a list of all idle DUTs in the given pool.
|
|
Go through all HostJobHistory objects across all pools,
|
selecting all DUTs identified as idle.
|
|
@param pool: The pool to be counted. If `None`, return the total list
|
across all pools.
|
|
@return A list of HostJobHistory objects.
|
"""
|
if pool is None:
|
l = []
|
for p in self._histories_by_pool.itervalues():
|
l.extend(p.get_idle_list())
|
return l
|
else:
|
return self._histories_by_pool[pool].get_idle_list()
|
|
def get_idle(self, pool=None):
|
"""Return the number of idle DUTs in a pool.
|
|
@param pool: The pool to be counted. If `None`, return the total
|
across all pools.
|
|
@return The total number of idle DUTs in the selected pool(s).
|
"""
|
return self._count_pool(_HostSetInventory.get_idle, pool)
|
|
def get_spares_buffer(self, spare_pool=SPARE_POOL):
|
"""Return the the nominal number of working spares.
|
|
Calculates and returns how many working spares there would
|
be in the spares pool if all broken DUTs were in the spares
|
pool. This number may be negative, indicating a shortfall
|
in the critical pools.
|
|
@return The total number DUTs in the spares pool, less the total
|
number of broken DUTs in all pools.
|
"""
|
return self.get_total(spare_pool) - self.get_broken()
|
|
def get_total(self, pool=None):
|
"""Return the total number of DUTs in a pool.
|
|
@param pool The pool to be counted. If `None`, return the
|
total across all pools.
|
|
@return The total number of DUTs in the selected pool(s).
|
"""
|
return self._count_pool(_HostSetInventory.get_total, pool)
|
|
def get_all_histories(self, pool=None):
|
if pool is None:
|
for p in self._histories_by_pool.itervalues():
|
for h in p.get_all_histories():
|
yield h
|
else:
|
for h in self._histories_by_pool[pool].get_all_histories():
|
yield h
|
|
|
def _is_migrated_to_skylab(afehost):
|
"""Return True if the provided frontend.Host has been migrated to skylab."""
|
return afehost.hostname.endswith('-migrated-do-not-use')
|
|
|
def _eligible_host(afehost):
|
"""Return whether this host is eligible for monitoring.
|
|
@param afehost The host to be tested for eligibility.
|
"""
|
if _is_migrated_to_skylab(afehost):
|
return False
|
|
# DUTs without an existing, unique 'model' or 'pool' label aren't meant to
|
# exist in the managed inventory; their presence generally indicates an
|
# error in the database. The _LabInventory constructor requires hosts to
|
# conform to the label restrictions. Failing an inventory run for a single
|
# bad entry is wrong, so we ignore these hosts.
|
models = [l for l in afehost.labels
|
if l.startswith(constants.Labels.MODEL_PREFIX)]
|
pools = [l for l in afehost.labels
|
if l.startswith(constants.Labels.POOL_PREFIX)]
|
excluded = _EXCLUDED_LABELS.intersection(afehost.labels)
|
return len(models) == 1 and len(pools) == 1 and not excluded
|
|
|
class _LabInventory(collections.Mapping):
|
"""Collection of `HostJobHistory` objects for the Lab's inventory.
|
|
This is a dict-like collection indexed by model. Indexing returns
|
the _PoolSetInventory object associated with the model.
|
"""
|
|
@classmethod
|
def create_inventory(cls, afe, start_time, end_time, modellist=[]):
|
"""Return a Lab inventory with specified parameters.
|
|
By default, gathers inventory from `HostJobHistory` objects for
|
all DUTs in the `MANAGED_POOLS` list. If `modellist` is
|
supplied, the inventory will be restricted to only the given
|
models.
|
|
@param afe AFE object for constructing the
|
`HostJobHistory` objects.
|
@param start_time Start time for the `HostJobHistory` objects.
|
@param end_time End time for the `HostJobHistory` objects.
|
@param modellist List of models to include. If empty,
|
include all available models.
|
@return A `_LabInventory` object for the specified models.
|
"""
|
target_pools = MANAGED_POOLS
|
label_list = [constants.Labels.POOL_PREFIX + l for l in target_pools]
|
afehosts = afe.get_hosts(labels__name__in=label_list)
|
if modellist:
|
# We're deliberately not checking host eligibility in this
|
# code path. This is a debug path, not used in production;
|
# it may be useful to include ineligible hosts here.
|
modelhosts = []
|
for model in modellist:
|
model_label = constants.Labels.MODEL_PREFIX + model
|
host_list = [h for h in afehosts
|
if model_label in h.labels]
|
modelhosts.extend(host_list)
|
afehosts = modelhosts
|
else:
|
afehosts = [h for h in afehosts if _eligible_host(h)]
|
create = lambda host: (
|
status_history.HostJobHistory(afe, host,
|
start_time, end_time))
|
return cls([create(host) for host in afehosts], target_pools)
|
|
def __init__(self, histories, pools):
|
models = {h.host_model for h in histories}
|
self._modeldata = {model: _PoolSetInventory(pools) for model in models}
|
self._dut_count = len(histories)
|
for h in histories:
|
self[h.host_model].record_host(h)
|
self._boards = {h.host_board for h in histories}
|
|
def __getitem__(self, key):
|
return self._modeldata.__getitem__(key)
|
|
def __len__(self):
|
return self._modeldata.__len__()
|
|
def __iter__(self):
|
return self._modeldata.__iter__()
|
|
def get_num_duts(self):
|
"""Return the total number of DUTs in the inventory."""
|
return self._dut_count
|
|
def get_num_models(self):
|
"""Return the total number of models in the inventory."""
|
return len(self)
|
|
def get_pool_models(self, pool):
|
"""Return all models in `pool`.
|
|
@param pool The pool to be inventoried for models.
|
"""
|
return {m for m, h in self.iteritems() if h.get_total(pool)}
|
|
def get_boards(self):
|
return self._boards
|
|
|
def _reportable_models(inventory, spare_pool=SPARE_POOL):
|
"""Iterate over all models subject to reporting.
|
|
Yields the contents of `inventory.iteritems()` filtered to include
|
only reportable models. A model is reportable if it has DUTs in
|
both `spare_pool` and at least one other pool.
|
|
@param spare_pool The spare pool to be tested for reporting.
|
"""
|
for model, poolset in inventory.iteritems():
|
spares = poolset.get_total(spare_pool)
|
total = poolset.get_total()
|
if spares != 0 and spares != total:
|
yield model, poolset
|
|
|
def _all_dut_histories(inventory):
|
for poolset in inventory.itervalues():
|
for h in poolset.get_all_histories():
|
yield h
|
|
|
def _sort_by_location(inventory_list):
|
"""Return a list of DUTs, organized by location.
|
|
Take the given list of `HostJobHistory` objects, separate it
|
into a list per lab, and sort each lab's list by location. The
|
order of sorting within a lab is
|
* By row number within the lab,
|
* then by rack number within the row,
|
* then by host shelf number within the rack.
|
|
Return a list of the sorted lists.
|
|
Implementation note: host locations are sorted by converting
|
each location into a base 100 number. If row, rack or
|
host numbers exceed the range [0..99], then sorting will
|
break down.
|
|
@return A list of sorted lists of DUTs.
|
"""
|
BASE = 100
|
lab_lists = {}
|
for history in inventory_list:
|
location = _HOSTNAME_PATTERN.match(history.host.hostname)
|
if location:
|
lab = location.group(1)
|
key = 0
|
for idx in location.group(2, 3, 4):
|
key = BASE * key + int(idx)
|
lab_lists.setdefault(lab, []).append((key, history))
|
return_list = []
|
for dut_list in lab_lists.values():
|
dut_list.sort(key=lambda t: t[0])
|
return_list.append([t[1] for t in dut_list])
|
return return_list
|
|
|
def _score_repair_set(buffer_counts, repair_list):
|
"""Return a numeric score rating a set of DUTs to be repaired.
|
|
`buffer_counts` is a dictionary mapping model names to the size of
|
the model's spares buffer.
|
|
`repair_list` is a list of `HostJobHistory` objects for the DUTs to
|
be repaired.
|
|
This function calculates the new set of buffer counts that would
|
result from the proposed repairs, and scores the new set using two
|
numbers:
|
* Worst case buffer count for any model (higher is better). This
|
is the more significant number for comparison.
|
* Number of models at the worst case (lower is better). This is
|
the less significant number.
|
|
Implementation note: The score could fail to reflect the intended
|
criteria if there are more than 1000 models in the inventory.
|
|
@param spare_counts A dictionary mapping models to buffer counts.
|
@param repair_list A list of `HostJobHistory` objects for the
|
DUTs to be repaired.
|
@return A numeric score.
|
"""
|
# Go through `buffer_counts`, and create a list of new counts
|
# that records the buffer count for each model after repair.
|
# The new list of counts discards the model names, as they don't
|
# contribute to the final score.
|
_NMODELS = 1000
|
pools = {h.host_pool for h in repair_list}
|
repair_inventory = _LabInventory(repair_list, pools)
|
new_counts = []
|
for m, c in buffer_counts.iteritems():
|
if m in repair_inventory:
|
newcount = repair_inventory[m].get_total()
|
else:
|
newcount = 0
|
new_counts.append(c + newcount)
|
# Go through the new list of counts. Find the worst available
|
# spares count, and count how many times that worst case occurs.
|
worst_count = new_counts[0]
|
num_worst = 1
|
for c in new_counts[1:]:
|
if c == worst_count:
|
num_worst += 1
|
elif c < worst_count:
|
worst_count = c
|
num_worst = 1
|
# Return the calculated score
|
return _NMODELS * worst_count - num_worst
|
|
|
def _generate_repair_recommendation(inventory, num_recommend):
|
"""Return a summary of selected DUTs needing repair.
|
|
Returns a message recommending a list of broken DUTs to be repaired.
|
The list of DUTs is selected based on these criteria:
|
* No more than `num_recommend` DUTs will be listed.
|
* All DUTs must be in the same lab.
|
* DUTs should be selected for some degree of physical proximity.
|
* DUTs for models with a low spares buffer are more important than
|
DUTs with larger buffers.
|
|
The algorithm used will guarantee that at least one DUT from a model
|
with the lowest spares buffer will be recommended. If the worst
|
spares buffer number is shared by more than one model, the algorithm
|
will tend to prefer repair sets that include more of those models
|
over sets that cover fewer models.
|
|
@param inventory `_LabInventory` object from which to generate
|
recommendations.
|
@param num_recommend Number of DUTs to recommend for repair.
|
"""
|
logging.debug('Creating DUT repair recommendations')
|
model_buffer_counts = {}
|
broken_list = []
|
for model, counts in _reportable_models(inventory):
|
logging.debug('Listing failed DUTs for %s', model)
|
if counts.get_broken() != 0:
|
model_buffer_counts[model] = counts.get_spares_buffer()
|
broken_list.extend(counts.get_broken_list())
|
# N.B. The logic inside this loop may seem complicated, but
|
# simplification is hard:
|
# * Calculating an initial recommendation outside of
|
# the loop likely would make things more complicated,
|
# not less.
|
# * It's necessary to calculate an initial lab slice once per
|
# lab _before_ the while loop, in case the number of broken
|
# DUTs in a lab is less than `num_recommend`.
|
recommendation = None
|
best_score = None
|
for lab_duts in _sort_by_location(broken_list):
|
start = 0
|
end = num_recommend
|
lab_slice = lab_duts[start : end]
|
lab_score = _score_repair_set(model_buffer_counts, lab_slice)
|
while end < len(lab_duts):
|
start += 1
|
end += 1
|
new_slice = lab_duts[start : end]
|
new_score = _score_repair_set(model_buffer_counts, new_slice)
|
if new_score > lab_score:
|
lab_slice = new_slice
|
lab_score = new_score
|
if recommendation is None or lab_score > best_score:
|
recommendation = lab_slice
|
best_score = lab_score
|
# N.B. The trailing space in `line_fmt` is manadatory: Without it,
|
# Gmail will parse the URL wrong. Don't ask. If you simply _must_
|
# know more, go try it yourself...
|
line_fmt = '%-30s %-16s %-6s\n %s '
|
message = ['Repair recommendations:\n',
|
line_fmt % ( 'Hostname', 'Model', 'Servo?', 'Logs URL')]
|
if recommendation:
|
for h in recommendation:
|
servo_name = servo_host.make_servo_hostname(h.host.hostname)
|
servo_present = utils.host_is_in_lab_zone(servo_name)
|
event = _get_diagnosis(h).task
|
line = line_fmt % (
|
h.host.hostname, h.host_model,
|
'Yes' if servo_present else 'No', event.job_url)
|
message.append(line)
|
else:
|
message.append('(No DUTs to repair)')
|
return '\n'.join(message)
|
|
|
def _generate_model_inventory_message(inventory):
|
"""Generate the "model inventory" e-mail message.
|
|
The model inventory is a list by model summarizing the number of
|
working, broken, and idle DUTs, and the total shortfall or surplus
|
of working devices relative to the minimum critical pool
|
requirement.
|
|
The report omits models with no DUTs in the spare pool or with no
|
DUTs in a critical pool.
|
|
N.B. For sample output text formattted as users can expect to
|
see it in e-mail and log files, refer to the unit tests.
|
|
@param inventory `_LabInventory` object to be reported on.
|
@return String with the inventory message to be sent.
|
"""
|
logging.debug('Creating model inventory')
|
nworking = 0
|
nbroken = 0
|
nidle = 0
|
nbroken_models = 0
|
ntotal_models = 0
|
summaries = []
|
column_names = (
|
'Model', 'Avail', 'Bad', 'Idle', 'Good', 'Spare', 'Total')
|
for model, counts in _reportable_models(inventory):
|
logging.debug('Counting %2d DUTS for model %s',
|
counts.get_total(), model)
|
# Summary elements laid out in the same order as the column
|
# headers:
|
# Model Avail Bad Idle Good Spare Total
|
# e[0] e[1] e[2] e[3] e[4] e[5] e[6]
|
element = (model,
|
counts.get_spares_buffer(),
|
counts.get_broken(),
|
counts.get_idle(),
|
counts.get_working(),
|
counts.get_total(SPARE_POOL),
|
counts.get_total())
|
if element[2]:
|
summaries.append(element)
|
nbroken_models += 1
|
ntotal_models += 1
|
nbroken += element[2]
|
nidle += element[3]
|
nworking += element[4]
|
ntotal = nworking + nbroken + nidle
|
summaries = sorted(summaries, key=lambda e: (e[1], -e[2]))
|
broken_percent = int(round(100.0 * nbroken / ntotal))
|
idle_percent = int(round(100.0 * nidle / ntotal))
|
working_percent = 100 - broken_percent - idle_percent
|
message = ['Summary of DUTs in inventory:',
|
'%10s %10s %10s %6s' % ('Bad', 'Idle', 'Good', 'Total'),
|
'%5d %3d%% %5d %3d%% %5d %3d%% %6d' % (
|
nbroken, broken_percent,
|
nidle, idle_percent,
|
nworking, working_percent,
|
ntotal),
|
'',
|
'Models with failures: %d' % nbroken_models,
|
'Models in inventory: %d' % ntotal_models,
|
'', '',
|
'Full model inventory:\n',
|
'%-22s %5s %5s %5s %5s %5s %5s' % column_names]
|
message.extend(
|
['%-22s %5d %5d %5d %5d %5d %5d' % e for e in summaries])
|
return '\n'.join(message)
|
|
|
_POOL_INVENTORY_HEADER = '''\
|
Notice to Infrastructure deputies: All models shown below are at
|
less than full strength, please take action to resolve the issues.
|
Once you're satisified that failures won't recur, failed DUTs can
|
be replaced with spares by running `balance_pool`. Detailed
|
instructions can be found here:
|
http://go/cros-manage-duts
|
'''
|
|
|
def _generate_pool_inventory_message(inventory):
|
"""Generate the "pool inventory" e-mail message.
|
|
The pool inventory is a list by pool and model summarizing the
|
number of working and broken DUTs in the pool. Only models with
|
at least one broken DUT are included in the list.
|
|
N.B. For sample output text formattted as users can expect to see it
|
in e-mail and log files, refer to the unit tests.
|
|
@param inventory `_LabInventory` object to be reported on.
|
@return String with the inventory message to be sent.
|
"""
|
logging.debug('Creating pool inventory')
|
message = [_POOL_INVENTORY_HEADER]
|
newline = ''
|
for pool in CRITICAL_POOLS:
|
message.append(
|
'%sStatus for pool:%s, by model:' % (newline, pool))
|
message.append(
|
'%-20s %5s %5s %5s %5s' % (
|
'Model', 'Bad', 'Idle', 'Good', 'Total'))
|
data_list = []
|
for model, counts in inventory.iteritems():
|
logging.debug('Counting %2d DUTs for %s, %s',
|
counts.get_total(pool), model, pool)
|
broken = counts.get_broken(pool)
|
idle = counts.get_idle(pool)
|
# models at full strength are not reported
|
if not broken and not idle:
|
continue
|
working = counts.get_working(pool)
|
total = counts.get_total(pool)
|
data_list.append((model, broken, idle, working, total))
|
if data_list:
|
data_list = sorted(data_list, key=lambda d: -d[1])
|
message.extend(
|
['%-20s %5d %5d %5d %5d' % t for t in data_list])
|
else:
|
message.append('(All models at full strength)')
|
newline = '\n'
|
return '\n'.join(message)
|
|
|
_IDLE_INVENTORY_HEADER = '''\
|
Notice to Infrastructure deputies: The hosts shown below haven't
|
run any jobs for at least 24 hours. Please check each host; locked
|
hosts should normally be unlocked; stuck jobs should normally be
|
aborted.
|
'''
|
|
|
def _generate_idle_inventory_message(inventory):
|
"""Generate the "idle inventory" e-mail message.
|
|
The idle inventory is a host list with corresponding pool and model,
|
where the hosts are identified as idle.
|
|
N.B. For sample output text format as users can expect to
|
see it in e-mail and log files, refer to the unit tests.
|
|
@param inventory `_LabInventory` object to be reported on.
|
@return String with the inventory message to be sent.
|
"""
|
logging.debug('Creating idle inventory')
|
message = [_IDLE_INVENTORY_HEADER]
|
message.append('Idle Host List:')
|
message.append('%-30s %-20s %s' % ('Hostname', 'Model', 'Pool'))
|
data_list = []
|
for pool in MANAGED_POOLS:
|
for model, counts in inventory.iteritems():
|
logging.debug('Counting %2d DUTs for %s, %s',
|
counts.get_total(pool), model, pool)
|
data_list.extend([(dut.host.hostname, model, pool)
|
for dut in counts.get_idle_list(pool)])
|
if data_list:
|
message.extend(['%-30s %-20s %s' % t for t in data_list])
|
else:
|
message.append('(No idle DUTs)')
|
return '\n'.join(message)
|
|
|
def _send_email(arguments, tag, subject, recipients, body):
|
"""Send an inventory e-mail message.
|
|
The message is logged in the selected log directory using `tag` for
|
the file name.
|
|
If the --debug option was requested, the message is neither logged
|
nor sent, but merely printed on stdout.
|
|
@param arguments Parsed command-line options.
|
@param tag Tag identifying the inventory for logging
|
purposes.
|
@param subject E-mail Subject: header line.
|
@param recipients E-mail addresses for the To: header line.
|
@param body E-mail message body.
|
"""
|
logging.debug('Generating email: "%s"', subject)
|
all_recipients = ', '.join(recipients)
|
report_body = '\n'.join([
|
'To: %s' % all_recipients,
|
'Subject: %s' % subject,
|
'', body, ''])
|
if arguments.debug:
|
print report_body
|
else:
|
filename = os.path.join(arguments.logdir, tag)
|
try:
|
report_file = open(filename, 'w')
|
report_file.write(report_body)
|
report_file.close()
|
except EnvironmentError as e:
|
logging.error('Failed to write %s: %s', filename, e)
|
try:
|
gmail_lib.send_email(all_recipients, subject, body)
|
except Exception as e:
|
logging.error('Failed to send e-mail to %s: %s',
|
all_recipients, e)
|
|
|
def _populate_model_counts(inventory):
|
"""Gather model counts while providing interactive feedback.
|
|
Gathering the status of all individual DUTs in the lab can take
|
considerable time (~30 minutes at the time of this writing).
|
Normally, we pay that cost by querying as we go. However, with
|
the `--debug` option, we expect a human being to be watching the
|
progress in real time. So, we force the first (expensive) queries
|
to happen up front, and provide simple ASCII output on sys.stdout
|
to show a progress bar and results.
|
|
@param inventory `_LabInventory` object from which to gather
|
counts.
|
"""
|
n = 0
|
total_broken = 0
|
for counts in inventory.itervalues():
|
n += 1
|
if n % 10 == 5:
|
c = '+'
|
elif n % 10 == 0:
|
c = '%d' % ((n / 10) % 10)
|
else:
|
c = '.'
|
sys.stdout.write(c)
|
sys.stdout.flush()
|
# This next call is where all the time goes - it forces all of a
|
# model's `HostJobHistory` objects to query the database and
|
# cache their results.
|
total_broken += counts.get_broken()
|
sys.stdout.write('\n')
|
sys.stdout.write('Found %d broken DUTs\n' % total_broken)
|
|
|
def _perform_model_inventory(arguments, inventory, timestamp):
|
"""Perform the model inventory report.
|
|
The model inventory report consists of the following:
|
* A list of DUTs that are recommended to be repaired. This list
|
is optional, and only appears if the `--recommend` option is
|
present.
|
* A list of all models that have failed DUTs, with counts
|
of working, broken, and spare DUTs, among others.
|
|
@param arguments Command-line arguments as returned by
|
`ArgumentParser`
|
@param inventory `_LabInventory` object to be reported on.
|
@param timestamp A string used to identify this run's timestamp
|
in logs and email output.
|
"""
|
if arguments.recommend:
|
recommend_message = _generate_repair_recommendation(
|
inventory, arguments.recommend) + '\n\n\n'
|
else:
|
recommend_message = ''
|
model_message = _generate_model_inventory_message(inventory)
|
_send_email(arguments,
|
'models-%s.txt' % timestamp,
|
'DUT model inventory %s' % timestamp,
|
arguments.model_notify,
|
recommend_message + model_message)
|
|
|
def _perform_pool_inventory(arguments, inventory, timestamp):
|
"""Perform the pool inventory report.
|
|
The pool inventory report consists of the following:
|
* A list of all critical pools that have failed DUTs, with counts
|
of working, broken, and idle DUTs.
|
* A list of all idle DUTs by hostname including the model and
|
pool.
|
|
@param arguments Command-line arguments as returned by
|
`ArgumentParser`
|
@param inventory `_LabInventory` object to be reported on.
|
@param timestamp A string used to identify this run's timestamp in
|
logs and email output.
|
"""
|
pool_message = _generate_pool_inventory_message(inventory)
|
idle_message = _generate_idle_inventory_message(inventory)
|
_send_email(arguments,
|
'pools-%s.txt' % timestamp,
|
'DUT pool inventory %s' % timestamp,
|
arguments.pool_notify,
|
pool_message + '\n\n\n' + idle_message)
|
|
|
def _dut_in_repair_loop(history):
|
"""Return whether a DUT's history indicates a repair loop.
|
|
A DUT is considered looping if it runs no tests, and no tasks pass
|
other than repair tasks.
|
|
@param history An instance of `status_history.HostJobHistory` to be
|
scanned for a repair loop. The caller guarantees
|
that this history corresponds to a working DUT.
|
@returns Return a true value if the DUT's most recent history
|
indicates a repair loop.
|
"""
|
# Our caller passes only histories for working DUTs; that means
|
# we've already paid the cost of fetching the diagnosis task, and
|
# we know that the task was successful. The diagnosis task will be
|
# one of the tasks we must scan to find a loop, so if the task isn't
|
# a repair task, then our history includes a successful non-repair
|
# task, and we're not looping.
|
#
|
# The for loop below is very expensive, because it must fetch the
|
# full history, regardless of how many tasks we examine. At the
|
# time of this writing, this check against the diagnosis task
|
# reduces the cost of finding loops in the full inventory from hours
|
# to minutes.
|
if _get_diagnosis(history).task.name != 'Repair':
|
return False
|
repair_ok_count = 0
|
for task in history:
|
if not task.is_special:
|
# This is a test, so we're not looping.
|
return False
|
if task.diagnosis == status_history.BROKEN:
|
# Failed a repair, so we're not looping.
|
return False
|
if (task.diagnosis == status_history.WORKING
|
and task.name != 'Repair'):
|
# Non-repair task succeeded, so we're not looping.
|
return False
|
# At this point, we have either a failed non-repair task, or
|
# a successful repair.
|
if task.name == 'Repair':
|
repair_ok_count += 1
|
if repair_ok_count >= _REPAIR_LOOP_THRESHOLD:
|
return True
|
|
|
def _report_untestable_dut(history, state):
|
fields = {
|
'dut_hostname': history.hostname,
|
'model': history.host_model,
|
'pool': history.host_pool,
|
'state': state,
|
}
|
logging.info('DUT in state %(state)s: %(dut_hostname)s, '
|
'model: %(model)s, pool: %(pool)s', fields)
|
_UNTESTABLE_PRESENCE_METRIC.set(True, fields=fields)
|
|
|
def _report_untestable_dut_metrics(inventory):
|
"""Scan the inventory for DUTs unable to run tests.
|
|
DUTs in the inventory are judged "untestable" if they meet one of
|
two criteria:
|
* The DUT is stuck in a repair loop; that is, it regularly passes
|
repair, but never passes other operations.
|
* The DUT runs no tasks at all, but is not locked.
|
|
This routine walks through the given inventory looking for DUTs in
|
either of these states. Results are reported via a Monarch presence
|
metric.
|
|
Note: To make sure that DUTs aren't flagged as "idle" merely
|
because there's no work, a separate job runs prior to regular
|
inventory runs which schedules trivial work on any DUT that appears
|
idle.
|
|
@param inventory `_LabInventory` object to be reported on.
|
"""
|
logging.info('Scanning for untestable DUTs.')
|
for history in _all_dut_histories(inventory):
|
# Managed DUTs with names that don't match
|
# _HOSTNAME_PATTERN shouldn't be possible. However, we
|
# don't want arbitrary strings being attached to the
|
# 'dut_hostname' field, so for safety, we exclude all
|
# anomalies.
|
if not _HOSTNAME_PATTERN.match(history.hostname):
|
continue
|
if _host_is_working(history):
|
if _dut_in_repair_loop(history):
|
_report_untestable_dut(history, 'repair_loop')
|
elif _host_is_idle(history):
|
if not history.host.locked:
|
_report_untestable_dut(history, 'idle_unlocked')
|
|
|
def _log_startup(arguments, startup_time):
|
"""Log the start of this inventory run.
|
|
Print various log messages indicating the start of the run. Return
|
a string based on `startup_time` that will be used to identify this
|
run in log files and e-mail messages.
|
|
@param startup_time A UNIX timestamp marking the moment when
|
this inventory run began.
|
@returns A timestamp string that will be used to identify this run
|
in logs and email output.
|
"""
|
timestamp = time.strftime('%Y-%m-%d.%H',
|
time.localtime(startup_time))
|
logging.debug('Starting lab inventory for %s', timestamp)
|
if arguments.model_notify:
|
if arguments.recommend:
|
logging.debug('Will include repair recommendations')
|
logging.debug('Will include model inventory')
|
if arguments.pool_notify:
|
logging.debug('Will include pool inventory')
|
return timestamp
|
|
|
def _create_inventory(arguments, end_time):
|
"""Create the `_LabInventory` instance to use for reporting.
|
|
@param end_time A UNIX timestamp for the end of the time range
|
to be searched in this inventory run.
|
"""
|
start_time = end_time - arguments.duration * 60 * 60
|
afe = frontend_wrappers.RetryingAFE(server=None)
|
inventory = _LabInventory.create_inventory(
|
afe, start_time, end_time, arguments.modelnames)
|
logging.info('Found %d hosts across %d models',
|
inventory.get_num_duts(),
|
inventory.get_num_models())
|
return inventory
|
|
|
def _perform_inventory_reports(arguments):
|
"""Perform all inventory checks requested on the command line.
|
|
Create the initial inventory and run through the inventory reports
|
as called for by the parsed command-line arguments.
|
|
@param arguments Command-line arguments as returned by
|
`ArgumentParser`.
|
"""
|
startup_time = time.time()
|
timestamp = _log_startup(arguments, startup_time)
|
inventory = _create_inventory(arguments, startup_time)
|
if arguments.debug:
|
_populate_model_counts(inventory)
|
if arguments.model_notify:
|
_perform_model_inventory(arguments, inventory, timestamp)
|
if arguments.pool_notify:
|
_perform_pool_inventory(arguments, inventory, timestamp)
|
if arguments.report_untestable:
|
_report_untestable_dut_metrics(inventory)
|
|
|
def _separate_email_addresses(address_list):
|
"""Parse a list of comma-separated lists of e-mail addresses.
|
|
@param address_list A list of strings containing comma
|
separate e-mail addresses.
|
@return A list of the individual e-mail addresses.
|
"""
|
newlist = []
|
for arg in address_list:
|
newlist.extend([email.strip() for email in arg.split(',')])
|
return newlist
|
|
|
def _verify_arguments(arguments):
|
"""Validate command-line arguments.
|
|
Join comma separated e-mail addresses for `--model-notify` and
|
`--pool-notify` in separate option arguments into a single list.
|
|
For non-debug uses, require that at least one inventory report be
|
requested. For debug, if a report isn't specified, treat it as "run
|
all the reports."
|
|
The return value indicates success or failure; in the case of
|
failure, we also write an error message to stderr.
|
|
@param arguments Command-line arguments as returned by
|
`ArgumentParser`
|
@return True if the arguments are semantically good, or False
|
if the arguments don't meet requirements.
|
"""
|
arguments.model_notify = _separate_email_addresses(
|
arguments.model_notify)
|
arguments.pool_notify = _separate_email_addresses(
|
arguments.pool_notify)
|
if not any([arguments.model_notify, arguments.pool_notify,
|
arguments.report_untestable]):
|
if not arguments.debug:
|
sys.stderr.write('Must request at least one report via '
|
'--model-notify, --pool-notify, or '
|
'--report-untestable\n')
|
return False
|
else:
|
# We want to run all the e-mail reports. An empty notify
|
# list will cause a report to be skipped, so make sure the
|
# lists are non-empty.
|
arguments.model_notify = ['']
|
arguments.pool_notify = ['']
|
return True
|
|
|
def _get_default_logdir(script):
|
"""Get the default directory for the `--logdir` option.
|
|
The default log directory is based on the parent directory
|
containing this script.
|
|
@param script Path to this script file.
|
@return A path to a directory.
|
"""
|
basedir = os.path.dirname(os.path.abspath(script))
|
basedir = os.path.dirname(basedir)
|
return os.path.join(basedir, _LOGDIR)
|
|
|
def _parse_command(argv):
|
"""Parse the command line arguments.
|
|
Create an argument parser for this command's syntax, parse the
|
command line, and return the result of the ArgumentParser
|
parse_args() method.
|
|
@param argv Standard command line argument vector; argv[0] is
|
assumed to be the command name.
|
@return Result returned by ArgumentParser.parse_args().
|
"""
|
parser = argparse.ArgumentParser(
|
prog=argv[0],
|
description='Gather and report lab inventory statistics')
|
parser.add_argument('-d', '--duration', type=int,
|
default=_DEFAULT_DURATION, metavar='HOURS',
|
help='number of hours back to search for status'
|
' (default: %d)' % _DEFAULT_DURATION)
|
parser.add_argument('--model-notify', action='append',
|
default=[], metavar='ADDRESS',
|
help='Generate model inventory message, '
|
'and send it to the given e-mail address(es)')
|
parser.add_argument('--pool-notify', action='append',
|
default=[], metavar='ADDRESS',
|
help='Generate pool inventory message, '
|
'and send it to the given address(es)')
|
parser.add_argument('-r', '--recommend', type=int, default=None,
|
help=('Specify how many DUTs should be '
|
'recommended for repair (default: no '
|
'recommendation)'))
|
parser.add_argument('--report-untestable', action='store_true',
|
help='Check for devices unable to run tests.')
|
parser.add_argument('--debug', action='store_true',
|
help='Print e-mail, metrics messages on stdout '
|
'without sending them.')
|
parser.add_argument('--no-metrics', action='store_false',
|
dest='use_metrics',
|
help='Suppress generation of Monarch metrics.')
|
parser.add_argument('--logdir', default=_get_default_logdir(argv[0]),
|
help='Directory where logs will be written.')
|
parser.add_argument('modelnames', nargs='*',
|
metavar='MODEL',
|
help='names of models to report on '
|
'(default: all models)')
|
arguments = parser.parse_args(argv[1:])
|
if not _verify_arguments(arguments):
|
return None
|
return arguments
|
|
|
def _configure_logging(arguments):
|
"""Configure the `logging` module for our needs.
|
|
How we log depends on whether the `--debug` option was provided on
|
the command line.
|
* Without the option, we configure the logging to capture all
|
potentially relevant events in a log file. The log file is
|
configured to rotate once a week on Friday evening, preserving
|
~3 months worth of history.
|
* With the option, we expect stdout to contain other
|
human-readable output (including the contents of the e-mail
|
messages), so we restrict the output to INFO level.
|
|
For convenience, when `--debug` is on, the logging format has
|
no adornments, so that a call like `logging.info(msg)` simply writes
|
`msg` to stdout, plus a trailing newline.
|
|
@param arguments Command-line arguments as returned by
|
`ArgumentParser`
|
"""
|
root_logger = logging.getLogger()
|
if arguments.debug:
|
root_logger.setLevel(logging.INFO)
|
handler = logging.StreamHandler(sys.stdout)
|
handler.setFormatter(logging.Formatter())
|
else:
|
if not os.path.exists(arguments.logdir):
|
os.mkdir(arguments.logdir)
|
root_logger.setLevel(logging.DEBUG)
|
logfile = os.path.join(arguments.logdir, _LOGFILE)
|
handler = logging.handlers.TimedRotatingFileHandler(
|
logfile, when='W4', backupCount=13)
|
formatter = logging.Formatter(_LOG_FORMAT,
|
time_utils.TIME_FMT)
|
handler.setFormatter(formatter)
|
# TODO(jrbarnette) This is gross. Importing client.bin.utils
|
# implicitly imported logging_config, which calls
|
# logging.basicConfig() *at module level*. That gives us an
|
# extra logging handler that we don't want. So, clear out all
|
# the handlers here.
|
for h in root_logger.handlers:
|
root_logger.removeHandler(h)
|
root_logger.addHandler(handler)
|
|
|
def main(argv):
|
"""Standard main routine.
|
|
@param argv Command line arguments, including `sys.argv[0]`.
|
"""
|
arguments = _parse_command(argv)
|
if not arguments:
|
sys.exit(1)
|
_configure_logging(arguments)
|
|
try:
|
if arguments.use_metrics:
|
if arguments.debug:
|
logging.info('Debug mode: Will not report metrics to monarch.')
|
metrics_file = '/dev/null'
|
else:
|
metrics_file = None
|
with site_utils.SetupTsMonGlobalState(
|
'lab_inventory', debug_file=metrics_file,
|
auto_flush=False):
|
success = False
|
try:
|
with metrics.SecondsTimer('%s/duration' % _METRICS_PREFIX):
|
_perform_inventory_reports(arguments)
|
success = True
|
finally:
|
metrics.Counter('%s/tick' % _METRICS_PREFIX).increment(
|
fields={'success': success})
|
metrics.Flush()
|
else:
|
_perform_inventory_reports(arguments)
|
except KeyboardInterrupt:
|
pass
|
except Exception:
|
# Our cron setup doesn't preserve stderr, so drop extra breadcrumbs.
|
logging.exception('Error escaped main')
|
raise
|
|
|
def get_inventory(afe):
|
end_time = int(time.time())
|
start_time = end_time - 24 * 60 * 60
|
return _LabInventory.create_inventory(afe, start_time, end_time)
|
|
|
def get_managed_boards(afe):
|
return get_inventory(afe).get_boards()
|
|
|
if __name__ == '__main__':
|
main(sys.argv)
|
