# -*- coding:utf-8 -*-
|
# Copyright 2016 The Android Open Source Project
|
#
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
# you may not use this file except in compliance with the License.
|
# You may obtain a copy of the License at
|
#
|
# http://www.apache.org/licenses/LICENSE-2.0
|
#
|
# Unless required by applicable law or agreed to in writing, software
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# See the License for the specific language governing permissions and
|
# limitations under the License.
|
|
"""Various utility functions."""
|
|
from __future__ import print_function
|
|
import errno
|
import functools
|
import os
|
import signal
|
import subprocess
|
import sys
|
import tempfile
|
import time
|
|
_path = os.path.realpath(__file__ + '/../..')
|
if sys.path[0] != _path:
|
sys.path.insert(0, _path)
|
del _path
|
|
# pylint: disable=wrong-import-position
|
import rh.shell
|
import rh.signals
|
|
|
class CommandResult(object):
|
"""An object to store various attributes of a child process."""
|
|
def __init__(self, cmd=None, error=None, output=None, returncode=None):
|
self.cmd = cmd
|
self.error = error
|
self.output = output
|
self.returncode = returncode
|
|
@property
|
def cmdstr(self):
|
"""Return self.cmd as a nicely formatted string (useful for logs)."""
|
return rh.shell.cmd_to_str(self.cmd)
|
|
|
class RunCommandError(Exception):
|
"""Error caught in RunCommand() method."""
|
|
def __init__(self, msg, result, exception=None):
|
self.msg, self.result, self.exception = msg, result, exception
|
if exception is not None and not isinstance(exception, Exception):
|
raise ValueError('exception must be an exception instance; got %r'
|
% (exception,))
|
Exception.__init__(self, msg)
|
self.args = (msg, result, exception)
|
|
def stringify(self, error=True, output=True):
|
"""Custom method for controlling what is included in stringifying this.
|
|
Each individual argument is the literal name of an attribute
|
on the result object; if False, that value is ignored for adding
|
to this string content. If true, it'll be incorporated.
|
|
Args:
|
error: See comment about individual arguments above.
|
output: See comment about individual arguments above.
|
"""
|
items = [
|
'return code: %s; command: %s' % (
|
self.result.returncode, self.result.cmdstr),
|
]
|
if error and self.result.error:
|
items.append(self.result.error)
|
if output and self.result.output:
|
items.append(self.result.output)
|
if self.msg:
|
items.append(self.msg)
|
return '\n'.join(items)
|
|
def __str__(self):
|
# __str__ needs to return ascii, thus force a conversion to be safe.
|
return self.stringify().decode('utf-8', 'replace').encode(
|
'ascii', 'xmlcharrefreplace')
|
|
def __eq__(self, other):
|
return (type(self) == type(other) and
|
self.args == other.args)
|
|
def __ne__(self, other):
|
return not self.__eq__(other)
|
|
|
class TerminateRunCommandError(RunCommandError):
|
"""We were signaled to shutdown while running a command.
|
|
Client code shouldn't generally know, nor care about this class. It's
|
used internally to suppress retry attempts when we're signaled to die.
|
"""
|
|
|
def sudo_run_command(cmd, user='root', **kwargs):
|
"""Run a command via sudo.
|
|
Client code must use this rather than coming up with their own RunCommand
|
invocation that jams sudo in- this function is used to enforce certain
|
rules in our code about sudo usage, and as a potential auditing point.
|
|
Args:
|
cmd: The command to run. See RunCommand for rules of this argument-
|
SudoRunCommand purely prefixes it with sudo.
|
user: The user to run the command as.
|
kwargs: See RunCommand options, it's a direct pass thru to it.
|
Note that this supports a 'strict' keyword that defaults to True.
|
If set to False, it'll suppress strict sudo behavior.
|
|
Returns:
|
See RunCommand documentation.
|
|
Raises:
|
This function may immediately raise RunCommandError if we're operating
|
in a strict sudo context and the API is being misused.
|
Barring that, see RunCommand's documentation- it can raise the same things
|
RunCommand does.
|
"""
|
sudo_cmd = ['sudo']
|
|
if user == 'root' and os.geteuid() == 0:
|
return run_command(cmd, **kwargs)
|
|
if user != 'root':
|
sudo_cmd += ['-u', user]
|
|
# Pass these values down into the sudo environment, since sudo will
|
# just strip them normally.
|
extra_env = kwargs.pop('extra_env', None)
|
extra_env = {} if extra_env is None else extra_env.copy()
|
|
sudo_cmd.extend('%s=%s' % (k, v) for k, v in extra_env.iteritems())
|
|
# Finally, block people from passing options to sudo.
|
sudo_cmd.append('--')
|
|
if isinstance(cmd, basestring):
|
# We need to handle shell ourselves so the order is correct:
|
# $ sudo [sudo args] -- bash -c '[shell command]'
|
# If we let RunCommand take care of it, we'd end up with:
|
# $ bash -c 'sudo [sudo args] -- [shell command]'
|
shell = kwargs.pop('shell', False)
|
if not shell:
|
raise Exception('Cannot run a string command without a shell')
|
sudo_cmd.extend(['/bin/bash', '-c', cmd])
|
else:
|
sudo_cmd.extend(cmd)
|
|
return run_command(sudo_cmd, **kwargs)
|
|
|
def _kill_child_process(proc, int_timeout, kill_timeout, cmd, original_handler,
|
signum, frame):
|
"""Used as a signal handler by RunCommand.
|
|
This is internal to Runcommand. No other code should use this.
|
"""
|
if signum:
|
# If we've been invoked because of a signal, ignore delivery of that
|
# signal from this point forward. The invoking context of this func
|
# restores signal delivery to what it was prior; we suppress future
|
# delivery till then since this code handles SIGINT/SIGTERM fully
|
# including delivering the signal to the original handler on the way
|
# out.
|
signal.signal(signum, signal.SIG_IGN)
|
|
# Do not trust Popen's returncode alone; we can be invoked from contexts
|
# where the Popen instance was created, but no process was generated.
|
if proc.returncode is None and proc.pid is not None:
|
try:
|
while proc.poll() is None and int_timeout >= 0:
|
time.sleep(0.1)
|
int_timeout -= 0.1
|
|
proc.terminate()
|
while proc.poll() is None and kill_timeout >= 0:
|
time.sleep(0.1)
|
kill_timeout -= 0.1
|
|
if proc.poll() is None:
|
# Still doesn't want to die. Too bad, so sad, time to die.
|
proc.kill()
|
except EnvironmentError as e:
|
print('Ignoring unhandled exception in _kill_child_process: %s' % e,
|
file=sys.stderr)
|
|
# Ensure our child process has been reaped.
|
proc.wait()
|
|
if not rh.signals.relay_signal(original_handler, signum, frame):
|
# Mock up our own, matching exit code for signaling.
|
cmd_result = CommandResult(cmd=cmd, returncode=signum << 8)
|
raise TerminateRunCommandError('Received signal %i' % signum,
|
cmd_result)
|
|
|
class _Popen(subprocess.Popen):
|
"""subprocess.Popen derivative customized for our usage.
|
|
Specifically, we fix terminate/send_signal/kill to work if the child process
|
was a setuid binary; on vanilla kernels, the parent can wax the child
|
regardless, on goobuntu this apparently isn't allowed, thus we fall back
|
to the sudo machinery we have.
|
|
While we're overriding send_signal, we also suppress ESRCH being raised
|
if the process has exited, and suppress signaling all together if the
|
process has knowingly been waitpid'd already.
|
"""
|
|
def send_signal(self, signum):
|
if self.returncode is not None:
|
# The original implementation in Popen allows signaling whatever
|
# process now occupies this pid, even if the Popen object had
|
# waitpid'd. Since we can escalate to sudo kill, we do not want
|
# to allow that. Fixing this addresses that angle, and makes the
|
# API less sucky in the process.
|
return
|
|
try:
|
os.kill(self.pid, signum)
|
except EnvironmentError as e:
|
if e.errno == errno.EPERM:
|
# Kill returns either 0 (signal delivered), or 1 (signal wasn't
|
# delivered). This isn't particularly informative, but we still
|
# need that info to decide what to do, thus error_code_ok=True.
|
ret = sudo_run_command(['kill', '-%i' % signum, str(self.pid)],
|
redirect_stdout=True,
|
redirect_stderr=True, error_code_ok=True)
|
if ret.returncode == 1:
|
# The kill binary doesn't distinguish between permission
|
# denied and the pid is missing. Denied can only occur
|
# under weird grsec/selinux policies. We ignore that
|
# potential and just assume the pid was already dead and
|
# try to reap it.
|
self.poll()
|
elif e.errno == errno.ESRCH:
|
# Since we know the process is dead, reap it now.
|
# Normally Popen would throw this error- we suppress it since
|
# frankly that's a misfeature and we're already overriding
|
# this method.
|
self.poll()
|
else:
|
raise
|
|
|
# pylint: disable=redefined-builtin
|
def run_command(cmd, error_message=None, redirect_stdout=False,
|
redirect_stderr=False, cwd=None, input=None,
|
shell=False, env=None, extra_env=None, ignore_sigint=False,
|
combine_stdout_stderr=False, log_stdout_to_file=None,
|
error_code_ok=False, int_timeout=1, kill_timeout=1,
|
stdout_to_pipe=False, capture_output=False,
|
quiet=False, close_fds=True):
|
"""Runs a command.
|
|
Args:
|
cmd: cmd to run. Should be input to subprocess.Popen. If a string, shell
|
must be true. Otherwise the command must be an array of arguments,
|
and shell must be false.
|
error_message: Prints out this message when an error occurs.
|
redirect_stdout: Returns the stdout.
|
redirect_stderr: Holds stderr output until input is communicated.
|
cwd: The working directory to run this cmd.
|
input: The data to pipe into this command through stdin. If a file object
|
or file descriptor, stdin will be connected directly to that.
|
shell: Controls whether we add a shell as a command interpreter. See cmd
|
since it has to agree as to the type.
|
env: If non-None, this is the environment for the new process.
|
extra_env: If set, this is added to the environment for the new process.
|
This dictionary is not used to clear any entries though.
|
ignore_sigint: If True, we'll ignore signal.SIGINT before calling the
|
child. This is the desired behavior if we know our child will handle
|
Ctrl-C. If we don't do this, I think we and the child will both get
|
Ctrl-C at the same time, which means we'll forcefully kill the child.
|
combine_stdout_stderr: Combines stdout and stderr streams into stdout.
|
log_stdout_to_file: If set, redirects stdout to file specified by this
|
path. If |combine_stdout_stderr| is set to True, then stderr will
|
also be logged to the specified file.
|
error_code_ok: Does not raise an exception when command returns a non-zero
|
exit code. Instead, returns the CommandResult object containing the
|
exit code.
|
int_timeout: If we're interrupted, how long (in seconds) should we give
|
the invoked process to clean up before we send a SIGTERM.
|
kill_timeout: If we're interrupted, how long (in seconds) should we give
|
the invoked process to shutdown from a SIGTERM before we SIGKILL it.
|
stdout_to_pipe: Redirect stdout to pipe.
|
capture_output: Set |redirect_stdout| and |redirect_stderr| to True.
|
quiet: Set |stdout_to_pipe| and |combine_stdout_stderr| to True.
|
close_fds: Whether to close all fds before running |cmd|.
|
|
Returns:
|
A CommandResult object.
|
|
Raises:
|
RunCommandError: Raises exception on error with optional error_message.
|
"""
|
if capture_output:
|
redirect_stdout, redirect_stderr = True, True
|
|
if quiet:
|
stdout_to_pipe, combine_stdout_stderr = True, True
|
|
# Set default for variables.
|
stdout = None
|
stderr = None
|
stdin = None
|
cmd_result = CommandResult()
|
|
# Force the timeout to float; in the process, if it's not convertible,
|
# a self-explanatory exception will be thrown.
|
kill_timeout = float(kill_timeout)
|
|
def _get_tempfile():
|
try:
|
return tempfile.TemporaryFile(bufsize=0)
|
except EnvironmentError as e:
|
if e.errno != errno.ENOENT:
|
raise
|
# This can occur if we were pointed at a specific location for our
|
# TMP, but that location has since been deleted. Suppress that
|
# issue in this particular case since our usage gurantees deletion,
|
# and since this is primarily triggered during hard cgroups
|
# shutdown.
|
return tempfile.TemporaryFile(bufsize=0, dir='/tmp')
|
|
# Modify defaults based on parameters.
|
# Note that tempfiles must be unbuffered else attempts to read
|
# what a separate process did to that file can result in a bad
|
# view of the file.
|
# The Popen API accepts either an int or a file handle for stdout/stderr.
|
# pylint: disable=redefined-variable-type
|
if log_stdout_to_file:
|
stdout = open(log_stdout_to_file, 'w+')
|
elif stdout_to_pipe:
|
stdout = subprocess.PIPE
|
elif redirect_stdout:
|
stdout = _get_tempfile()
|
|
if combine_stdout_stderr:
|
stderr = subprocess.STDOUT
|
elif redirect_stderr:
|
stderr = _get_tempfile()
|
# pylint: enable=redefined-variable-type
|
|
# If subprocesses have direct access to stdout or stderr, they can bypass
|
# our buffers, so we need to flush to ensure that output is not interleaved.
|
if stdout is None or stderr is None:
|
sys.stdout.flush()
|
sys.stderr.flush()
|
|
# If input is a string, we'll create a pipe and send it through that.
|
# Otherwise we assume it's a file object that can be read from directly.
|
if isinstance(input, basestring):
|
stdin = subprocess.PIPE
|
elif input is not None:
|
stdin = input
|
input = None
|
|
if isinstance(cmd, basestring):
|
if not shell:
|
raise Exception('Cannot run a string command without a shell')
|
cmd = ['/bin/bash', '-c', cmd]
|
shell = False
|
elif shell:
|
raise Exception('Cannot run an array command with a shell')
|
|
# If we are using enter_chroot we need to use enterchroot pass env through
|
# to the final command.
|
env = env.copy() if env is not None else os.environ.copy()
|
env.update(extra_env if extra_env else {})
|
|
cmd_result.cmd = cmd
|
|
proc = None
|
# Verify that the signals modules is actually usable, and won't segfault
|
# upon invocation of getsignal. See signals.SignalModuleUsable for the
|
# details and upstream python bug.
|
use_signals = rh.signals.signal_module_usable()
|
try:
|
proc = _Popen(cmd, cwd=cwd, stdin=stdin, stdout=stdout,
|
stderr=stderr, shell=False, env=env,
|
close_fds=close_fds)
|
|
if use_signals:
|
old_sigint = signal.getsignal(signal.SIGINT)
|
if ignore_sigint:
|
handler = signal.SIG_IGN
|
else:
|
handler = functools.partial(
|
_kill_child_process, proc, int_timeout, kill_timeout, cmd,
|
old_sigint)
|
signal.signal(signal.SIGINT, handler)
|
|
old_sigterm = signal.getsignal(signal.SIGTERM)
|
handler = functools.partial(_kill_child_process, proc, int_timeout,
|
kill_timeout, cmd, old_sigterm)
|
signal.signal(signal.SIGTERM, handler)
|
|
try:
|
(cmd_result.output, cmd_result.error) = proc.communicate(input)
|
finally:
|
if use_signals:
|
signal.signal(signal.SIGINT, old_sigint)
|
signal.signal(signal.SIGTERM, old_sigterm)
|
|
if stdout and not log_stdout_to_file and not stdout_to_pipe:
|
# The linter is confused by how stdout is a file & an int.
|
# pylint: disable=maybe-no-member,no-member
|
stdout.seek(0)
|
cmd_result.output = stdout.read()
|
stdout.close()
|
|
if stderr and stderr != subprocess.STDOUT:
|
# The linter is confused by how stderr is a file & an int.
|
# pylint: disable=maybe-no-member,no-member
|
stderr.seek(0)
|
cmd_result.error = stderr.read()
|
stderr.close()
|
|
cmd_result.returncode = proc.returncode
|
|
if not error_code_ok and proc.returncode:
|
msg = 'cwd=%s' % cwd
|
if extra_env:
|
msg += ', extra env=%s' % extra_env
|
if error_message:
|
msg += '\n%s' % error_message
|
raise RunCommandError(msg, cmd_result)
|
except OSError as e:
|
estr = str(e)
|
if e.errno == errno.EACCES:
|
estr += '; does the program need `chmod a+x`?'
|
if error_code_ok:
|
cmd_result = CommandResult(cmd=cmd, error=estr, returncode=255)
|
else:
|
raise RunCommandError(estr, CommandResult(cmd=cmd), exception=e)
|
finally:
|
if proc is not None:
|
# Ensure the process is dead.
|
_kill_child_process(proc, int_timeout, kill_timeout, cmd, None,
|
None, None)
|
|
return cmd_result
|
# pylint: enable=redefined-builtin
|
|
|
def collection(classname, **kwargs):
|
"""Create a new class with mutable named members.
|
|
This is like collections.namedtuple, but mutable. Also similar to the
|
python 3.3 types.SimpleNamespace.
|
|
Example:
|
# Declare default values for this new class.
|
Foo = collection('Foo', a=0, b=10)
|
# Create a new class but set b to 4.
|
foo = Foo(b=4)
|
# Print out a (will be the default 0) and b (will be 4).
|
print('a = %i, b = %i' % (foo.a, foo.b))
|
"""
|
|
def sn_init(self, **kwargs):
|
"""The new class's __init__ function."""
|
# First verify the kwargs don't have excess settings.
|
valid_keys = set(self.__slots__[1:])
|
these_keys = set(kwargs.keys())
|
invalid_keys = these_keys - valid_keys
|
if invalid_keys:
|
raise TypeError('invalid keyword arguments for this object: %r' %
|
invalid_keys)
|
|
# Now initialize this object.
|
for k in valid_keys:
|
setattr(self, k, kwargs.get(k, self.__defaults__[k]))
|
|
def sn_repr(self):
|
"""The new class's __repr__ function."""
|
return '%s(%s)' % (classname, ', '.join(
|
'%s=%r' % (k, getattr(self, k)) for k in self.__slots__[1:]))
|
|
# Give the new class a unique name and then generate the code for it.
|
classname = 'Collection_%s' % classname
|
expr = '\n'.join((
|
'class %(classname)s(object):',
|
' __slots__ = ["__defaults__", "%(slots)s"]',
|
' __defaults__ = {}',
|
)) % {
|
'classname': classname,
|
'slots': '", "'.join(sorted(str(k) for k in kwargs)),
|
}
|
|
# Create the class in a local namespace as exec requires.
|
namespace = {}
|
exec expr in namespace # pylint: disable=exec-used
|
new_class = namespace[classname]
|
|
# Bind the helpers.
|
new_class.__defaults__ = kwargs.copy()
|
new_class.__init__ = sn_init
|
new_class.__repr__ = sn_repr
|
|
return new_class
|
