blob: e5990f3276b4caba9065b291f72530df102f8a00 [file] [log] [blame]
import logging, os, sys, subprocess, tempfile, traceback
import time
from autotest_lib.client.common_lib import utils
from autotest_lib.server import utils as server_utils
from autotest_lib.server.hosts import abstract_ssh, monitors
MONITORDIR = monitors.__path__[0]
SUPPORTED_PYTHON_VERS = ('2.4', '2.5', '2.6')
DEFAULT_PYTHON = '/usr/bin/python'
class Error(Exception):
pass
class InvalidPatternsPathError(Error):
"""An invalid patterns_path was specified."""
class InvalidConfigurationError(Error):
"""An invalid configuration was specified."""
class FollowFilesLaunchError(Error):
"""Error occurred launching followfiles remotely."""
def list_remote_pythons(host):
"""List out installed pythons on host."""
result = host.run('ls /usr/bin/python[0-9]*')
return result.stdout.splitlines()
def select_supported_python(installed_pythons):
"""Select a supported python from a list"""
for python in installed_pythons:
if python[-3:] in SUPPORTED_PYTHON_VERS:
return python
def copy_monitordir(host):
"""Copy over monitordir to a tmpdir on the remote host."""
tmp_dir = host.get_tmp_dir()
host.send_file(MONITORDIR, tmp_dir)
return os.path.join(tmp_dir, 'monitors')
def launch_remote_followfiles(host, lastlines_dirpath, follow_paths):
"""Launch followfiles.py remotely on follow_paths."""
logging.info('Launching followfiles on target: %s, %s, %s',
host.hostname, lastlines_dirpath, str(follow_paths))
# First make sure a supported Python is on host
installed_pythons = list_remote_pythons(host)
supported_python = select_supported_python(installed_pythons)
if not supported_python:
if DEFAULT_PYTHON in installed_pythons:
logging.info('No versioned Python binary found, '
'defaulting to: %s', DEFAULT_PYTHON)
supported_python = DEFAULT_PYTHON
else:
raise FollowFilesLaunchError('No supported Python on host.')
remote_monitordir = copy_monitordir(host)
remote_script_path = os.path.join(remote_monitordir, 'followfiles.py')
followfiles_cmd = '%s %s --lastlines_dirpath=%s %s' % (
supported_python, remote_script_path,
lastlines_dirpath, ' '.join(follow_paths))
remote_ff_proc = subprocess.Popen(host._make_ssh_cmd(followfiles_cmd),
stdin=open(os.devnull, 'r'),
stdout=subprocess.PIPE, shell=True)
# Give it enough time to crash if it's going to (it shouldn't).
time.sleep(5)
doa = remote_ff_proc.poll()
if doa:
raise FollowFilesLaunchError('ssh command crashed.')
return remote_ff_proc
def resolve_patterns_path(patterns_path):
"""Resolve patterns_path to existing absolute local path or raise.
As a convenience we allow users to specify a non-absolute patterns_path.
However these need to be resolved before allowing them to be passed down
to console.py.
For now we expect non-absolute ones to be in self.monitordir.
"""
if os.path.isabs(patterns_path):
if os.path.exists(patterns_path):
return patterns_path
else:
raise InvalidPatternsPathError('Absolute path does not exist.')
else:
patterns_path = os.path.join(MONITORDIR, patterns_path)
if os.path.exists(patterns_path):
return patterns_path
else:
raise InvalidPatternsPathError('Relative path does not exist.')
def launch_local_console(
input_stream, console_log_path, pattern_paths=None):
"""Launch console.py locally.
This will process the output from followfiles and
fire warning messages per configuration in pattern_paths.
"""
r, w = os.pipe()
local_script_path = os.path.join(MONITORDIR, 'console.py')
console_cmd = [sys.executable, local_script_path]
if pattern_paths:
console_cmd.append('--pattern_paths=%s' % ','.join(pattern_paths))
console_cmd += [console_log_path, str(w)]
# Setup warning stream before we actually launch
warning_stream = os.fdopen(r, 'r', 0)
devnull_w = open(os.devnull, 'w')
# Launch console.py locally
console_proc = subprocess.Popen(
console_cmd, stdin=input_stream,
stdout=devnull_w, stderr=devnull_w)
os.close(w)
return console_proc, warning_stream
def _log_and_ignore_exceptions(f):
"""Decorator: automatically log exception during a method call.
"""
def wrapped(self, *args, **dargs):
try:
return f(self, *args, **dargs)
except Exception, e:
print "LogfileMonitor.%s failed with exception %s" % (f.__name__, e)
print "Exception ignored:"
traceback.print_exc(file=sys.stdout)
wrapped.__name__ = f.__name__
wrapped.__doc__ = f.__doc__
wrapped.__dict__.update(f.__dict__)
return wrapped
class LogfileMonitorMixin(abstract_ssh.AbstractSSHHost):
"""This can monitor one or more remote files using tail.
This class and its counterpart script, monitors/followfiles.py,
add most functionality one would need to launch and monitor
remote tail processes on self.hostname.
This can be used by subclassing normally or by calling
NewLogfileMonitorMixin (below)
It is configured via two class attributes:
follow_paths: Remote paths to monitor
pattern_paths: Local paths to alert pattern definition files.
"""
follow_paths = ()
pattern_paths = ()
def _initialize(self, console_log=None, *args, **dargs):
super(LogfileMonitorMixin, self)._initialize(*args, **dargs)
self._lastlines_dirpath = None
self._console_proc = None
self._console_log = console_log or 'logfile_monitor.log'
def reboot_followup(self, *args, **dargs):
super(LogfileMonitorMixin, self).reboot_followup(*args, **dargs)
self.__stop_loggers()
self.__start_loggers()
def start_loggers(self):
super(LogfileMonitorMixin, self).start_loggers()
self.__start_loggers()
def remote_path_exists(self, remote_path):
"""Return True if remote_path exists, False otherwise."""
return not self.run(
'ls %s' % remote_path, ignore_status=True).exit_status
def check_remote_paths(self, remote_paths):
"""Return list of remote_paths that currently exist."""
return [
path for path in remote_paths if self.remote_path_exists(path)]
@_log_and_ignore_exceptions
def __start_loggers(self):
"""Start multifile monitoring logger.
Launch monitors/followfiles.py on the target and hook its output
to monitors/console.py locally.
"""
# Check if follow_paths exist, in the case that one doesn't
# emit a warning and proceed.
follow_paths_set = set(self.follow_paths)
existing = self.check_remote_paths(follow_paths_set)
missing = follow_paths_set.difference(existing)
if missing:
# Log warning that we are missing expected remote paths.
logging.warn('Target %s is missing expected remote paths: %s',
self.hostname, ', '.join(missing))
# If none of them exist just return (for now).
if not existing:
return
# Create a new lastlines_dirpath on the remote host if not already set.
if not self._lastlines_dirpath:
self._lastlines_dirpath = self.get_tmp_dir(parent='/var/tmp')
# Launch followfiles on target
try:
self._followfiles_proc = launch_remote_followfiles(
self, self._lastlines_dirpath, existing)
except FollowFilesLaunchError:
# We're hosed, there is no point in proceeding.
logging.fatal('Failed to launch followfiles on target,'
' aborting logfile monitoring: %s', self.hostname)
if self.job:
# Put a warning in the status.log
self.job.record(
'WARN', None, 'logfile.monitor',
'followfiles launch failed')
return
# Ensure we have sane pattern_paths before launching console.py
sane_pattern_paths = []
for patterns_path in set(self.pattern_paths):
try:
patterns_path = resolve_patterns_path(patterns_path)
except InvalidPatternsPathError, e:
logging.warn('Specified patterns_path is invalid: %s, %s',
patterns_path, str(e))
else:
sane_pattern_paths.append(patterns_path)
# Launch console.py locally, pass in output stream from followfiles.
self._console_proc, self._logfile_warning_stream = \
launch_local_console(
self._followfiles_proc.stdout, self._console_log,
sane_pattern_paths)
if self.job:
self.job.warning_loggers.add(self._logfile_warning_stream)
def stop_loggers(self):
super(LogfileMonitorMixin, self).stop_loggers()
self.__stop_loggers()
@_log_and_ignore_exceptions
def __stop_loggers(self):
if self._console_proc:
utils.nuke_subprocess(self._console_proc)
utils.nuke_subprocess(self._followfiles_proc)
self._console_proc = self._followfile_proc = None
if self.job:
self.job.warning_loggers.discard(self._logfile_warning_stream)
self._logfile_warning_stream.close()
def NewLogfileMonitorMixin(follow_paths, pattern_paths=None):
"""Create a custom in-memory subclass of LogfileMonitorMixin.
Args:
follow_paths: list; Remote paths to tail.
pattern_paths: list; Local alert pattern definition files.
"""
if not follow_paths or (pattern_paths and not follow_paths):
raise InvalidConfigurationError
return type(
'LogfileMonitorMixin%d' % id(follow_paths),
(LogfileMonitorMixin,),
{'follow_paths': follow_paths,
'pattern_paths': pattern_paths or ()})