| #!/usr/bin/python |
| # |
| # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """System Monitor. |
| |
| This program monitors the health of Chrome OS devices in the AutoTest testbed. |
| |
| Classes: |
| |
| Monitor - The Monitor is responsible for managing the overall process of |
| keeping an updated status of each host available to AutoTest. |
| |
| RemoteWorker - responsible for SSHing to remote hosts to gather resources. |
| |
| Resource - maintains all of the resources that are monitored, and methods to |
| parse their data for consumption by RRDTool. |
| |
| RRD - maintains all interfaces to RRDTool, including graph definitions, and |
| methods to create, update, and graph resources. |
| |
| TestBed - a global class used to hold configuration data and data collected |
| from each remote host. Additionally, formatted data for RRD will be kept |
| associated with each host, and some general information about the update |
| process of each remote host. |
| |
| |
| Usage: |
| The following options are supported: |
| --webdir: Systemhealth web directory. |
| --url: URL for landing page. |
| --datadir: Non-NFS directory for RRD files. |
| |
| --graph: Create 1, 4, and 24 hour graphs for each host. |
| --all_graphs: Create all graphs for each host. |
| --html: Build HTML pages for hosts. |
| --update: Collect data from hosts. |
| --skip_at_status: Don't collect data about hosts from autotest CLI. |
| --timout: Seconds to wait for remote commands to complete. |
| |
| --log_file: Write log messages to specified log file. |
| --skip_console: Do not write log messages to the console. |
| --verbose: Set the logging level to debug. |
| |
| --cli: Autotest CLI executable location. |
| --acl: Autotest ACL Group to query for host machines. |
| --label: Only run on hosts with the specified label. |
| --status: Only run on hosts with the specified status. |
| --user: Only run on hosts with the specified user. |
| |
| Arguments should be space separated. |
| """ |
| |
| __author__ = ('kdlucas@gmail.com (Kelly Lucas) & ' |
| 'pauldean@google.com (Paul Pendlebury)') |
| __version__ = '3.10' |
| |
| |
| import cPickle |
| import datetime |
| import json |
| import logging |
| import optparse |
| import os |
| import shutil |
| import sys |
| import time |
| import traceback |
| |
| import chromeos_test_common |
| from chromeos_test import autotest_util |
| from chromeos_test import common_util |
| from chromeos_test import mp_log_util |
| from chromeos_test import mp_thread_pool as tp |
| import IPy |
| |
| |
| class RemoteWorker(object): |
| """Obtain resource data from remote hosts using monitor_remote_worker.py.""" |
| |
| def __init__(self, hostname, platform, testbed): |
| """Inits RemoteWorker with hostname and test configuration. |
| |
| Args: |
| hostname: string, hostname of AutoTest host. |
| platform: string, platform of hostname. |
| testbed: testbed object for this run. |
| """ |
| self.h = hostname |
| self.platform = platform |
| self.tb = testbed |
| |
| # Set up some dictionaries for each host. |
| self.host_data = {} |
| self.host_data['rrddata'] = {} # Formatted data. |
| self.host_data['status'] = False |
| self.host_data['time'] = None |
| for v in self.tb.version: |
| self.host_data[v] = {} |
| self.host_data[v]['PTR'] = None |
| |
| def Run(self, logger): |
| """Method called into by thread pool.""" |
| |
| logger.debug('Starting host %s.', self.h) |
| updated_html_needed = False |
| data_file = os.path.join(self.tb.datadir, 'hosts', self.h, 'data.pkl') |
| local_script = os.path.join(chromeos_test_common.CURRENT_DIR, |
| 'monitor_remote_worker.py') |
| remote_script = '/tmp/monitor_remote_worker.py' |
| |
| try: |
| if self.tb.update: |
| if not os.path.isfile(local_script): |
| logger.error('Script file %s missing.', local_script) |
| return |
| |
| # Copy script |
| try: |
| common_util.RemoteCopy(self.h, 'root', local_script, remote_script) |
| except common_util.ChromeOSTestError: |
| logger.error('Skipping unreachable host %s.', self.h) |
| return |
| # Run Script |
| try: |
| output = common_util.RemoteCommand(self.h, 'root', remote_script, |
| output=True) |
| self.host_data = cPickle.loads(output) |
| except common_util.ChromeOSTestError: |
| logger.exception('Error running script on host %s.', self.h) |
| self.host_data['status'] = 'CollectionError' |
| else: |
| # If it exists, load saved host_data. |
| if os.path.isfile(data_file): |
| with open(data_file, 'rb') as in_file: |
| self.host_data = cPickle.load(in_file) |
| |
| advisor = Resource() |
| if ((self.tb.update or self.tb.graph) and |
| self.host_data['status'] != 'CollectionError'): |
| updated_html_needed = self.UpdateRelease(logger) |
| advisor.ProcessHostRRD(self.h, self.host_data, self.tb, logger) |
| if self.tb.html: |
| advisor.BuildHTML(self.h, self.platform, self.host_data, self.tb, |
| updated_html_needed) |
| |
| # Save successful host data so it can be loaded later. |
| if self.tb.update and self.host_data['status'] == 'True': |
| # rrd data is no longer needed, so don't save it. |
| del self.host_data['rrddata'] |
| self.host_data['rrddata'] = {} |
| with open(data_file, 'wb') as out_file: |
| cPickle.dump(self.host_data, out_file, cPickle.HIGHEST_PROTOCOL) |
| |
| # Lots of exception handling happening here. This is the entry point |
| # for this thread/process and if we let an exception go unhandled |
| # we wouldn't it from the main thread and we would miss any |
| # notifications of problems. |
| except (KeyboardInterrupt, SystemExit): |
| logging.exception('Shutdown requested.') |
| sys.exit(1) |
| except Exception: |
| logging.exception('Unexpected Exception on %s', self.h) |
| raise |
| logger.debug('Finished host %s.', self.h) |
| |
| def UpdateRelease(self, logger): |
| """Update Release info with most current release versions. |
| |
| The PTR key points to the most recent released version. This will also |
| preserve the last known release version in case the host is down. |
| |
| Args: |
| logger: multiprocess logger |
| |
| Returns: |
| True/False if new HTML files are needed for this host. |
| """ |
| rrd_dir = os.path.join(self.tb.datadir, 'hosts', self.h, 'rrd') |
| # Check if the host directory exists, if not create it. |
| common_util.MakedirsExisting(rrd_dir) |
| |
| update_html = False |
| for v in self.tb.version: |
| update_file = False |
| relfile = os.path.join(rrd_dir, v) |
| tmpfile = os.path.join(rrd_dir, v + '.tmp') |
| if os.path.isfile(relfile): |
| try: |
| rf = open(relfile, 'r') |
| lines = rf.readlines() |
| except IOError, e: |
| logger.error('Parsing release file %s\n%s', relfile, e) |
| finally: |
| rf.close() |
| |
| for line in lines: |
| fields = line.split('=') |
| # The correct format will have two strings separated by =. |
| if len(fields) == 2: |
| if fields[0] == 'PTR': |
| if self.host_data[v]['PTR']: |
| if self.host_data[v]['PTR'] != fields[1]: |
| # Most recent version has changed. |
| update_file = True |
| lines.pop(lines.index(line)) |
| self.host_data[v][self.tb.time] = (self.host_data[v]['PTR']) |
| else: |
| # Host is down so use last known value. |
| self.host_data[v]['PTR'] = (fields[1].strip()) |
| else: |
| self.host_data[v][fields[0]] = (fields[1].strip()) |
| elif len(line) > 3: |
| # This means the release file has the wrong format, so |
| # we'll just write a new one with current values. |
| update_file = True |
| lines.pop(lines.index(line)) |
| else: |
| # If we get here than it's probably a blank line. |
| update_file = True |
| lines.pop(lines.index(line)) |
| |
| if update_file: |
| update_html = True |
| logger.debug('Updating %s', relfile) |
| shutil.move(relfile, tmpfile) |
| # Put the most recent update in the new file, and make the |
| # PTR key to point to it. |
| lines.append('%s=%s\n' % (self.tb.time, self.host_data[v]['PTR'])) |
| lines.append('PTR=%s' % self.host_data[v]['PTR']) |
| try: |
| rf = open(relfile, 'w') |
| for line in lines: |
| rf.write(line) |
| except IOError, e: |
| logger.error('Writing %s\n%s', relfile, e) |
| finally: |
| rf.close() |
| else: |
| # Create a new release file, as it does not exist. |
| if self.host_data[v]['PTR']: |
| update_html = True |
| logger.info('Creating new %s', relfile) |
| try: |
| rf = open(relfile, 'w') |
| rf.write('%s=%s\n' % (self.tb.time, self.host_data[v]['PTR'])) |
| rf.write('PTR=%s' % self.host_data[v]['PTR']) |
| except IOError, e: |
| logger.error('Writing %s\n%s', relfile, e) |
| finally: |
| rf.close() |
| |
| self.host_data[v][self.tb.time] = (self.host_data[v]['PTR']) |
| return update_html |
| |
| |
| class TestBed(object): |
| """Used to hold all of the global variables.""" |
| |
| def __init__(self, options): |
| """Inits TestBed with run options. |
| |
| Args: |
| options: Command line args for this run. |
| """ |
| # Save run start time. |
| self.time = int(time.time()) |
| |
| # Setup logging. |
| self.options = options |
| self.logfile = options.log_file |
| |
| logger = logging.getLogger() |
| mp_log_util.InitializeLogging(logger, **vars(options)) |
| |
| # Warn and exit if SSH is not in the environment. |
| if not 'SSH_AGENT_PID' in os.environ: |
| logger.error('SSH_AGENT_PID not in environment, ssh commands will fail ' |
| 'to execute.') |
| sys.exit(1) |
| |
| # Verify RRD is installed where we expect it. |
| if not os.path.exists('/usr/bin/rrdtool'): |
| logger.error('RRD is not installed to /usr/bin/rrdtool. Run \'sudo ' |
| 'apt-get install rrdtool\'.') |
| sys.exit(1) |
| |
| # Assign TestBed values used for RRD and HTML pages. |
| self.version = ['ec_firmware', 'firmware', 'release'] |
| self.rrdtimes = ['-1hours', '-4hours', '-24hours', '-1week', '-1month', |
| '-1year'] |
| |
| # Make sure directories exist to hold status and data files. |
| run_dir = os.path.normpath('/tmp/systemhealth') |
| common_util.MakedirsExisting(run_dir) |
| |
| # Default status files. Used to prevent more than one instance from |
| # running at the same time. |
| self.update_runfile = os.path.join(run_dir, 'update.running') |
| self.graph_runfile = os.path.join(run_dir, 'graph.running') |
| |
| # Requested run actions. |
| self.graph = options.graph |
| self.all_graphs = options.all_graphs |
| self.html = options.html |
| self.update = options.update |
| self.timeout = options.timeout |
| self.skip_at_status = options.skip_at_status |
| |
| # Machine setup. |
| self.webdir = options.webdir |
| self.url = options.url |
| self.datadir = options.datadir |
| |
| # Output some debug info. |
| self.run_description = str(os.getpid()) + ':' |
| if self.update: |
| self.run_description += ' Update' |
| if self.graph: |
| self.run_description += ' Graph' |
| if self.all_graphs: |
| self.run_description += '_All' |
| if self.html: |
| self.run_description += ' HTML' |
| if not self.skip_at_status: |
| self.run_description += ' Status' |
| mp_log_util.LogWithHeader('Start ' + self.run_description, logger) |
| |
| |
| class Monitor(object): |
| """Main class used to manage the monitoring of remote hosts. |
| |
| This class is used to determine the current status of hosts in the AutoTest |
| testbed. AutoTest will be queried to populate self.rhosts. It will populate |
| a list of RemoteWorkes and submit that list to MultiProcWorkPool to query |
| each host to gather resource data. |
| """ |
| |
| def __init__(self, testbed, options): |
| """Monitor will use config data from TestBed.""" |
| self.tb = testbed |
| self.options = options |
| self.mp_wp = tp.MultiProcWorkPool() |
| self.afe_hosts = autotest_util.GetHostData(self.tb.options.cli, |
| self.tb.options.acl, |
| self.tb.options.label, |
| self.tb.options.user, |
| self.tb.options.status) |
| self.host_status = [] |
| |
| def UpdateStatus(self): |
| """Update data from all monitored hosts.""" |
| |
| # Don't attempt work when no hosts are known. |
| if not self.afe_hosts: |
| return |
| |
| # Record known host status from Autotest |
| if not self.options.skip_at_status: |
| self.RecordAutotestHostStatus(self.afe_hosts) |
| |
| # Create instance of RemoteWorker class for every host from atest. |
| self.host_status = [RemoteWorker(host, self.afe_hosts[host]['platform'], |
| self.tb) for host in self.afe_hosts.keys()] |
| |
| # Submit RemoteWorker items to thread pool. |
| self.host_status = self.mp_wp.ExecuteWorkItems( |
| self.host_status, 'Run', provide_logger=True, |
| logger_init_callback=mp_log_util.InitializeLogging, |
| **vars(self.options)) |
| |
| loglevel = logging.getLogger().getEffectiveLevel() |
| if loglevel == logging.DEBUG: |
| for worker in self.host_status: |
| logging.debug('%s status is %s/%s', worker.h, |
| worker.host_data['status'], |
| self.afe_hosts[worker.h]['status']) |
| |
| def RecordAutotestHostStatus(self, hosts): |
| """Record Autotest status of all hosts in rrd files. |
| |
| Args: |
| hosts: Dictionary of host information from autotest cli. |
| """ |
| |
| # Maps a host status string to an index in an array. |
| status_key = {'Repairing': 0, 'Verifying': 1, 'Repair_Failed': 2, |
| 'Running': 3, 'Cleaning': 4, 'Ready': 5, 'Pending': 6} |
| |
| # lab_status holds the lab data in the format rrd needs. The special |
| # netbook_ALL platform is the sum of all the platforms. |
| lab_status = {'netbook_ALL': [0] * len(status_key)} |
| |
| # Loop through all the hosts recording their status in lab_status |
| for host in hosts: |
| status = hosts[host]['status'].replace(' ', '_') |
| platform = hosts[host]['platform'] |
| |
| if platform not in lab_status: |
| lab_status[platform] = [0] * len(status_key) |
| if status in status_key: |
| lab_status[platform][status_key[status]] += 1 |
| lab_status['netbook_ALL'][status_key[status]] += 1 |
| else: |
| logging.error('Status=%s not a known status of %s', status, status_key) |
| |
| Resource().ProcessAutotestRRD(lab_status, self.tb, logging.getLogger()) |
| |
| # Save data for later analysis in a pickled data file. |
| for platform in lab_status: |
| data_folder = os.path.join(self.tb.datadir, 'hosts', platform) |
| common_util.MakedirsExisting(data_folder) |
| |
| data_file = os.path.join(data_folder, 'utilization.pkl') |
| platform_data = {} |
| if os.path.isfile(data_file): |
| with open(data_file, 'rb') as in_file: |
| platform_data = cPickle.load(in_file) |
| |
| date_entry = datetime.datetime.strftime(datetime.datetime.now(), |
| '%Y_%m_%d_%H_%M_%S') |
| platform_data[date_entry] = lab_status[platform] |
| with open(data_file, 'wb') as out_file: |
| cPickle.dump(platform_data, out_file, cPickle.HIGHEST_PROTOCOL) |
| |
| @staticmethod |
| def ValidIP(address): |
| """Verify address is a valid IP address. |
| |
| Args: |
| address: string. |
| Returns: |
| boolean: True = valid IP address, False = not valid IP address. |
| """ |
| octets = address.split('.') |
| if len(octets) != 4: |
| return False |
| for octet in octets: |
| if not 0 <= int(octet) <= 255: |
| return False |
| return True |
| |
| def SortAFEHosts(self, afelist): |
| """Sort AFE host list by IP address. |
| |
| Args: |
| afelist: list of AFE host objects. |
| Returns: |
| newlist: list of sorted AFE host objects. |
| """ |
| iplist = [] |
| hostlist = [] |
| |
| for h in afelist: |
| if self.ValidIP(h): |
| iplist.append(h) |
| else: |
| hostlist.append(h) |
| |
| templist = [(IPy.IP(h).int(), h) for h in iplist] |
| templist.sort() |
| newlist = [h[1] for h in templist] |
| hostlist.sort() |
| newlist.extend(hostlist) |
| |
| return newlist |
| |
| def BuildLandingPage(self): |
| """Build the initial HTML landing page with links to all hosts.""" |
| logging.debug('Building Landing Page') |
| sorted_hosts = [] |
| downhosts = 0 |
| down_repair = 0 |
| down_running = 0 |
| down_ready = 0 |
| down_other = 0 |
| |
| readyhosts = 0 |
| ready_repair = 0 |
| ready_running = 0 |
| ready_ready = 0 |
| ready_other = 0 |
| |
| scripthosts = 0 |
| script_repair = 0 |
| script_running = 0 |
| script_ready = 0 |
| script_other = 0 |
| |
| hostlist = self.afe_hosts.keys() |
| sorted_ip = self.SortAFEHosts(hostlist) |
| |
| # Create a dictionary to easily map host name to host result. |
| host_results = {} |
| for host in self.host_status: |
| host_results[host.h] = host |
| |
| # Put host that are down first |
| for h in sorted_ip: |
| insert_offset = 0 |
| # Up hosts. |
| if host_results[h].host_data['status'] == 'True': |
| readyhosts += 1 |
| insert_offset += downhosts + scripthosts |
| if self.afe_hosts[h]['status'] == 'Repair': |
| insert_offset += ready_repair |
| ready_repair += 1 |
| self.afe_hosts[h]['color'] = '#96BAC6' |
| self.afe_hosts[h]['status_string'] = 'Repair' |
| elif self.afe_hosts[h]['status'] == 'Running': |
| insert_offset += ready_repair + ready_running |
| ready_running += 1 |
| self.afe_hosts[h]['color'] = '#BBD9EE' |
| self.afe_hosts[h]['status_string'] = 'Running' |
| elif self.afe_hosts[h]['status'] == 'Ready': |
| insert_offset += ready_repair + ready_running + ready_ready |
| ready_ready += 1 |
| self.afe_hosts[h]['color'] = '#FFFFFF' |
| self.afe_hosts[h]['status_string'] = 'Ready' |
| else: |
| insert_offset += (ready_repair + ready_running + ready_ready + |
| ready_other) |
| ready_other += 1 |
| self.afe_hosts[h]['color'] = '#788D9A' |
| status_str = self.afe_hosts[h]['status'] |
| self.afe_hosts[h]['status_string'] = status_str |
| # Up hosts with python problems. |
| elif host_results[h].host_data['status'] == 'CollectionError': |
| scripthosts += 1 |
| insert_offset += downhosts |
| if self.afe_hosts[h]['status'] == 'Repair': |
| insert_offset += script_repair |
| script_repair += 1 |
| self.afe_hosts[h]['color'] = '#245403' |
| self.afe_hosts[h]['status_string'] = 'ScriptError/Repair' |
| elif self.afe_hosts[h]['status'] == 'Running': |
| insert_offset += script_repair + script_running |
| script_running += 1 |
| self.afe_hosts[h]['color'] = '#406331' |
| self.afe_hosts[h]['status_string'] = 'ScriptError/Running' |
| elif self.afe_hosts[h]['status'] == 'Ready': |
| insert_offset += (script_repair + script_running + script_ready) |
| script_ready += 1 |
| self.afe_hosts[h]['color'] = '#5E924E' |
| self.afe_hosts[h]['status_string'] = 'ScriptError/Ready' |
| else: |
| insert_offset += (script_repair + script_running + script_ready + |
| script_other) |
| script_other += 1 |
| self.afe_hosts[h]['color'] = '#183503' |
| status_str = 'ScriptError/' + self.afe_hosts[h]['status'] |
| self.afe_hosts[h]['status_string'] = status_str |
| # Down hosts. |
| else: |
| downhosts += 1 |
| if self.afe_hosts[h]['status'] == 'Repair': |
| insert_offset += down_repair |
| down_repair += 1 |
| self.afe_hosts[h]['color'] = '#867146' |
| self.afe_hosts[h]['status_string'] = 'Down/Repair' |
| elif self.afe_hosts[h]['status'] == 'Running': |
| insert_offset += down_repair + down_running |
| down_running += 1 |
| self.afe_hosts[h]['color'] = '#E5DCBD' |
| self.afe_hosts[h]['status_string'] = 'Down/Running' |
| elif self.afe_hosts[h]['status'] == 'Ready': |
| insert_offset += down_repair + down_running + down_ready |
| down_ready += 1 |
| self.afe_hosts[h]['color'] = '#D6C085' |
| self.afe_hosts[h]['status_string'] = 'Down/Ready' |
| else: |
| insert_offset += (down_repair + down_running + down_ready + |
| down_other) |
| down_other += 1 |
| self.afe_hosts[h]['color'] = '#4F4126' |
| status_str = 'Down/' + self.afe_hosts[h]['status'] |
| self.afe_hosts[h]['status_string'] = status_str |
| sorted_hosts.insert(insert_offset, h) |
| |
| # If we didn't connect to the host this run, load data from |
| # the last successful run. |
| if host_results[h].host_data['status'] != 'True': |
| data_file = os.path.join(self.tb.datadir, 'hosts', h, 'data.pkl') |
| if os.path.isfile(data_file): |
| with open(data_file, 'rb') as in_file: |
| host_results[h].host_data = cPickle.load(in_file) |
| |
| # Create symlink to the log file if it does not exist. |
| log_filename = os.path.join(self.tb.webdir, 'monitor.log') |
| if not os.path.isfile(log_filename): |
| try: |
| os.symlink(self.tb.logfile, log_filename) |
| except OSError, e: |
| logging.error('Linking to logfile\n%s', e) |
| land_page_file = os.path.join(self.tb.webdir, 'index.html') |
| # The temp file is used so that there will always be viewable html page |
| # when the new page is being built. |
| land_page_temp = os.path.join(self.tb.webdir, 'temp.html') |
| f = open(land_page_temp, 'w') |
| f.write('<HTML><HEAD>') |
| f.write('<LINK REL="stylesheet" TYPE="text/css" HREF="table.css">') |
| f.write('<TITLE>AutoTest System Health Check</TITLE></HEAD>') |
| f.write('<BODY>') |
| f.write('<img src="chrome.png" style="float:left;"/>') |
| f.write('<table style="float: right">') |
| f.write(('<TR><TD><a href=%s>%s</a><TD>Hosts<TD>Ready<TD>Repair<TD>' |
| 'Running<TD>Other') % ('monitor.log', 'Log File')) |
| f.write('<TR><TD>Total') |
| f.write('<TD>%d<TD>%d<TD>%d<TD>%d<TD>%d' % ( |
| downhosts + readyhosts + scripthosts, |
| down_ready + ready_ready + script_ready, |
| down_repair + ready_repair + script_repair, |
| down_running + ready_running + script_running, |
| down_other + ready_other + script_other)) |
| f.write('<TR><TD>Inaccessible') |
| f.write('<TD>%d<TD>%d<TD>%d<TD>%d<TD>%d' % (downhosts, down_ready, |
| down_repair, down_running, |
| down_other)) |
| f.write('<TR><TD>Script Error') |
| f.write('<TD>%d<TD>%d<TD>%d<TD>%d<TD>%d' % (scripthosts, script_ready, |
| script_repair, script_running, |
| script_other)) |
| f.write('<TR><TD>Accessible') |
| f.write('<TD>%d<TD>%d<TD>%d<TD>%d<TD>%d' % (readyhosts, ready_ready, |
| ready_repair, ready_running, |
| ready_other)) |
| f.write('</table>') |
| f.write('<center><H1>CAUTOTEST Testbed</H1>') |
| f.write('<H2>System Health</H2>') |
| plat_graph = os.path.join(self.tb.url, 'hosts', 'netbook_ALL', |
| 'utilization-24hours.png') |
| f.write('<BR><BR><img src=%s ><BR><BR>' % plat_graph) |
| f.write('<table>') |
| f.write('<CAPTION>Hosts last updated: %s</CAPTION>' % time.strftime( |
| '%d %b %Y - %I:%M:%S %p %Z', time.localtime())) |
| f.write('<TR><TH>Hostname<TH>Status<TH>Labels<TH>Last Update') |
| f.write('<TH>Release<TH>Health</TR>') |
| for h in sorted_hosts: |
| link_dir = 'hosts/' + h |
| web_dir = os.path.join(self.tb.webdir, 'hosts', h) |
| common_util.MakedirsExisting(web_dir, 0755) |
| fqn = 'http://cautotest.corp.google.com/' |
| view_host = 'afe/#tab_id=view_host&object_id=%s' % h |
| hlink = fqn + view_host |
| f.write('<tr bgcolor=%s><th>' % self.afe_hosts[h]['color']) |
| f.write('<a href=%s>%s</a></th>' % (hlink, h)) |
| f.write('<td><em>%s</em>' % self.afe_hosts[h]['status_string']) |
| f.write('<td>') |
| f.write('<em><b>%s</b></em><br>' % self.afe_hosts[h]['platform']) |
| for label in self.afe_hosts[h]['labels']: |
| f.write('%s<br>' % label) |
| f.write('<td>%s' % host_results[h].host_data['time']) |
| if host_results[h].host_data['release']['PTR']: |
| f.write('<td>%s' % host_results[h].host_data['release']['PTR']) |
| else: |
| f.write('<td>Unknown') |
| index_file = os.path.join(web_dir, 'index.html') |
| if os.path.isfile(index_file): |
| f.write('<td><a href=%s' % self.tb.url) |
| f.write('%s/index.html target="_blank">' % link_dir) |
| f.write('health</a></td>') |
| else: |
| f.write('<td>None</td>') |
| f.write('</table><p>\n</center>\n</BODY></HTML>') |
| f.close() |
| shutil.copyfile(land_page_temp, land_page_file) |
| os.chmod(land_page_file, 0644) |
| |
| |
| class Resource(object): |
| """Contains structures and methods to collect health data on hosts. |
| |
| For each resource in self.resources, there must also be a corresponding |
| method to format the data into what RRDTool expects. |
| """ |
| |
| def __init__(self): |
| self.resources = [ |
| 'battery', |
| 'boot', |
| 'cpu', |
| 'load', |
| 'memory', |
| 'network', |
| 'power', |
| 'temp', |
| 'uptime' |
| ] |
| self.fs = [ |
| 'rootfsA_space', |
| 'rootfsA_inodes', |
| 'rootfsA_stats', |
| 'rootfsB_space', |
| 'rootfsB_inodes', |
| 'rootfsB_stats', |
| 'stateful_space', |
| 'stateful_inodes', |
| 'stateful_stats' |
| ] |
| self.resources += self.fs |
| |
| @staticmethod |
| def ProcessAutotestRRD(hosts, testbed, logger): |
| """Process formatted data into RRD files for each host in hosts. |
| |
| Args: |
| hosts: dictionary of platforms and their data for rrd. |
| testbed: configuration data for this run. |
| logger: logger for this process/thread. |
| """ |
| for platform in hosts: |
| rrd_dir = os.path.join(testbed.datadir, 'hosts', platform, 'rrd') |
| web_dir = os.path.join(testbed.webdir, 'hosts', platform) |
| |
| common_util.MakedirsExisting(rrd_dir) |
| common_util.MakedirsExisting(web_dir, 0755) |
| |
| rrd_list = [] |
| for v in hosts[platform]: |
| rrd_list += [str(v)] |
| |
| rrd_dict = {'rrddata': {'utilization': rrd_list}} |
| rrd = RRD('utilization', platform, rrd_dir, web_dir, testbed) |
| if not os.path.exists(rrd.rrdfile): |
| rrd.Create(logger, 600) |
| rrd.Update(rrd_dict, logger) |
| rrd.Graph(rrd_dict, logger, False) |
| |
| def ProcessHostRRD(self, hostname, hostdata, testbed, logger): |
| """Process formatted data into RRD files for host hostname. |
| |
| Args: |
| hostname: string, hostname of AutoTest host. |
| hostdata: raw data from the host. |
| testbed: configuration data for this run. |
| logger: logger for this process/thread. |
| """ |
| rrd_dir = os.path.join(testbed.datadir, 'hosts', hostname, 'rrd') |
| web_dir = os.path.join(testbed.webdir, 'hosts', hostname) |
| |
| common_util.MakedirsExisting(rrd_dir) |
| common_util.MakedirsExisting(web_dir, 0755) |
| |
| for r in self.resources: |
| dk = None # datakey only needs to be set if it's a file system. |
| if r in self.fs: |
| if '_space' in r: |
| dk = 'fs_space' |
| elif '_inode' in r: |
| dk = 'fs_inode' |
| elif '_stat' in r: |
| dk = 'fs_stat' |
| |
| rrd = RRD(r, hostname, rrd_dir, web_dir, testbed, dk) |
| if not os.path.exists(rrd.rrdfile): |
| rrd.Create(logger) |
| if testbed.update == True: |
| logger.debug('Updating %s for host %s', r, hostname) |
| rrd.Update(hostdata, logger) |
| if testbed.graph: |
| logger.debug('Building %s graphs for %s', r, hostname) |
| rrd.Graph(hostdata, logger) |
| |
| def BuildHTML(self, hostname, platform, hostdata, testbed, |
| update_needed=False): |
| """Create HTML pages for to display the graphs. |
| |
| Args: |
| hostname: string, hostname of AutoTest host. |
| platform: string, platform of hostname. |
| hostdata: raw data from the host. |
| testbed: configuration data for this run. |
| update_needed: new html needed, existing has wrong info. |
| """ |
| web_dir = os.path.join(testbed.webdir, 'hosts', hostname) |
| plat_dir = os.path.join(testbed.url, 'hosts', platform) |
| index_file = os.path.join(web_dir, 'index.html') |
| |
| # If the index file exists, and the release info hasn't changed, skip. |
| if os.path.isfile(index_file) and not update_needed: |
| return |
| |
| mainindex = testbed.url + 'index.html' |
| resource_list = [] |
| for r in self.resources: |
| resource_list.append(r) |
| resource_list.sort() |
| |
| html_file = {} |
| for t in testbed.rrdtimes: |
| html_file[t] = hostname + t + '.html' |
| pathname = {} |
| for name in html_file: |
| pathname[name] = os.path.join(web_dir, html_file[name]) |
| |
| # Create directory for html/graphs. |
| common_util.MakedirsExisting(web_dir, 0755) |
| |
| # Create HTML files for each time period we are graphing. |
| for path in pathname: |
| f = open(pathname[path], 'w') |
| f.write('<HTML><HEAD>') |
| f.write('<center><TITLE>%s System Health</TITLE></HEAD>' % hostname) |
| f.write('<BODY><H1>%s System Health</H1>' % hostname) |
| for v in testbed.version: |
| f.write('<H4>%s: %s</H4>' % (v, hostdata[v]['PTR'])) |
| for t in testbed.rrdtimes: |
| f.write('<a href="%s">%s</a> <b>|</b>' % (html_file[t], t)) |
| f.write('<a href="%s">SystemHealth Home</a>' % mainindex) |
| f.write('<p><HR>') |
| plat_graph = os.path.join(plat_dir, 'utilization' + path + '.png') |
| f.write('<img src=%s ><BR><BR>' % plat_graph) |
| f.write('<table border=1 bgcolor=#EEEEEE>') |
| newrow = True |
| for r in resource_list: |
| if newrow: |
| f.write('<tr>') |
| f.write('<td>%s<br><a href=%s.html>' % (r, r)) |
| f.write('<img src=%s%s.png width=475 height=250></a></td>' % (r, path)) |
| if newrow: |
| newrow = False |
| else: |
| f.write('</tr>\n') |
| newrow = True |
| f.write('</table><p>\n') |
| f.write('</center>\n') |
| f.write('<H5>Last Update: %s</H5>' % hostdata['time']) |
| f.write('</BODY></HTML>') |
| f.close() |
| os.chmod(pathname[path], 0644) |
| # Set default landing page to 24-hour graphs |
| if not os.path.isfile(index_file): |
| os.symlink(pathname[testbed.rrdtimes[2]], index_file) |
| |
| # Create HTML files for each resource for all time periods. |
| for r in resource_list: |
| rrdfile = os.path.join(web_dir, r + '.html') |
| f = open(rrdfile, 'w') |
| f.write('<HTML><HEAD>') |
| f.write('<center><TITLE>%s %s Resources</TITLE></HEAD>' % (hostname, r)) |
| f.write('<BODY><H1>%s %s Resources</H1>' % (hostname, r)) |
| for v in testbed.version: |
| f.write('<H4>%s: %s</H4>' % (v, hostdata[v]['PTR'])) |
| f.write('<table border=5 bgcolor=#B5B5B5>') |
| f.write('<tr>') |
| for t in testbed.rrdtimes: |
| f.write('<td><a href="#%s"><b>%s</b></a>' % (t, t)) |
| f.write('</table>') |
| f.write('<HR>') |
| f.write('<table border=1 bgcolor=#EEEEEE>') |
| for t in testbed.rrdtimes: |
| f.write('<tr><td><a name="%s"><img src=%s%s.png>' % (t, r, t)) |
| f.write('</a></td></tr>\n') |
| f.write('</table><p>\n') |
| f.write('</center>\n') |
| f.write('<H5>Last Update: %s</H5>' % hostdata['time']) |
| f.write('</BODY></HTML>') |
| f.close() |
| os.chmod(rrdfile, 0644) |
| |
| |
| class RRD(object): |
| """The class to create and update RRD data stores and graph them. |
| |
| This class should be used to access all of the functions of RRDTool. It will |
| create the data files, update them, and create graphs/charts based on that |
| data. Datakey is needed when we are using the same data definitions for many |
| items of the same type, like file systems. |
| """ |
| |
| def __init__(self, rrdname, hostname, rrd_dir, web_dir, tb, datakey=None): |
| """Inits RRD class. |
| |
| Args: |
| rrdname: string, item name(should match key from Resources) |
| hostname: string, hostname of the machine. |
| rrd_dir: string, directory for rrd files. |
| web_dir: string, directory for generated graphs. |
| tb: testbase object for this run. |
| datakey: string, overrides which data definition to use. |
| """ |
| self.tb = tb |
| self.rrdtool = '/usr/bin/rrdtool' |
| self.rrd_dir = rrd_dir |
| self.web_dir = web_dir |
| self.rrdname = rrdname |
| self.hostname = hostname |
| rrd_filename = rrdname + '.rrd' |
| self.rrdfile = os.path.join(self.rrd_dir, rrd_filename) |
| file_system = 'Unknown' |
| |
| if not datakey: |
| datakey = rrdname |
| else: |
| fields = rrdname.split('_') |
| if fields[0]: |
| file_system = fields[0] |
| |
| self.dd = json.load(open(os.path.join(sys.path[0], 'rrd.json')))[datakey] |
| self.dd['title'] %= {'host': self.hostname, 'file_system': file_system} |
| |
| def Create(self, logger, step=600): |
| """Create an empty RRD file. |
| |
| Args: |
| logger: Multiprocess logger. |
| step: Default rrdtool step. |
| |
| Returns: |
| boolean: True = Success, False = failure. |
| """ |
| |
| stime = int(time.time()) - 5 * 86400 |
| rrd_suffix = ['RRA:AVERAGE:0.5:1:576', 'RRA:AVERAGE:0.5:6:672', |
| 'RRA:AVERAGE:0.5:24:732', 'RRA:AVERAGE:0.5:144:1460'] |
| |
| rrd_cmd = [self.rrdtool, 'create', self.rrdfile, '--start', str(stime), |
| '--step', str(step)] |
| for ds in self.dd['items']: |
| ds_str = 'DS:%s:%s:%s:%s:%s' % (ds, self.dd['type'], self.dd['heartbeat'], |
| self.dd['min'], self.dd['max']) |
| rrd_cmd.append(ds_str) |
| rrd_cmd += rrd_suffix |
| # Convert the rrd_cmd to a string with space separated commands. |
| exec_str = ' '.join(rrd_cmd) |
| try: |
| common_util.RunCommand(exec_str) |
| except common_util.ChromeOSTestError: |
| logger.error('Executing: "%s".', exec_str) |
| return False |
| return True |
| |
| def Update(self, hostdata, logger): |
| """Update an existing RRD file. |
| |
| Args: |
| hostdata: dictionary of raw data from this host. |
| logger: logger for this process/thread |
| |
| Returns: |
| boolean: True = Success, False = errors. |
| """ |
| if self.rrdname in hostdata['rrddata']: |
| data_count = len(hostdata['rrddata'][self.rrdname]) |
| if data_count == 0: |
| logger.debug('Key "%s" empty in hostdata for host %s.', self.rrdname, |
| self.hostname) |
| return False |
| |
| if data_count < 2: |
| data = 'N:' + hostdata['rrddata'][self.rrdname][0] |
| else: |
| data = 'N:' + ':'.join(hostdata['rrddata'][self.rrdname]) |
| rrd_cmd = [self.rrdtool, 'update', self.rrdfile, data] |
| exec_str = ' '.join(rrd_cmd) |
| try: |
| common_util.RunCommand(exec_str) |
| except common_util.ChromeOSTestError: |
| logger.error('Executing: "%s".', exec_str) |
| return False |
| |
| return True |
| else: |
| logger.debug('Key "%s" not found in hostdata for host %s.', self.rrdname, |
| self.hostname) |
| return False |
| |
| def Graph(self, hostdata, logger, include_updates=True, file_prefix=''): |
| """Create a graph of a tracked resource. |
| |
| Args: |
| hostdata: Dictionary of raw data from this host. |
| logger: Logger for this process/thread. |
| include_updates: Include firmware update history in graphs. |
| file_prefix: String to append to front of graph file names. |
| """ |
| width = '850' |
| height = '300' |
| end = 'now' |
| rcolor = {'release': '#9966FF', 'firmware': '#990033', |
| 'ec_firmware': '#009933'} |
| |
| if self.tb.all_graphs: |
| rrdtimes = self.tb.rrdtimes |
| else: |
| rrdtimes = self.tb.rrdtimes[:3] |
| |
| for rrdtime in rrdtimes: |
| png_filename = file_prefix + self.rrdname + rrdtime + '.png' |
| png_file = os.path.join(self.web_dir, png_filename) |
| |
| title = self.dd['title'] + ' ' + rrdtime + '"' |
| |
| rrd_cmd = [self.rrdtool, 'graph', png_file, '--imgformat PNG', '-s', |
| rrdtime, '--end', end, '--width', width, '--height', height, |
| '--vertical-label', self.dd['units'], '--title', title] |
| |
| for ds in self.dd['items']: |
| rrd_cmd.append('DEF:%s=%s:%s:AVERAGE' % (ds, self.rrdfile, ds)) |
| rrd_cmd += self.dd['graph'] |
| if include_updates: |
| rrd_cmd.append('COMMENT:"Release History \\s"') |
| rrd_cmd.append('COMMENT:"=============== \\n"') |
| for v in self.tb.version: |
| sorted_items = [] |
| for k in hostdata[v]: |
| if k != 'PTR': |
| sorted_items.append(k) |
| sorted_items.sort() |
| for i in sorted_items: |
| # Get a date/time string to display, localtime requires |
| # a float, so convert i to float. |
| fw_datetime = time.strftime('%D %H\\:%M', time.localtime(float(i))) |
| # Need to escape any ':' for RRDTool. |
| filter_val = (hostdata[v][i].replace(':', '\\:')) |
| if not self.tb.all_graphs: |
| # Insert Veritical Lines for release and firmware updates. |
| vrule = 'VRULE:%s%s:"%s %s=%s \\n"' % (i, rcolor[v], fw_datetime, |
| v, filter_val) |
| else: |
| # On Week + graphs, only insert release comment. There are too |
| # many vertical lines on the longer graphs to make see anything |
| # else. |
| vrule = 'COMMENT:"%s %s=%s \\n"' % (fw_datetime, v, filter_val) |
| rrd_cmd.append(vrule) |
| |
| exec_str = ' '.join(rrd_cmd) |
| try: |
| common_util.RunCommand(exec_str) |
| except common_util.ChromeOSTestError: |
| logger.error('Executing: "%s".', exec_str) |
| if os.path.isfile(png_file): |
| os.chmod(png_file, 0644) |
| |
| |
| def ParseArgs(): |
| """Parse all command line options.""" |
| homedir = os.environ['HOME'] |
| datadir = os.path.normpath('/usr/local/google/%s/systemhealth' % homedir) |
| systemhealth_webdir = os.path.join(homedir, 'www', 'systemhealth') |
| logfile = os.path.join(systemhealth_webdir, 'monitor.log') |
| defaul_url = 'http://www/~%s/systemhealth/' % os.environ['USER'] |
| |
| parser = optparse.OptionParser(version=__version__) |
| |
| # Args for describing the environment of the server machine. |
| group = optparse.OptionGroup( |
| parser, title='Server Configuration', |
| description=('Options specifying the layout of this machine.')) |
| group.add_option( |
| '-w', '--webdir', |
| help='Systemhealth web directory [default: %default]', |
| default=systemhealth_webdir, |
| dest='webdir') |
| group.add_option( |
| '-u', '--url', |
| help='URL for landing page [default: %default]', |
| default=defaul_url, |
| dest='url') |
| group.add_option( |
| '-d', '--datadir', |
| help='Non-NFS directory for RRD. [default: %default]', |
| default=datadir, |
| dest='datadir') |
| parser.add_option_group(group) |
| |
| # Args for describing logging. |
| mp_log_util.AddOptions(parser) |
| |
| # Args for selecting hosts from Autotest. |
| autotest_util.AddOptions(parser) |
| |
| # Args for describing what work to perform. |
| group = optparse.OptionGroup( |
| parser, title='Run Configuration', |
| description=('Options specifying what actions the script will perform.')) |
| group.add_option( |
| '--graph', |
| help=('Create 1, 4, & 24 hour graphs for each host [default: %default]'), |
| default=False, action='store_true', dest='graph') |
| group.add_option( |
| '--all_graphs', |
| help='Create all graphs for each host [default: %default]', |
| default=False, action='store_true', dest='all_graphs') |
| group.add_option( |
| '--html', |
| help='Build HTML pages for hosts [default: %default]', |
| default=False, action='store_true', dest='html') |
| group.add_option( |
| '--update', |
| help='Collect data from hosts [default: %default]', |
| default=False, action='store_true', dest='update') |
| group.add_option( |
| '--timout', |
| help=('Timeout for remote commands to complete [default: %default]'), |
| default=30, dest='timeout') |
| group.add_option( |
| '--skip_at_status', |
| help=('Record the host status in autotest [default: %default]'), |
| default=False, action='store_true', dest='skip_at_status') |
| |
| parser.add_option_group(group) |
| |
| options = parser.parse_args()[0] |
| |
| if not options.log_file: |
| options.log_file = logfile |
| |
| if options.all_graphs: |
| options.graph = True |
| |
| if not (options.graph or options.html or options.update): |
| parser.error('Must specify at least one of the --graph, --html, or ' |
| '--update options.') |
| |
| # Create required directories if they don't exist. |
| common_util.MakedirsExisting(options.datadir) |
| common_util.MakedirsExisting(options.webdir, 0755) |
| common_util.MakedirsExisting(os.path.join(options.webdir, 'hosts'), 0755) |
| |
| return options |
| |
| |
| def CheckRun(action, tb): |
| """Check the run status of monitor.py, and add/remove run files. |
| |
| This function will ensure we only running one program with either the graph |
| or update option. |
| Args: |
| action: string, indicates if monitor.py is starting or stopping. |
| tb: options for this run. |
| """ |
| if action == 'start': |
| if tb.update == True: |
| if os.path.isfile(tb.update_runfile): |
| logging.info('Exiting, already running with update option') |
| sys.exit(1) |
| else: |
| try: |
| open(tb.update_runfile, 'w').close() |
| except IOError, e: |
| logging.error('Opening %s\n%s', tb.update_runfile, e) |
| if tb.graph: |
| if os.path.isfile(tb.graph_runfile): |
| logging.info('Exiting, already running with graph option') |
| sys.exit(1) |
| else: |
| try: |
| open(tb.graph_runfile, 'w').close() |
| except IOError, e: |
| logging.error('Opening %s\n%s', tb.graph_runfile, e) |
| elif action == 'stop': |
| if tb.update == True: |
| if os.path.isfile(tb.update_runfile): |
| try: |
| os.remove(tb.update_runfile) |
| except IOError, e: |
| logging.error('Removing %s\n%s', tb.update_runfile, e) |
| if tb.graph: |
| if os.path.isfile(tb.graph_runfile): |
| try: |
| os.remove(tb.graph_runfile) |
| except IOError, e: |
| logging.error('Removing %s\n%s', tb.graph_runfile, e) |
| else: |
| logging.error('Unknown option passed to CheckRun(): %s', action) |
| sys.exit(1) |
| |
| |
| def main(): |
| start_time = time.time() |
| options = ParseArgs() |
| |
| test_bed = TestBed(options) |
| CheckRun('start', test_bed) |
| try: |
| sysmon = Monitor(test_bed, options) |
| if not sysmon.afe_hosts: |
| logging.error('No hosts found, nothing to do, exiting.') |
| sys.exit(1) |
| sysmon.UpdateStatus() |
| if test_bed.update: |
| sysmon.BuildLandingPage() |
| |
| runtime = time.time() - start_time |
| msg = 'End [ %s ] Runtime %d seconds' % (test_bed.run_description, runtime) |
| mp_log_util.LogWithHeader(msg, symbol='-') |
| |
| except (KeyboardInterrupt, SystemExit): |
| logging.error('Shutdown requested.') |
| sys.exit(1) |
| except Exception, e: |
| logging.error('Exception: %s\n%s', e, traceback.format_exc()) |
| raise |
| finally: |
| CheckRun('stop', test_bed) |
| os.chmod(options.log_file, 0755) |
| |
| if __name__ == '__main__': |
| main() |
