| #!/usr/bin/python |
| # |
| # Copyright (c) 2010 The Chromium OS Authors. All Rights Reserved. |
| # Use of this souce code is governed by a BSD-sytle license that can be |
| # found in the LICENSE file. |
| |
| """System Monitor. |
| |
| This program monitors the health of Chrome OS devices in the AutoTest testbed. |
| |
| Classes: |
| |
| Monitor - The Monitor is responsible for managing the overall process of |
| keeping an updated status of each host available to AutoTest. |
| |
| MonitorThread - a very small threaded class that gets hosts from a queue and |
| will create a RemoteWorker object for each thread. |
| |
| RemoteWorker - responsible for SSHing to remote hosts to gather resources. |
| |
| Resource - maintains all of the resources that are monitored, and methods to |
| parse their data for consumption by RRDTool. |
| |
| RRD - maintains all interfaces to RRDTool, including graph definitions, and |
| methods to create, update, and graph resources. |
| |
| TestBed - a global class used to hold configuration data and data collected |
| from each remote host. Additionally, formatted data for RRD will be kept |
| associated with each host, and some general information about the update |
| process of each remote host. |
| |
| TBQueue - a subclass of Queue.Queue(), in order to override method join(), |
| since we need a timeout in case one of the paramiko ssh sessions hangs. |
| |
| |
| Usage: |
| The following options are supported: |
| --debug: set the debug level. Requires one of the following parameters: |
| debug |
| info (default) |
| warning |
| error |
| critical |
| --gclient: the source directory for ChromeOS source code. |
| --graph: boolean, if True, it will create new graphs for resources. |
| --home: the top level directory for systemhealth files to be placed. |
| --html: boolean, if set to True it will build html pages. |
| --logfile: set the file name of the log file. Default: monitor.log |
| --update: boolean, if True, it will collect new data from all monitored |
| hosts, and update the RRD databases with the newly collected data. |
| --url: string, the base URL for the landing page. |
| |
| Arguments should be space separated. |
| """ |
| |
| __author__ = 'kdlucas@gmail.com (Kelly Lucas)' |
| __version__ = '2.02' |
| |
| import logging, logging.handlers, optparse, os, paramiko, Queue, shutil |
| import subprocess, sys, threading |
| |
| from IPy import IP |
| from time import * |
| |
| TIMEOUT = 5 # Timeout for accessing remote hosts. |
| RUNTIME = 240 # Total time to allow the host queue to finish all tasks. |
| AUTOTEST_BIN = '/home/build/static/projects/chromeos/autotest' |
| |
| def SetLogger(namespace, logfile, loglevel, log_to_stdout=False): |
| """Create a log handler and set log level. |
| |
| Args: |
| namespace: name of the logger. |
| logfile: log file name. |
| loglevel: debug level of logger. |
| log_to_stdout: boolean, True = send msgs to stdout and logfile, |
| False = send msgs to log file only. |
| Returns: |
| Logger object. |
| We use RotatingFileHandler to handle rotating the log files when they reach |
| MAXSIZE in bytes. |
| """ |
| # The logs are linked to the landing page, so we don't want to allow them to |
| # grow too large. |
| MAXSIZE = 1024000 # Max size to grow log files, in bytes. |
| |
| levels = {'debug': logging.DEBUG, |
| 'info': logging.INFO, |
| 'warning': logging.WARNING, |
| 'error': logging.ERROR, |
| 'critical': logging.CRITICAL, |
| } |
| |
| logger = logging.getLogger(namespace) |
| c = logging.StreamHandler() |
| h = logging.handlers.RotatingFileHandler(logfile, maxBytes=MAXSIZE, |
| backupCount=10) |
| hf = logging.Formatter('%(asctime)s %(process)d %(levelname)s: %(message)s') |
| cf = logging.Formatter('%(levelname)s: %(message)s') |
| logger.addHandler(h) |
| h.setFormatter(hf) |
| if log_to_stdout: |
| logger.addHandler(c) |
| c.setFormatter(cf) |
| |
| logger.setLevel(levels.get(loglevel, logging.INFO)) |
| |
| return logger |
| |
| |
| class MonitorThread(threading.Thread): |
| """Get AutoTest hosts from queue and create remote host monitors.""" |
| |
| def __init__(self): |
| threading.Thread.__init__(self) |
| self.tb = TB # In case the thread outlives the main program. |
| |
| |
| def run(self): |
| host = self.tb.q.get() |
| worker = RemoteWorker(host) |
| try: |
| worker.run() |
| except Exception, e: |
| self.tb.logger.error('Error on remote host: %s\n%s', host, e) |
| # Notify Queue that process is finished. |
| self.tb.logger.debug('Releasing host %s from queue', host) |
| self.tb.q.task_done() |
| |
| |
| class RemoteWorker(object): |
| """SSH into remote hosts to obtain resource data.""" |
| |
| def __init__(self, hostname): |
| """ |
| Args: |
| hostname: string, hostname of AutoTest host. |
| """ |
| self.h = hostname |
| self.client = paramiko.SSHClient() |
| self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) |
| self.tb = TB # In case the thread outlives the main program. |
| # Send paramiko messages to it's own log file. |
| # paramiko.util.log_to_file() will overwrite the log file each time you |
| # call it, therefore do not send paramiko messages to the update log. |
| self.logger = SetLogger('SSH', self.tb.sshlog, self.tb.loglevel) |
| paramiko.util.log_to_file(self.tb.sshlog, level=30) |
| |
| |
| def run(self): |
| try: |
| self.client.connect(self.h, username='root', |
| key_filename=self.tb.privkey, timeout=TIMEOUT) |
| self.tb.hosts[self.h]['status'] = True |
| except Exception, e: |
| self.logger.error('Host %s: %s', self.h, e) |
| self.tb.hosts[self.h]['status'] = False |
| finally: |
| if self.tb.hosts[self.h]['status']: |
| try: |
| self.ReadRelease() # Must be done before UpdateRelease(). |
| self.ReadFirmware() |
| except Exception, e: |
| self.logger.error('Error on : %s\n%s', self.h, e) |
| self.UpdateRelease() # Must be done before ReadResources(). |
| if self.tb.hosts[self.h]['status']: |
| try: |
| self.ReadResources() |
| except Exception, e: |
| self.tb.logger.error('Error on: %s\n%s', self.h, e) |
| self.tb.logger.debug('Closing client for %s', self.h) |
| self.client.close() |
| self.tb.hosts[self.h]['time'] = strftime( |
| '%d%b%Y %H:%M:%S', localtime()) |
| |
| |
| def ReadRelease(self): |
| """Get the Chrome OS Release version.""" |
| # The PTR key will mark the current version. |
| |
| cmd = 'cat /etc/lsb-release' |
| try: |
| stdin, stdout, stderr = self.client.exec_command(cmd) |
| for line in stdout: |
| if 'CHROMEOS_RELEASE_DESCRIPTION' in line: |
| release = line.split('=') |
| self.tb.hosts[self.h]['release']['PTR'] = release[1].strip() |
| except Exception, e: |
| self.tb.logger.error('Error getting release version on host %s\n%s', |
| self.h, e) |
| |
| |
| def ReadFirmware(self): |
| """Get the Firmware versions.""" |
| # The PTR key will mark the current versions. |
| cmd = '/usr/sbin/mosys -k smbios info bios' |
| try: |
| stdin, stdout, stderr = self.client.exec_command(cmd) |
| for line in stdout: |
| lines = line.split('" ') |
| for item in lines: |
| if 'ec_version' in item: |
| fields = item.split('=') |
| # We must sanitize the string for RRDTool. |
| val = fields[1].strip('\n" ') |
| self.tb.hosts[self.h]['ec_firmware']['PTR'] = val |
| elif 'version' in item: |
| fields = item.split('=') |
| val = fields[1].strip('\n" ') |
| self.tb.hosts[self.h]['firmware']['PTR'] = val |
| except Exception, e: |
| self.tb.logger.error('Error getting firmware versions on %s\n%s', |
| self.h, e) |
| |
| |
| def UpdateRelease(self): |
| """Update Release info with most current release versions. |
| |
| The PTR key points to the most recent released version. This will also |
| preserve the last known release version in case the host is down. |
| """ |
| rrd_dir = os.path.join(self.tb.home, 'hosts', self.h, 'rrd') |
| for v in self.tb.version: |
| update_file = False |
| relfile = os.path.join(rrd_dir, v) |
| tmpfile = os.path.join(rrd_dir, v + '.tmp') |
| if os.path.isfile(relfile): |
| try: |
| rf = open(relfile, 'r') |
| lines = rf.readlines() |
| except IOError, e: |
| self.tb.logger.error('Error parsing release file %s\n%s', |
| relfile, e) |
| finally: |
| rf.close() |
| |
| for line in lines: |
| fields = line.split('=') |
| # The correct format will have two strings separated by =. |
| if len(fields) == 2: |
| if fields[0] == 'PTR': |
| if self.tb.hosts[self.h][v]['PTR']: |
| if self.tb.hosts[self.h][v]['PTR'] != fields[1]: |
| # Most recent version has changed. |
| update_file = True |
| lines.pop(lines.index(line)) |
| self.tb.hosts[self.h][v][self.tb.time] = ( |
| self.tb.hosts[self.h][v]['PTR']) |
| else: |
| # Host is down so use last known value. |
| self.tb.hosts[self.h][v]['PTR'] = ( |
| fields[1].strip()) |
| else: |
| self.tb.hosts[self.h][v][fields[0]] = ( |
| fields[1].strip()) |
| elif len(line) > 3: |
| # This means the release file has the wrong format, so |
| # we'll just write a new one with current values. |
| update_file = True |
| lines.pop(lines.index(line)) |
| else: |
| # If we get here than it's probably a blank line. |
| update_file = True |
| lines.pop(lines.index(line)) |
| |
| if update_file: |
| self.tb.logger.debug('Updating %s', relfile) |
| shutil.move(relfile, tmpfile) |
| # Put the most recent update in the new file, and make the |
| # PTR key to point to it. |
| lines.append('%s=%s\n' % (self.tb.time, |
| self.tb.hosts[self.h][v]['PTR'])) |
| lines.append('PTR=%s' % self.tb.hosts[self.h][v]['PTR']) |
| try: |
| rf = open(relfile, 'w') |
| for line in lines: |
| rf.write(line) |
| except IOError, e: |
| self.tb.logger.error('Error writing %s\n%s', relfile, e) |
| finally: |
| rf.close() |
| else: |
| # Create a new release file, as it does not exist. |
| if self.tb.hosts[self.h][v]['PTR']: |
| self.tb.logger.info('Creating new %s', relfile) |
| try: |
| rf = open(relfile, 'w') |
| rf.write('%s=%s\n' % ( |
| self.tb.time,self.tb.hosts[self.h][v]['PTR'])) |
| rf.write('PTR=%s' % self.tb.hosts[self.h][v]['PTR']) |
| except IOError, e: |
| self.tb.logger.error('Error writing %s\n%s', relfile, e) |
| finally: |
| rf.close() |
| |
| self.tb.hosts[self.h][v][self.tb.time] = ( |
| self.tb.hosts[self.h][v]['PTR']) |
| |
| |
| def ReadResources(self): |
| """Get resources that we are monitoring on the host.""" |
| |
| advisor = Resource() |
| if self.tb.update == True: |
| self.tb.logger.debug('Collecting data on %s', self.h) |
| cmd = advisor.GetCommands() |
| for r in advisor.resources: |
| output = [] |
| try: |
| stdin, stdout, stderr = self.client.exec_command(cmd[r]) |
| for line in stdout: |
| output.append(line) |
| except Exception, e: |
| self.tb.logger.error('Cannot read %s from %s', r, self.h) |
| self.tb.hosts[self.h]['data'][r] = "".join(output) |
| self.tb.logger.debug('Formatting data for %s', self.h) |
| advisor.FormatData(self.h) |
| advisor.ProcessRRD(self.h) |
| if self.tb.html: |
| self.tb.logger.debug('Building HTML files for %s', self.h) |
| advisor.BuildHTML(self.h) |
| |
| |
| class TestBed(object): |
| """Used to hold all of the testbed machine data and some global varibles. |
| |
| This class will be instantiated as a global object so that all of the other |
| classes in this module will have read/write access to it's variables. It |
| will also hold some general configuration data as well as all remote hosts |
| raw and formatted data that was collected. |
| """ |
| |
| def __init__(self, logfile, log_to_stdout, debug, graph, home, html, src, |
| update, url): |
| """ |
| Args: |
| logfile: string, name of logfile. |
| log_to_stdout: boolean, True = send log msgs to stdout. |
| debug: string, the debug log level. |
| graph: boolean, flag to create graphs. |
| home: string, pathname of root directory of monitor files. |
| html: boolean, flag to build html files. |
| src: pathname of Chrome OS src directory. |
| update: boolean, flag to get update data from remote hosts. |
| url: string, base URL of system health monitor. |
| """ |
| self.version = ['ec_firmware', 'firmware', 'release'] |
| start_time = strftime('%H:%M:%S', localtime()) |
| self.time = int(time()) |
| rundir = '/var/run/systemhealth' |
| if not os.path.isdir(rundir): |
| os.mkdir(rundir) |
| self.update_runfile = os.path.join(rundir, 'update.running') |
| self.graph_runfile = os.path.join(rundir, 'graph.running') |
| self.logfile = logfile |
| self.sshlog = os.path.join(home, 'ssh.log') |
| self.logger = SetLogger('SystemMonitor', logfile, debug, |
| log_to_stdout=log_to_stdout) |
| self.logger.info('=================================================') |
| self.logger.info('=====================Run Start===================') |
| self.logger.info('=================================================') |
| self.logger.info('Script started at: %s', start_time) |
| self.loglevel = debug |
| self.graph = graph |
| self.home = home |
| self.html = html |
| self.rrdtimes = ['-1hours', '-4hours', '-24hours', '-1week', '-1month', |
| '-1year'] |
| self.update = update |
| self.url = url |
| self.hosts = {} # Dictionary to hold data from each host. |
| |
| # Create a queue for checking resources on remote hosts. |
| self.q = TBQueue() |
| cros_keys = 'scripts/mod_for_test_scripts/ssh_keys' |
| self.privkey = os.path.join(src, cros_keys, 'testing_rsa') |
| |
| |
| class Monitor(object): |
| """Main class used to manage the monitoring of remote hosts. |
| |
| This class is used to determine the current status of hosts in the AutoTest |
| testbed. AutoTest will be queried to populate self.rhosts. It will populate |
| a Queue and start a threaded operation using the MonitorThread class, to |
| access each host in the AutoTest testbed and get resource data. |
| """ |
| |
| def __init__(self): |
| """Monitor will use config data from TestBed.""" |
| self.afe_hosts = self.LoadHosts() |
| self.threads = [] |
| |
| def LoadHosts(self): |
| """Get a list of hosnames from the AutoTest server.""" |
| # We only want AclGroup acl_cros_test, and depend on the atest tool to |
| # get the hosts from the AutoTest server. |
| # We'll return a dictionary keyed by hostname/ip address. |
| afe_hosts = {} |
| cmd = '%s/cli/atest host list -a acl_cros_test' % AUTOTEST_BIN |
| atest_out = Exec(cmd, output=True) |
| lines = atest_out.split('\n') |
| for line in lines: |
| fields = line.split() |
| if len(fields) > 4: |
| afe_hosts[fields[0]] = {} |
| afe_hosts[fields[0]]['status'] = fields[1] |
| afe_hosts[fields[0]]['locked'] = fields[2] |
| afe_hosts[fields[0]]['platform'] = fields[3] |
| afe_hosts[fields[0]]['labels'] = fields[4:] |
| # Remove atest's field labels. |
| if 'Host' in afe_hosts: |
| del afe_hosts['Host'] |
| # Set up some dictionaries for each host. |
| for host in afe_hosts: |
| TB.hosts[host] = {} |
| TB.hosts[host]['data'] = {} # Raw data from hosts. |
| TB.hosts[host]['rrddata'] = {} # Formatted data. |
| TB.hosts[host]['status'] = None |
| TB.hosts[host]['time'] = None |
| for v in TB.version: |
| TB.hosts[host][v] = {} |
| TB.hosts[host][v]['PTR'] = None |
| |
| return afe_hosts |
| |
| |
| def UpdateStatus(self): |
| """Update data from all monitored hosts.""" |
| |
| # Create new threads of class MonitorThread. |
| self.hostqty = len(self.afe_hosts) |
| for i in range(self.hostqty): |
| t = MonitorThread() |
| t.setDaemon(True) |
| self.threads.append(t) |
| t.start() |
| |
| # Fill the requests queue with AutoTest host objects. |
| for host in self.afe_hosts: |
| TB.logger.debug('Placing %s in host queue.', host) |
| TB.q.put(host) |
| |
| |
| if TB.graph: |
| # Graphing takes much longer, so increase the max runtime. |
| maxtime = RUNTIME * 6 |
| else: |
| maxtime = RUNTIME |
| # Queue.join() will wait for all jobs in the queue to finish, or |
| # until the timeout value is reached. |
| TB.logger.debug('Joining run queue.') |
| TB.q.join(timeout=maxtime) |
| TB.logger.info('%s hosts left in host queue', TB.q.qsize()) |
| TB.logger.info('Runtime is %d seconds', (time() - TB.time)) |
| |
| loglevel = TB.logger.getEffectiveLevel() |
| if loglevel == 10: |
| for host in self.afe_hosts: |
| TB.logger.debug('%s status is %s', host, |
| TB.hosts[host]['status']) |
| |
| # If there are any running threads, give them time to finish. |
| # Since paramiko is multi threaded, this check is needed. A deadline |
| # will be set since we don't want the process to run indefinitely. |
| # A deadline needs to be set because one or more of the paramiko |
| # threads could hang. |
| deadline = time() + maxtime |
| running = True |
| while running: |
| for t in self.threads: |
| if t.isAlive(): |
| continue |
| else: |
| self.threads.remove(t) |
| if len(self.threads) == 0: |
| TB.logger.info('All threads have completed their jobs.') |
| |
| running = False |
| else: |
| TB.logger.debug('%d threads still running.', len(self.threads)) |
| sleep(TIMEOUT) |
| waittime = deadline - time() |
| if waittime < 0: |
| TB.logger.info('Deadline reached. Aborting running threads.') |
| TB.logger.info('%d threads still running!', len(self.threads)) |
| running = False |
| |
| |
| def CheckStatus(self, hostname): |
| """Check the status of one host. |
| |
| Args: |
| hostname: hostname or ip address of host to check. |
| This method is primarily used for debugging purposes. |
| """ |
| t = MonitorThread() |
| t.setDaemon(True) |
| t.start() |
| |
| for host in self.afe_hosts: |
| if host == hostname: |
| TB.q.put(host) |
| break |
| TB.q.join(timeout=TIMEOUT) |
| TB.logger.info('%s status is %s', hostname, |
| TB.hosts[hostname]['status']) |
| |
| |
| def ShowData(self): |
| """Show raw data collected from each host.""" |
| |
| for host in self.afe_hosts: |
| TB.logger.info('Hostname: %s', host) |
| for k in TB.hosts[host]['data']: |
| TB.logger.info('%s: %s' , k, TB.hosts[host]['data'][k]) |
| |
| |
| def ValidIP(self, address): |
| """Verify address is a valid IP address. |
| |
| Args: |
| address: string. |
| Returns: |
| boolean: True = valid IP address, False = not valid IP address. |
| """ |
| octets = address.split('.') |
| if len(octets) != 4: |
| return False |
| for octet in octets: |
| if not 0 <= int(octet) <= 255: |
| return False |
| return True |
| |
| |
| def SortAFEHosts(self, afelist): |
| """Sort AFE host list by IP address. |
| |
| Args: |
| afelist: list of AFE host objects. |
| Returns: |
| newlist: list of sorted AFE host objects. |
| """ |
| iplist = [] |
| hostlist = [] |
| |
| for h in afelist: |
| if self.ValidIP(h): |
| iplist.append(h) |
| else: |
| hostlist.append(h) |
| |
| templist = [(IP(h).int(), h) for h in iplist] |
| templist.sort() |
| newlist = [h[1] for h in templist] |
| hostlist.sort() |
| newlist.extend(hostlist) |
| |
| return newlist |
| |
| |
| def BuildLandingPage(self): |
| """Build the initial HTML landing page with links to all hosts.""" |
| TB.logger.debug('Building Langing Page') |
| sorted_ip = [] |
| sorted_hosts = [] |
| downhosts = 0 |
| readyhosts = 0 |
| hostlist = self.afe_hosts.keys() |
| sorted_ip = self.SortAFEHosts(hostlist) |
| # Put host that are down first |
| for h in sorted_ip: |
| if not TB.hosts[h]['status']: |
| sorted_hosts.insert(downhosts, h) |
| downhosts = downhosts + 1 |
| else: |
| sorted_hosts.append(h) |
| readyhosts = readyhosts + 1 |
| |
| # Create symlink to the log file if it does not exist. |
| if TB.graph: |
| log_filename = os.path.join(TB.home, 'monitor-graph.log') |
| else: |
| log_filename = os.path.join(TB.home, 'monitor.log') |
| if not os.path.isfile(log_filename): |
| try: |
| os.symlink(TB.logfile, log_filename) |
| except OSError, e: |
| TB.logger.error('Error linking to logfile\n%s', e) |
| LandPageFile = os.path.join(TB.home, 'index.html') |
| # The temp file is used so that there will always be viewable html page |
| # when the new page is being built. |
| LandPageTemp = os.path.join(TB.home, 'temp.html') |
| f = open(LandPageTemp, 'w') |
| f.write('<HTML><HEAD>') |
| f.write('<LINK REL="stylesheet" TYPE="text/css" HREF="table.css">') |
| f.write('<center><TITLE>AutoTest System Health Check</TITLE></HEAD>') |
| f.write('<BODY>') |
| f.write('<img src="chrome.png" style="float:left;"/>') |
| f.write('<table style="float: right">') |
| f.write('<TR><TD><em>Total Hosts</em><TD>%d' % (downhosts + readyhosts)) |
| f.write('<TR><TD><em>Inaccessible Hosts</em><TD>%d' % downhosts) |
| f.write('<TR><TD><em>Accessible Hosts</em><TD>%d' % readyhosts) |
| f.write('<TR><TD><em>Update Log<em><TD><a href=%s>%s</a>' % ( |
| 'monitor.log', 'log')) |
| f.write('<TR><TD><em>Graphing Log<em><TD><a href=%s>%s</a>' % ( |
| 'monitor-graph.log', 'log')) |
| f.write('<TR><TD><em>Connection Log<em><TD><a href=%s>%s</a>' % ( |
| 'ssh.log', 'log')) |
| f.write('</table>') |
| f.write('<H1>CAUTOTEST Testbed</H1>') |
| f.write('<H2>System Health</H2>') |
| f.write('<BR><BR>') |
| f.write('<HR>') |
| f.write('<table>') |
| f.write('<CAPTION><EM>Graphs updated every hour</EM></CAPTION>') |
| f.write('<TR><TH>Hostname<TH>Status<TH>Labels<TH>Last Update') |
| f.write('<TH>Release<TH>Health</TR>') |
| for h in sorted_hosts: |
| link_dir = 'hosts/' + h + '/rrd' |
| rrd_dir = os.path.join(TB.home, 'hosts', h, 'rrd') |
| downfile = os.path.join(rrd_dir, 'downtime') |
| fqn = 'http://cautotest.corp.google.com/' |
| view_host = 'afe/#tab_id=view_host&object_id=%s' % h |
| hlink = fqn + view_host |
| if not os.path.isdir(rrd_dir): |
| os.makedirs(rrd_dir) |
| os.chmod(rrd_dir, 0755) |
| if TB.hosts[h]['status']: |
| if os.path.isfile(downfile): |
| try: |
| os.remove(downfile) |
| except OSError, e: |
| TB.logger.error('Error deleting %s\n%s', downfile, e) |
| if self.afe_hosts[h]['status'] == 'Running': |
| status = 'Running' |
| bgcolor = '#99CCFF' |
| else: |
| status = 'Ready' |
| bgcolor = '#FFFFFF' |
| else: |
| if os.path.isfile(downfile): |
| status = 'Down' |
| bgcolor = '#FF9999' |
| df = open(downfile, 'r') |
| TB.hosts[h]['time'] = df.read() |
| df.close() |
| else: |
| status = 'Unknown' |
| bgcolor = '#E8E8E8' |
| df = open(downfile, 'w') |
| df.write(TB.hosts[h]['time']) |
| df.close() |
| |
| f.write('<tr bgcolor=%s><th>' % bgcolor) |
| f.write('<a href=%s>%s</a></th>' % (hlink, h)) |
| f.write('<td><em>%s</em>' % status) |
| f.write('<td>') |
| f.write('<em><b>%s</b></em><br>' % self.afe_hosts[h]['platform']) |
| for label in self.afe_hosts[h]['labels']: |
| f.write('%s<br>' % label) |
| f.write('<td>%s' % TB.hosts[h]['time']) |
| if TB.hosts[h]['release']['PTR']: |
| f.write('<td>%s' % TB.hosts[h]['release']['PTR']) |
| else: |
| f.write('<td>Unknown') |
| index_file = os.path.join(rrd_dir, 'index.html') |
| if os.path.isfile(index_file): |
| f.write('<td><a href=%s' % TB.url) |
| f.write('%s/index.html target="_blank">' % link_dir) |
| f.write('health</a></td>') |
| else: |
| f.write('<td>None</td>') |
| f.write('</table><p>\n</center>\n</BODY></HTML>') |
| f.close() |
| shutil.copyfile(LandPageTemp, LandPageFile) |
| os.chmod(LandPageFile, 0644) |
| |
| |
| class Resource(object): |
| """Contains structures and methods to collect health data on hosts. |
| For each resource in self.resources, there must also be a corresponding |
| method to format the data into what RRDTool expects. |
| """ |
| |
| def __init__(self): |
| self.files = { |
| 'battery': '/proc/acpi/battery/BAT?/state', |
| 'boot': '/tmp/firmware-boot-time /tmp/uptime-login-prompt-ready', |
| 'cpu': '/proc/stat', |
| 'load': '/proc/loadavg', |
| 'memory': '/proc/meminfo', |
| 'network': '/proc/net/dev', |
| 'power': '/proc/acpi/processor/CPU0/throttling', |
| 'temp': '/proc/acpi/thermal_zone/*/temperature', |
| 'uptime': '/proc/uptime', |
| } |
| self.fs = { |
| 'rootfsA_space': '/', |
| 'rootfsA_inodes': '/', |
| 'rootfsA_stats': 'sda2 ', |
| 'rootfsB_space': '/', |
| 'rootfsB_inodes': '/', |
| 'rootfsB_stats': 'sda5 ', |
| 'stateful_space': '/mnt/stateful_partition', |
| 'stateful_inodes': '/mnt/stateful_partition', |
| 'stateful_stats': 'sda1 ', |
| } |
| self.resources = [] |
| for k in self.files: |
| self.resources.append(k) |
| for k in self.fs: |
| self.resources.append(k) |
| |
| |
| def FormatData(self, hostname): |
| """Convert collected data into the correct format for RRDTool. |
| |
| Args: |
| hostname: string, hostname of AutoTest host. |
| """ |
| |
| ParseMethod = { |
| 'battery': self.ParseBattery, |
| 'boot': self.ParseBoot, |
| 'fs': self.ParseFS, |
| 'diskstats': self.ParseDiskStats, |
| 'cpu': self.ParseStat, |
| 'load': self.ParseLoadAvg, |
| 'memory': self.ParseMemInfo, |
| 'network': self.ParseNetDev, |
| 'power': self.ParsePower, |
| 'temp': self.ParseTemp, |
| 'uptime': self.ParseUpTime, |
| } |
| |
| # method_key is used here because multiple resource keys will use the |
| # same method to parse them. |
| for key in TB.hosts[hostname]['data']: |
| method_key = key |
| if key in self.fs: |
| if '_space' in key: |
| method_key = 'fs' |
| elif '_inode' in key: |
| method_key = 'fs' |
| elif '_stats' in key: |
| method_key = 'diskstats' |
| else: |
| TB.logger.error('Error in key name of %s', key) |
| if method_key in ParseMethod: |
| if len(TB.hosts[hostname]['data'][key]) > 0: |
| ParseMethod[method_key](hostname, key) |
| else: |
| TB.logger.error('No method to parse resource %s', key) |
| |
| |
| def ProcessRRD(self, hostname): |
| """Process formatted data into RRD files. |
| |
| Args: |
| hostname: string, hostname of AutoTest host. |
| """ |
| hostdir = os.path.join(TB.home, 'hosts', hostname) |
| rrd_dir = os.path.join(hostdir, 'rrd') |
| if not os.path.exists(rrd_dir): |
| os.makedirs(rrd_dir) |
| os.chmod(rrd_dir, 0755) |
| os.chmod(hostdir, 0755) |
| for r in self.resources: |
| dk = None # datakey only needs to be set if it's a file system. |
| if r in self.fs: |
| if '_space' in r: |
| dk = 'fs_space' |
| elif '_inode' in r: |
| dk = 'fs_inode' |
| elif '_stat' in r: |
| dk = 'fs_stat' |
| |
| |
| rrd = RRD(r, hostname, rrd_dir, datakey=dk) |
| if not os.path.exists(rrd.rrdfile): |
| rrd.Create() |
| if TB.update == True: |
| TB.logger.debug('Updating %s for host %s', r, hostname) |
| rrd.Update() |
| if TB.graph: |
| TB.logger.debug('Building %s graphs for %s', r, hostname) |
| rrd.Graph() |
| |
| |
| def BuildHTML(self, hostname): |
| """Create HTML pages for to display the graphs. |
| |
| Args: |
| hostname: string, host of AutoTest host. |
| """ |
| mainindex = TB.url + 'index.html' |
| rrd_dir = os.path.join(TB.home, 'hosts', hostname, 'rrd') |
| resource_list = [] |
| for r in self.resources: |
| resource_list.append(r) |
| resource_list.sort() |
| index_file = os.path.join(rrd_dir, 'index.html') |
| html_file = {} |
| for t in TB.rrdtimes: |
| html_file[t] = hostname + t + '.html' |
| pathname = {} |
| for name in html_file: |
| pathname[name] = os.path.join(rrd_dir, html_file[name]) |
| |
| # Create HTML files for each time period we are graphing. |
| for path in pathname: |
| f = open(pathname[path], 'w') |
| f.write('<HTML><HEAD>') |
| f.write('<center><TITLE>%s System Health</TITLE></HEAD>' % |
| hostname) |
| f.write('<BODY><H1>%s System Health</H1>' % hostname) |
| for v in TB.version: |
| f.write('<H4>%s: %s</H4>' % (v, TB.hosts[hostname][v]['PTR'])) |
| for t in TB.rrdtimes: |
| f.write('<a href="%s">%s</a> <b>|</b>' % ( |
| html_file[t], t)) |
| f.write('<a href="%s">SystemHealth Home</a>' % mainindex) |
| f.write('<p><HR>') |
| |
| f.write('<b>%s</b><table border=1 bgcolor=#EEEEEE>' % path) |
| newrow = True |
| for r in resource_list: |
| if newrow: |
| f.write('<tr>') |
| f.write('<td>%s<br><a href=%s.html>' % (r, r)) |
| f.write('<img src=%s%s.png width=475 height=250></a></td>' % ( |
| r, path)) |
| if newrow: |
| newrow = False |
| else: |
| f.write('</tr>\n') |
| newrow = True |
| f.write('</table><p>\n') |
| f.write('</center>\n') |
| f.write('<H5>Last Update: %s</H5>' % TB.hosts[hostname]['time']) |
| f.write('</BODY></HTML>') |
| f.close() |
| os.chmod(pathname[path], 0644) |
| if not os.path.isfile(index_file): |
| os.symlink(pathname[TB.rrdtimes[0]], index_file) |
| |
| # Create HTML files for each resource for all time periods. |
| for r in resource_list: |
| rrdfile = os.path.join(rrd_dir, r + '.html') |
| f = open(rrdfile, 'w') |
| f.write('<HTML><HEAD>') |
| f.write('<center><TITLE>%s %s Resources</TITLE></HEAD>' % ( |
| hostname, r)) |
| f.write('<BODY><H1>%s %s Resources</H1>' % (hostname, r)) |
| for v in TB.version: |
| f.write('<H4>%s: %s</H4>' % (v, TB.hosts[hostname][v]['PTR'])) |
| f.write('<table border=5 bgcolor=#B5B5B5>') |
| f.write('<tr>') |
| for t in TB.rrdtimes: |
| f.write('<td><a href="#%s"><b>%s</b></a>' % (t, t)) |
| f.write('</table>') |
| f.write('<HR>') |
| f.write('<table border=1 bgcolor=#EEEEEE>') |
| for t in TB.rrdtimes: |
| f.write('<tr><td><a name="%s"><img src=%s%s.png>' % (t, r, t)) |
| f.write('</a></td></tr>\n') |
| f.write('</table><p>\n') |
| f.write('</center>\n') |
| f.write('<H5>Last Update: %s</H5>' % TB.hosts[hostname]['time']) |
| f.write('</BODY></HTML>') |
| f.close() |
| os.chmod(rrdfile, 0644) |
| |
| |
| def ParseBattery(self, h, k): |
| """Convert /proc/acpi/battery/BAT0/state to a list of strings. |
| |
| Args: |
| h: string, hostname of host in AutoTest. |
| k: string, resource key. |
| We only care about the values corresponding to the rrdkeys, so the other |
| values will be discarded. |
| """ |
| rrdkeys = ['charging state', 'present rate', 'remaining capacity',] |
| TB.hosts[h]['rrddata'][k] = [] |
| statlist = TB.hosts[h]['data'][k].split('\n') |
| for stat in statlist: |
| for key in rrdkeys: |
| if key in stat: |
| stats = stat.split(':') |
| temp = stats[1].split() |
| if key == 'charging state': |
| if temp[0] == 'discharging': |
| TB.hosts[h]['rrddata'][k].append('0') |
| else: |
| TB.hosts[h]['rrddata'][k].append('1') |
| else: |
| TB.hosts[h]['rrddata'][k].append(temp[0]) |
| |
| |
| def ParseBoot(self, h, k): |
| """Parse /tmp/uptime-login-prompt-ready for boot time. |
| |
| Args: |
| h: string, hostname of host in AutoTest. |
| k: string, resource key. |
| We only want the first and 2nd values from the raw data. |
| """ |
| fields = [] |
| TB.hosts[h]['rrddata'][k] = [] |
| lines = TB.hosts[h]['data'][k].split('\n') |
| for line in lines: |
| if not '==>' in line: |
| fields.extend(line.split()) |
| TB.hosts[h]['rrddata'][k] = fields[0:2] |
| |
| |
| def ParseFS(self, h, k): |
| """Convert file system space and inode readings to a list of strings. |
| |
| Args: |
| h: string, hostname of host in AutoTest. |
| k: string, resource key. |
| """ |
| TB.hosts[h]['rrddata'][k] = [] |
| lines = TB.hosts[h]['data'][k].split('\n') |
| for line in lines: |
| if not line.startswith('Filesystem'): |
| fields = line.split() |
| if len(fields) > 4: |
| TB.hosts[h]['rrddata'][k].append(fields[2]) |
| TB.hosts[h]['rrddata'][k].append(fields[3]) |
| |
| |
| def ParseDiskStats(self, h, k): |
| """Parse read and write sectors from /proc/diskstats to list of strings. |
| |
| Args: |
| h: string, hostname of host in AutoTest. |
| k: string, resource key. |
| """ |
| TB.hosts[h]['rrddata'][k] = [] |
| fields = TB.hosts[h]['data'][k].split() |
| if len(fields) > 9: |
| TB.hosts[h]['rrddata'][k].append(fields[5]) |
| TB.hosts[h]['rrddata'][k].append(fields[9]) |
| |
| |
| def ParseStat(self, h, k): |
| """Convert /proc/stat to lists for CPU usage. |
| |
| Args: |
| h: string, hostname of host in AutoTest. |
| k: string, resource key. |
| """ |
| lines = TB.hosts[h]['data'][k].split('\n') |
| for line in lines: |
| if 'cpu ' in line: |
| vals = line.split() |
| TB.hosts[h]['rrddata'][k] = vals[1:5] |
| |
| |
| def ParseLoadAvg(self, h, k): |
| """Convert /proc/loadavg to a list of strings to monitor. |
| |
| Args: |
| h: string, hostname of host in AutoTest. |
| k: string, resource key. |
| Process ID is discarded, as it's not needed. |
| """ |
| statlist = TB.hosts[h]['data'][k].split() |
| TB.hosts[h]['rrddata'][k] = statlist[0:3] |
| |
| |
| def ParseMemInfo(self, h, k): |
| """Convert specified fields in /proc/meminfo to a list of strings. |
| |
| Args: |
| h: string, hostname of host in AutoTest. |
| k: string, resoruce key. |
| """ |
| TB.hosts[h]['rrddata'][k] = [] |
| mem_keys = ['MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', |
| 'SwapFree'] |
| lines = TB.hosts[h]['data'][k].split('\n') |
| for line in lines: |
| for key in mem_keys: |
| if key in line: |
| if not 'SwapCached' in line: |
| fields = line.split() |
| TB.hosts[h]['rrddata'][k].append(fields[1]) |
| |
| |
| def ParseNetDev(self, h, k): |
| """Convert /proc/net/dev to a list of strings of rec and xmit values. |
| |
| Args: |
| h: string, hostname of host in AutoTest. |
| k: string, resource key. |
| """ |
| net_keys = ['eth0', 'wlan0'] |
| rrdlist = ['0', '0', '0', '0'] |
| lines = TB.hosts[h]['data'][k].split('\n') |
| for key in net_keys: |
| for line in lines: |
| if key in line: |
| # The following routine will ensure that the values are |
| # placed in the correct order in case there is an expected |
| # interface is not present. |
| index = net_keys.index(key) |
| if index: |
| index = index*2 |
| (device, data) = line.split(':') |
| fields = data.split() |
| rrdlist[index] = fields[0] |
| rrdlist[index+1] = fields[8] |
| |
| TB.hosts[h]['rrddata'][k] = rrdlist |
| |
| |
| def ParsePower(self, h, k): |
| """Convert /proc/acpi/processor/CPU0/throttling to power percentage. |
| |
| Args: |
| h: string, hostname of host in AutoTest. |
| k: string, resource key. |
| """ |
| TB.hosts[h]['rrddata'][k] = [] |
| lines = TB.hosts[h]['data'][k].split('\n') |
| for line in lines: |
| line = line.strip() |
| if line.startswith('*'): |
| fields = line.split(':') |
| |
| if len(fields) > 1: |
| percent = fields[1].strip('%') |
| percent = percent.strip() |
| TB.hosts[h]['rrddata'][k].append(percent) |
| |
| |
| def ParseTemp(self, h, k): |
| """Convert temperature readings to a list of strings. |
| |
| Args: |
| h: string, hostname of host in AutoTest. |
| k: string, resource key. |
| """ |
| TB.hosts[h]['rrddata'][k] = [] |
| statlist = TB.hosts[h]['data'][k].split() |
| if len(statlist) > 1: |
| TB.hosts[h]['rrddata'][k].append(statlist[1]) |
| |
| |
| def ParseUpTime(self, h, k): |
| """Convert /proc/uptime to a list of strings. |
| |
| Args: |
| h: string, hostname of host in AutoTest. |
| k: string, resource key. |
| Returns: |
| list of strings. |
| """ |
| |
| TB.hosts[h]['rrddata'][k] = TB.hosts[h]['data'][k].split() |
| |
| |
| def GetCommands(self): |
| """Routine for gathering data from files and file systems. |
| Returns: |
| dictionary of commands to run on hosts. |
| """ |
| |
| command = {} |
| |
| for r in self.resources: |
| if r in self.files: |
| if r == 'boot': |
| command[r] = 'head %s' % self.files[r] |
| else: |
| command[r] = 'cat %s' % self.files[r] |
| elif r in self.fs: |
| if '_space' in r: |
| command[r] = 'df -lP %s' % self.fs[r] |
| elif '_inode' in r: |
| command[r] = 'df -iP %s' % self.fs[r] |
| elif '_stat' in r: |
| command[r] = 'cat /proc/diskstats | grep %s' % self.fs[r] |
| else: |
| TB.logger.error('Error in key name of %s', r) |
| return command |
| |
| |
| class RRD(object): |
| """The class to create and update RRD data stores and graph them. |
| |
| This class should be used to access all of the functions of RRDTool. It will |
| create the data files, update them, and create graphs/charts based on that |
| data. Datakey is needed when we are using the same data definitions for many |
| items of the same type, like file systems. |
| """ |
| def __init__(self, rrdname, hostname, rrd_dir, datakey=None): |
| """ |
| Args: |
| rrdname: string, item name(should match key from Resources) |
| hostname: string, hostname of the machine. |
| rrd_dir: string, directory for all rrd files and graphs. |
| datakey: string, overrides which data definition to use. |
| """ |
| self.rrdtool = '/usr/bin/rrdtool' |
| self.rrd_dir = rrd_dir |
| if not os.path.exists(self.rrd_dir): |
| try: |
| os.makedirs(self.rrd_dir) |
| os.chmod(self.rrd_dir, 0755) |
| except OSError: |
| TB.logger.error('Error creating %s', self.rrd_dir) |
| self.rrdname = rrdname |
| self.hostname = hostname |
| rrd_filename = rrdname + '.rrd' |
| self.rrdfile = os.path.join(self.rrd_dir, rrd_filename) |
| file_system = 'Unknown' |
| |
| if not datakey: |
| datakey = rrdname |
| else: |
| fields = rrdname.split('_') |
| if len(fields[0]) > 0: |
| file_system = fields[0] |
| |
| data_def = { |
| 'battery': { |
| 'heartbeat': '600', |
| 'min': '0', |
| 'max': 'U', |
| 'title': '"%s Battery Status' % self.hostname, |
| 'type': 'GAUGE', |
| 'units': '"Mili Amps"', |
| 'items': [ |
| 'State', |
| 'Rate', |
| 'Capacity', |
| ], |
| 'graph': [ |
| '-l 0 -r', |
| 'CDEF:bat=State,1,LT,50,UNKN,IF', |
| 'CDEF:ac=State,1,LT,UNKN,50,IF', |
| 'CDEF:RateD=State,1,LT,Rate,UNKN,IF', |
| 'CDEF:RateC=State,1,LT,UNKN,Rate,IF', |
| 'CDEF:bg=Capacity,UN,0,Capacity,IF,0,GT,INF,UNKN,IF', |
| 'AREA:bg#DDDDDD:', |
| 'AREA:Capacity#99CCFF:"Capacity "', |
| 'LINE1:Capacity#3399FF:', |
| 'VDEF:max1=Capacity,MAXIMUM', |
| 'VDEF:min1=Capacity,MINIMUM', |
| 'VDEF:avg1=Capacity,AVERAGE', |
| 'GPRINT:max1:"Max %6.3lf%s"', |
| 'GPRINT:min1:"Min %6.3lf%s"', |
| 'GPRINT:avg1:" Avg %6.3lf%s"', |
| 'AREA:bat#CC0000:"Battery \\n"', |
| 'LINE2:RateD#990033:"Discharge Rate"', |
| 'VDEF:max2=RateD,MAXIMUM', |
| 'VDEF:min2=RateD,MINIMUM', |
| 'VDEF:avg2=RateD,AVERAGE', |
| 'GPRINT:max2:"Max %6.3lf%s"', |
| 'GPRINT:min2:"Min %6.3lf%s"', |
| 'GPRINT:avg2:"Avg %6.3lf%s"', |
| 'AREA:ac#33FF66:"AC Connected \\n"', |
| 'LINE2:RateC#009966:"Charge Rate "', |
| 'VDEF:max3=RateC,MAXIMUM', |
| 'VDEF:min3=RateC,MINIMUM', |
| 'VDEF:avg3=RateC,AVERAGE', |
| 'GPRINT:max3:"Max %6.3lf%s"', |
| 'GPRINT:min3:"Min %6.3lf%s"', |
| 'GPRINT:avg3:" Avg %6.3lf%s\\n"', |
| ], |
| }, |
| 'boot': { |
| 'heartbeat': '600', |
| 'min': '0', |
| 'max': 'U', |
| 'title': '"%s Boot time to Login Prompt' % self.hostname, |
| 'type': 'GAUGE', |
| 'units': '"Seconds"', |
| 'items': [ |
| 'firmware', |
| 'ready', |
| ], |
| 'graph': [ |
| '-l 0 -u 30 -r', |
| 'CDEF:total=firmware,ready,+', |
| 'CDEF:bg=total,UN,0,total,IF,0,GT,UNKN,INF,IF', |
| 'AREA:bg#DDDDDD:', |
| 'AREA:firmware#26466D:"Firmware "', |
| 'LINE1:firmware#660000:', |
| 'VDEF:maxF=firmware,MAXIMUM', |
| 'VDEF:minF=firmware,MINIMUM', |
| 'VDEF:avgF=firmware,AVERAGE', |
| 'GPRINT:minF:"Min %2.1lf"', |
| 'GPRINT:maxF:"Max %2.1lf"', |
| 'GPRINT:avgF:"Avg %2.1lf Seconds \\n"', |
| 'AREA:ready#0BB5FF:"Login Prompt":STACK', |
| 'LINE1:firmware#660000:', |
| 'VDEF:maxR=ready,MAXIMUM', |
| 'VDEF:minR=ready,MINIMUM', |
| 'VDEF:avgR=ready,AVERAGE', |
| 'GPRINT:minR:"Min %2.1lf"', |
| 'GPRINT:maxR:"Max %2.1lf"', |
| 'GPRINT:avgR:"Avg %2.1lf Seconds \\n"', |
| 'VDEF:maxT=total,MAXIMUM', |
| 'VDEF:minT=total,MINIMUM', |
| 'VDEF:avgT=total,AVERAGE', |
| 'GPRINT:minT:"Total Min %2.1lf"', |
| 'GPRINT:maxT:"Max %2.1lf"', |
| 'GPRINT:avgT:"Avg %2.1lf Seconds \\n"', |
| 'HRULE:15#FF0000', |
| 'HRULE:10#FFA500', |
| ], |
| }, |
| 'cpu': { |
| 'heartbeat': '600', |
| 'min': '0', |
| 'max': 'U', |
| 'title': '"%s CPU Usage' % self.hostname, |
| 'type': 'DERIVE', |
| 'units': 'jiffies', |
| 'items': [ |
| 'user', |
| 'nice', |
| 'system', |
| 'idle', |
| ], |
| 'graph': [ |
| '-l 0 -r -u 99.99', |
| 'CDEF:l=user,0.1,0.1,IF', |
| 'CDEF:bg=user,UN,0,user,IF,0,GT,UNKN,INF,IF', |
| 'AREA:bg#DDDDDD:', |
| 'CDEF:tj=user,nice,+,system,+,idle,+', |
| 'CDEF:usr=100,user,*,tj,/', |
| 'CDEF:nic=100,nice,*,tj,/', |
| 'CDEF:sys=100,system,*,tj,/', |
| 'CDEF:idl=100,idle,*,tj,/', |
| 'CDEF:tot=100,tj,*,tj,/', |
| 'AREA:nic#0040A2:"Nice "', |
| 'VDEF:maxN=nic,MAXIMUM', |
| 'VDEF:minN=nic,MINIMUM', |
| 'VDEF:avgN=nic,AVERAGE', |
| 'GPRINT:maxN:"Max %6.2lf%s"', |
| 'GPRINT:minN:"Min %6.2lf%s"', |
| 'GPRINT:avgN:"Avg %6.2lf%s \\n"', |
| 'AREA:sys#3399FF:System:STACK', |
| 'LINE2:l#70A5AC::STACK', |
| 'VDEF:maxS=sys,MAXIMUM', |
| 'VDEF:minS=sys,MINIMUM', |
| 'VDEF:avgS=sys,AVERAGE', |
| 'GPRINT:maxS:"Max %6.2lf%s"', |
| 'GPRINT:minS:"Min %6.2lf%s"', |
| 'GPRINT:avgS:"Avg %6.2lf%s \\n"', |
| 'AREA:usr#B0F5EC:"User ":STACK', |
| 'LINE2:l#90C5CC::STACK', |
| 'VDEF:maxU=usr,MAXIMUM', |
| 'VDEF:minU=usr,MINIMUM', |
| 'VDEF:avgU=usr,AVERAGE', |
| 'GPRINT:maxU:"Max %6.2lf%s"', |
| 'GPRINT:minU:"Min %6.2lf%s"', |
| 'GPRINT:avgU:"Avg %6.2lf%s \\n"', |
| 'AREA:idl#EEFFFF:"Idle ":STACK', |
| 'VDEF:maxI=idl,MAXIMUM', |
| 'VDEF:minI=idl,MINIMUM', |
| 'VDEF:avgI=idl,AVERAGE', |
| 'GPRINT:maxI:"Max %6.2lf%s"', |
| 'GPRINT:minI:"Min %6.2lf%s"', |
| 'GPRINT:avgI:"Avg %6.2lf%s \\n"', |
| ], |
| }, |
| 'fs_inode': { |
| 'heartbeat': '600', |
| 'min': '0', |
| 'max': 'U', |
| 'title': '"%s %s File System Inodes' % (self.hostname, |
| file_system), |
| 'type': 'GAUGE', |
| 'units': 'Quantity', |
| 'items': [ |
| 'Used', |
| 'Free', |
| ], |
| 'graph': [ |
| '-l 0 -r', |
| 'CDEF:bg=Used,UN,0,Used,IF,0,GT,UNKN,INF,IF', |
| 'AREA:bg#DDDDDD', |
| 'CDEF:inodes=Used,Free,+', |
| 'VDEF:inodesTotal=inodes,LAST', |
| 'GPRINT:inodesTotal:"Total %6.2lf %s\\n"', |
| 'AREA:Used#000066:"Used"', |
| 'VDEF:usedLast=Used,LAST', |
| 'GPRINT:usedLast:"%6.2lf %s"', |
| 'CDEF:usedPct=Used,100,*,inodes,/', |
| 'VDEF:pctUsed=usedPct,LAST', |
| 'GPRINT:pctUsed:"%6.2lf%%\\n"', |
| 'AREA:Free#3399FF:"Free":STACK', |
| 'VDEF:freeLast=Free,LAST', |
| 'GPRINT:freeLast:"%6.2lf %s"', |
| 'CDEF:freePct=100,usedPct,-', |
| 'VDEF:pctFree=freePct,LAST', |
| 'GPRINT:pctFree:"%6.2lf%%\\n"', |
| ], |
| }, |
| 'fs_space': { |
| 'heartbeat': '600', |
| 'min': '0', |
| 'max': 'U', |
| 'title': '"%s %s File System Space' % (self.hostname, |
| file_system), |
| 'type': 'GAUGE', |
| 'units': 'Bytes', |
| 'items': [ |
| 'Used', |
| 'Free', |
| ], |
| 'graph': [ |
| '-l 0 -r', |
| 'CDEF:bg=Used,UN,0,Used,IF,0,GT,UNKN,INF,IF', |
| 'AREA:bg#DDDDDD', |
| 'CDEF:UsedB=Used,1024,*', |
| 'CDEF:FreeB=Free,1024,*', |
| 'CDEF:fs=UsedB,FreeB,+', |
| 'VDEF:fsTotal=fs,LAST', |
| 'GPRINT:fsTotal:"Total %6.2lf %sB\\n"', |
| 'AREA:UsedB#003399:"Used"', |
| 'VDEF:usedLast=UsedB,LAST', |
| 'GPRINT:usedLast:"%6.2lf %sB"', |
| 'CDEF:usedPct=UsedB,100,*,fs,/', |
| 'VDEF:pctUsed=usedPct,LAST', |
| 'GPRINT:pctUsed:"%6.2lf%%\\n"', |
| 'AREA:FreeB#6699CC:"Free":STACK', |
| 'VDEF:freeLast=FreeB,LAST', |
| 'GPRINT:freeLast:"%6.2lf %sB"', |
| 'CDEF:freePct=100,usedPct,-', |
| 'VDEF:pctFree=freePct,LAST', |
| 'GPRINT:pctFree:"%6.2lf%%\\n"', |
| ], |
| }, |
| 'fs_stat': { |
| 'heartbeat': '600', |
| 'min': 'U', |
| 'max': 'U', |
| 'title': '"%s %s File System Activity' % (self.hostname, |
| file_system), |
| 'type': 'DERIVE', |
| 'units': '"Bytes"', |
| 'items': [ |
| 'Reads', |
| 'Writes', |
| ], |
| 'graph': [ |
| '-r', |
| 'CDEF:bWrites=Writes,-512,*', |
| 'CDEF:bReads=Reads,512,*', |
| 'AREA:bWrites#990000:"Bytes Written\\n"', |
| 'AREA:bReads#0066CC:"Bytes Read"', |
| 'HRULE:0#000000', |
| ], |
| }, |
| 'load': { |
| 'heartbeat': '600', |
| 'min': '0', |
| 'max': '100', |
| 'title': '"%s Load Levels' % self.hostname, |
| 'type': 'GAUGE', |
| 'units': 'proc/min', |
| 'items': [ |
| 'load_1', |
| 'load_5', |
| 'load_15', |
| ], |
| 'graph': [ |
| '-r', |
| 'CDEF:bg=load_1,UN,0,load_1,IF,0,GT,UNKN,INF,IF', |
| 'AREA:bg#DDDDDD:', |
| 'CDEF:bi=load_1,UN,0,load_1,IF,0,GT,INF,UNKN,IF', |
| 'AREA:bi#FEFEED:', |
| 'HRULE:1.0#44B5FF', |
| 'AREA:load_15#99FFCC:"Last 15 min"', |
| 'VDEF:max3=load_15,MAXIMUM', |
| 'VDEF:min3=load_15,MINIMUM', |
| 'VDEF:avg3=load_15,AVERAGE', |
| 'GPRINT:max3:"Max %6.2lf"', |
| 'GPRINT:min3:"Min %6.2lf"', |
| 'GPRINT:avg3:"Avg %6.2lf\\n"', |
| 'LINE2:load_5#3399FF:"Last 5 min "', |
| 'VDEF:max2=load_5,MAXIMUM', |
| 'VDEF:min2=load_5,MINIMUM', |
| 'VDEF:avg2=load_5,AVERAGE', |
| 'GPRINT:max2:"Max %6.2lf"', |
| 'GPRINT:min2:"Min %6.2lf"', |
| 'GPRINT:avg2:"Avg %6.2lf\\n"', |
| 'LINE2:load_1#993366:"Last 1 min "', |
| 'VDEF:max1=load_1,MAXIMUM', |
| 'VDEF:min1=load_1,MINIMUM', |
| 'VDEF:avg1=load_1,AVERAGE', |
| 'GPRINT:max1:"Max %6.2lf"', |
| 'GPRINT:min1:"Min %6.2lf"', |
| 'GPRINT:avg1:"Avg %6.2lf\\n"', |
| ], |
| }, |
| 'memory': { |
| 'heartbeat': '600', |
| 'min': '0', |
| 'max': '10000000', |
| 'title': '"%s Memory Usage' % self.hostname, |
| 'type': 'GAUGE', |
| 'units': 'bytes', |
| 'items': [ |
| 'MemTotal', |
| 'MemFree', |
| 'Buffers', |
| 'Cached', |
| 'SwapTotal', |
| 'SwapFree', |
| ], |
| 'graph': [ |
| '-r', |
| 'CDEF:bg=MemTotal,UN,0,MemTotal,IF,0,GT,UNKN,INF,IF', |
| 'AREA:bg#DDDDDD:', |
| 'CDEF:sum=MemTotal,1024,*', |
| 'CDEF:free=MemFree,1024,*', |
| 'CDEF:buff=Buffers,1024,*', |
| 'CDEF:buffP=buff,100,*,sum,/', |
| 'CDEF:cache=Cached,1024,*', |
| 'CDEF:user=MemTotal,MemFree,Cached,+,Buffers,+,-,1024,*', |
| 'CDEF:l=user,1,1,IF', |
| 'AREA:user#003366:"User "', |
| 'LINE2:l#AC1300::STACK', |
| 'VDEF:maxUser=user,MAXIMUM', |
| 'VDEF:minUser=user,MINIMUM', |
| 'VDEF:avgUser=user,AVERAGE', |
| 'VDEF:curUser=user,LAST', |
| 'GPRINT:curUser:"Last %6.2lf %s"', |
| 'GPRINT:avgUser:"Avg %6.2lf %s"', |
| 'GPRINT:maxUser:"Max %6.2lf %s"', |
| 'GPRINT:minUser:"Min %6.2lf %s\\n"', |
| 'AREA:cache#336699:"Cached ":STACK', |
| 'LINE2:l#DF7900::STACK', |
| 'VDEF:maxCache=cache,MAXIMUM', |
| 'VDEF:minCache=cache,MINIMUM', |
| 'VDEF:avgCache=cache,AVERAGE', |
| 'VDEF:curCache=cache,LAST', |
| 'GPRINT:curCache:"Last %6.2lf %s"', |
| 'GPRINT:avgCache:"Avg %6.2lf %s"', |
| 'GPRINT:maxCache:"Max %6.2lf %s"', |
| 'GPRINT:minCache:"Min %6.2lf %s\\n"', |
| 'AREA:buff#99CCFF:"Buffers":STACK', |
| 'LINE2:l#DFAC00::STACK', |
| 'VDEF:maxBuff=buff,MAXIMUM', |
| 'VDEF:minBuff=buff,MINIMUM', |
| 'VDEF:avgBuff=buff,AVERAGE', |
| 'VDEF:curBuff=buff,LAST', |
| 'GPRINT:curBuff:"Last %6.2lf %s"', |
| 'GPRINT:avgBuff:"Avg %6.2lf %s"', |
| 'GPRINT:maxBuff:"Max %6.2lf %s"', |
| 'GPRINT:minBuff:"Min %6.2lf %s\\n"', |
| 'AREA:free#CCFFCC:"Unused ":STACK', |
| 'VDEF:maxFree=free,MAXIMUM', |
| 'VDEF:minFree=free,MINIMUM', |
| 'VDEF:avgFree=free,AVERAGE', |
| 'VDEF:curFree=free,LAST', |
| 'GPRINT:curFree:"Last %6.2lf %s"', |
| 'GPRINT:avgFree:"Avg %6.2lf %s"', |
| 'GPRINT:maxFree:"Max %6.2lf %s"', |
| 'GPRINT:minFree:"Min %6.2lf %s\\n"', |
| ], |
| }, |
| 'network': { |
| 'heartbeat': '600', |
| 'min': '0', |
| 'max': '12500000', |
| 'title': '"%s Network Traffic' % self.hostname, |
| 'type': 'DERIVE', |
| 'units': 'bytes/s', |
| 'items': [ |
| 'r_eth0', |
| 'x_eth0', |
| 'r_wlan0', |
| 'x_wlan0', |
| ], |
| 'graph': [ |
| '-r', |
| 'VDEF:max1=r_eth0,MAXIMUM', |
| 'CDEF:eoff=r_eth0,UN,0,r_eth0,IF,0,GT,UNKN,0,IF', |
| 'CDEF:eon=0,r_eth0,UN,0,r_eth0,IF,0,GT,max1,50,/,UNKN,IF,-', |
| 'CDEF:bi=r_eth0,UN,0,r_eth0,IF,0,GT,INF,UNKN,IF', |
| 'CDEF:bg=r_eth0,UN,0,r_eth0,IF,0,GT,UNKN,INF,IF', |
| 'AREA:bi#DDDDDD:', |
| 'AREA:r_eth0#000066:"Eth0 In "', |
| 'LINE1:r_eth0#0000CC:', |
| 'VDEF:min1=r_eth0,MINIMUM', |
| 'VDEF:avg1=r_eth0,AVERAGE', |
| 'VDEF:tot1=r_eth0,TOTAL', |
| 'GPRINT:max1:"Max %6.2lf%s"', |
| 'GPRINT:min1:"Min %6.2lf%s"', |
| 'GPRINT:avg1:"Avg %6.2lf%s"', |
| 'GPRINT:tot1:"Sum %6.2lf%s\\n"', |
| 'CDEF:xmit0=x_eth0,-1,*', |
| 'AREA:xmit0#990033:"Eth0 Out"', |
| 'VDEF:max2=x_eth0,MAXIMUM', |
| 'VDEF:min2=x_eth0,MINIMUM', |
| 'VDEF:avg2=x_eth0,AVERAGE', |
| 'VDEF:tot2=x_eth0,TOTAL', |
| 'GPRINT:max2:"Max %6.2lf%s"', |
| 'GPRINT:min2:"Min %6.2lf%s"', |
| 'GPRINT:avg2:"Avg %6.2lf%s"', |
| 'GPRINT:tot2:"Sum %6.2lf%s\\n"', |
| 'AREA:bg#DDDDDD:', |
| 'LINE3:eoff#000000:"Eth0 Offline \\n"', |
| 'LINE3:eon#00CC66:"Eth0 Online \\n"', |
| 'AREA:r_wlan0#6699CC:"Wlan0 In "', |
| 'VDEF:min3=r_wlan0,MINIMUM', |
| 'VDEF:max3=r_wlan0,MAXIMUM', |
| 'VDEF:avg3=r_wlan0,AVERAGE', |
| 'VDEF:tot3=r_wlan0,TOTAL', |
| 'GPRINT:max3:"Max %6.2lf%s"', |
| 'GPRINT:min3:"Min %6.2lf%s"', |
| 'GPRINT:avg3:"Avg %6.2lf%s"', |
| 'GPRINT:tot3:"Sum %6.2lf%s\\n"', |
| 'CDEF:xmit1=x_wlan0,-1,*', |
| 'AREA:xmit1#FF6666:"Wlan0 Out"', |
| 'VDEF:max4=x_wlan0,MAXIMUM', |
| 'VDEF:min4=x_wlan0,MINIMUM', |
| 'VDEF:avg4=x_wlan0,AVERAGE', |
| 'VDEF:tot4=x_wlan0,TOTAL', |
| 'GPRINT:max4:"Max %6.2lf%s"', |
| 'GPRINT:min4:"Min %6.2lf%s"', |
| 'GPRINT:avg4:"Avg %6.2lf%s"', |
| 'GPRINT:tot4:"Sum %6.2lf%s\\n"', |
| ], |
| }, |
| 'power': { |
| 'heartbeat': '600', |
| 'min': '0', |
| 'max': '100', |
| 'title': '"%s Power State' % self.hostname, |
| 'type': 'GAUGE', |
| 'units': 'Percentage', |
| 'items': ['state',], |
| 'graph': [ |
| '-l 0 -r', |
| 'CDEF:bg=state,UN,0,state,IF,0,GT,UNKN,INF,IF', |
| 'AREA:bg#DDDDDD:', |
| 'VDEF:pstate=state,LAST', |
| 'AREA:pstate#CC3333:"Power Setting "', |
| 'VDEF:pstateMax=state,MAXIMUM', |
| 'VDEF:pstateMin=state,MINIMUM', |
| 'VDEF:pstateAvg=state,AVERAGE', |
| 'GPRINT:pstateMax:"Max %6.2lf%s%%"', |
| 'GPRINT:pstateMin:"Min %6.2lf%s%%"', |
| 'GPRINT:pstateAvg:"Avg %6.2lf%s%%\\n"', |
| ], |
| }, |
| 'temp': { |
| 'heartbeat': '600', |
| 'min': '0', |
| 'max': '100', |
| 'title': '"%s Temperature Readings' % self.hostname, |
| 'type': 'GAUGE', |
| 'units': 'Celsius', |
| 'items': [ |
| 'cpu', |
| ], |
| 'graph': [ |
| '-l 20 -r', |
| 'CDEF:bg=cpu,UN,0,cpu,IF,0,GT,UNKN,INF,IF', |
| 'AREA:bg#DDDDDD:', |
| 'CDEF:cool=cpu,40,LE,cpu,UNKN,IF', |
| 'CDEF:warm=cpu,40,60,LIMIT', |
| 'CDEF:hot=cpu,60,GE,cpu,UNKN,IF', |
| 'AREA:cool#B0F5EC:"Cool "', |
| 'AREA:warm#FFCC00:"Warm "', |
| 'AREA:hot#CC3300:"Hot \\n"', |
| 'VDEF:maxC=cpu,MAXIMUM', |
| 'VDEF:minC=cpu,MINIMUM', |
| 'VDEF:avgC=cpu,AVERAGE', |
| 'GPRINT:minC:"Min %2.1lf"', |
| 'GPRINT:maxC:"Max %2.1lf"', |
| 'GPRINT:avgC:"Avg %2.1lf Celsius \\n"', |
| 'LINE1:cpu#660000:', |
| 'HRULE:60#FF0000', |
| 'HRULE:20#FFA500', |
| ], |
| }, |
| 'uptime': { |
| 'heartbeat': '600', |
| 'min': '0', |
| 'max': 'U', |
| 'title': '"%s Uptime Readings' % self.hostname, |
| 'type': 'GAUGE', |
| 'units': 'hours', |
| 'items': [ |
| 'uptime', |
| 'idletime', |
| ], |
| 'graph': [ |
| '-r', |
| 'CDEF:bg=uptime,UN,0,uptime,IF,0,GT,UNKN,INF,IF', |
| 'AREA:bg#DDDDDD:', |
| 'CDEF:upHours=uptime,3600,/', |
| 'CDEF:idleHours=idletime,3600,/', |
| 'AREA:upHours#99CC99:"Uptime "', |
| 'GPRINT:upHours:MIN:"Min %8.2lf"', |
| 'GPRINT:upHours:MAX:"Max %8.2lf"', |
| 'GPRINT:upHours:AVERAGE:"Avg %8.2lf"', |
| 'GPRINT:upHours:LAST:"Last %8.2lf\\n"', |
| 'LINE2:idleHours#333333:"Idletime"', |
| 'GPRINT:idleHours:MIN:"Min %8.2lf"', |
| 'GPRINT:idleHours:MAX:"Max %8.2lf"', |
| 'GPRINT:idleHours:AVERAGE:"Avg %8.2lf"', |
| 'GPRINT:idleHours:LAST:"Last %8.2lf\\n"', |
| ], |
| }, |
| } |
| |
| self.dd = data_def[datakey] |
| |
| def Create(self): |
| """Create an empty RRD file. |
| |
| Returns: |
| boolean: True = Success, False = failure. |
| """ |
| |
| stime = int(time()) -5 * 86400 |
| rrd_suffix = ['RRA:AVERAGE:0.5:1:576', |
| 'RRA:AVERAGE:0.5:6:672', |
| 'RRA:AVERAGE:0.5:24:732', |
| 'RRA:AVERAGE:0.5:144:1460', |
| ] |
| |
| rrd_cmd = [self.rrdtool, 'create', self.rrdfile, '--start', str(stime), |
| '--step', '300'] |
| for ds in self.dd['items']: |
| ds_str = 'DS:%s:%s:%s:%s:%s' % (ds, self.dd['type'], |
| self.dd['heartbeat'], |
| self.dd['min'], |
| self.dd['max']) |
| rrd_cmd.append(ds_str) |
| rrd_cmd = rrd_cmd + rrd_suffix |
| # Convert the rrd_cmd to a string with space separated commands. |
| exec_str = ' '.join(rrd_cmd) |
| result = Exec(exec_str) |
| if result: |
| TB.logger.error('Error executing:') |
| TB.logger.error('%s\n', exec_str) |
| |
| return result |
| |
| |
| def Update(self): |
| """Update an existing RRD file. |
| |
| Returns: |
| boolean: True = Success, False = errors. |
| """ |
| if len(TB.hosts[self.hostname]['rrddata'][self.rrdname]) < 2: |
| data = 'N:' + TB.hosts[self.hostname]['rrddata'][self.rrdname][0] |
| else: |
| data = 'N:' + ':'.join( |
| TB.hosts[self.hostname]['rrddata'][self.rrdname]) |
| rrd_cmd = [self.rrdtool, 'update', self.rrdfile, data] |
| exec_str = ' '.join(rrd_cmd) |
| result = Exec(exec_str) |
| if result: |
| TB.logger.error('Error executing:') |
| TB.logger.error('%s\n', exec_str) |
| |
| return result |
| |
| |
| def Graph(self): |
| """Create a graph of a tracked resource.""" |
| width = '850' |
| height = '300' |
| end = 'now' |
| rcolor = {'release': '#9966FF', |
| 'firmware': '#990033', |
| 'ec_firmware': '#009933', |
| } |
| |
| for time in TB.rrdtimes: |
| png_filename = self.rrdname + time + '.png' |
| png_file = os.path.join(self.rrd_dir, png_filename) |
| |
| title = self.dd['title'] + ' ' + time + '"' |
| |
| rrd_cmd = [self.rrdtool, 'graph', png_file, '--imgformat PNG', |
| '-s', time, '--end', end, |
| '--width', width, '--height', height, |
| '--vertical-label', self.dd['units'], '--title', title] |
| |
| for ds in self.dd['items']: |
| rrd_cmd.append('DEF:%s=%s:%s:AVERAGE' % (ds, self.rrdfile, ds)) |
| rrd_cmd = rrd_cmd + self.dd['graph'] |
| rrd_cmd.append('COMMENT:"Release History \\s"') |
| rrd_cmd.append('COMMENT:"=============== \\n"') |
| for v in TB.version: |
| sorted_items = [] |
| for k in TB.hosts[self.hostname][v]: |
| if k != 'PTR': |
| sorted_items.append(k) |
| sorted_items.sort() |
| for i in sorted_items: |
| # Get a date/time string to display, localtime requires |
| # a float, so convert i to float. |
| datetime = strftime('%D %H\\:%M', localtime(float(i))) |
| # Need to escape any ':' for RRDTool. |
| filter_val = ( |
| TB.hosts[self.hostname][v][i].replace(':', '\\:')) |
| # Insert Veritical Lines for release and firmware updates. |
| vrule = 'VRULE:%s%s:"%s %s=%s \\n"' % (i, rcolor[v], |
| datetime, v, filter_val) |
| rrd_cmd.append(vrule) |
| |
| exec_str = ' '.join(rrd_cmd) |
| result = Exec(exec_str) |
| if result: |
| TB.logger.error('Error while running %s', exec_str) |
| if os.path.isfile(png_file): |
| os.chmod(png_file, 0644) |
| |
| |
| class TBQueue(Queue.Queue): |
| """A subclass of class Queue to override join method with timeout.""" |
| |
| def __init__(self): |
| Queue.Queue.__init__(self) |
| |
| |
| def join(self, timeout=None): |
| deadline = None |
| waittime = None |
| if timeout: |
| deadline = time() + timeout |
| self.all_tasks_done.acquire() |
| try: |
| while self.unfinished_tasks: |
| if deadline: |
| waittime = deadline - time() |
| if waittime < 0: |
| break |
| self.all_tasks_done.wait(waittime) |
| finally: |
| self.all_tasks_done.release() |
| |
| |
| def Exec(cmd, output=False): |
| """Run subprocess.Popen() and return output if output=True. |
| |
| Args: |
| cmd: string, represents command with arguments. |
| output: boolean, True=capture and return output. |
| Returns: |
| string if output = True |
| return code of command if output = False |
| """ |
| |
| p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE) |
| p.wait() |
| out = p.stdout.read() |
| errors = p.stderr.read() |
| if p.returncode: |
| TB.logger.error(out) |
| TB.logger.error(errors) |
| |
| if output: |
| return out |
| else: |
| return p.returncode |
| |
| |
| def ParseArgs(): |
| """Parse all command line options.""" |
| # Assume Chrome OS source is located on /usr/local/google. |
| homedir = os.environ['HOME'] |
| logfile = os.path.join(homedir, 'monitor.log') |
| cros_src = '/usr/local/google' + homedir + '/chromeos/src' |
| systemhealth_home = os.path.join(homedir, 'www', 'systemhealth') |
| |
| parser = optparse.OptionParser(version= __version__) |
| parser.add_option('--debug', |
| help='Set the debug level [default: %default]', |
| type='choice', |
| choices=['debug', 'info', 'warning', 'error', |
| 'critical',], |
| default='info', |
| dest='debug') |
| parser.add_option('--gclient', |
| help='pathname of Chrome OS source [default: %default]', |
| default=cros_src, |
| dest='gclient') |
| parser.add_option('--graph', |
| help='Create graphs for each host [default: %default]', |
| default=False, |
| dest='graph') |
| parser.add_option('--home', |
| help='Systemhealth home directory [default: %default]', |
| default=systemhealth_home, |
| dest='home') |
| parser.add_option('--html', |
| help='Build HTML pages for hosts [default: %default]', |
| default=False, |
| dest='html') |
| parser.add_option('--logfile', |
| help='name of logfile [default: %default]', |
| default=logfile, |
| dest='logfile') |
| parser.add_option('--log_to_stdout', |
| help='Send output to StdOut [default: %default]', |
| default=False, |
| dest='log_to_stdout') |
| parser.add_option('--update', |
| help='Collect data from hosts [default: %default]', |
| default=True, |
| dest='update') |
| parser.add_option('--url', |
| help='URL for landing page [default: %default]', |
| default='http://www/~chromeos-test/systemhealth/', |
| dest='url') |
| |
| return parser.parse_args() |
| |
| |
| def CheckRun(action): |
| """Check the run status of monitor.py, and add/remove run files. |
| |
| This function will ensure we only running one program with either the graph |
| or update option. |
| Args: |
| action: string, indicates if monitor.py is starting or stopping. |
| """ |
| if action == 'start': |
| if TB.update == True: |
| if os.path.isfile(TB.update_runfile): |
| TB.logger.info('Exiting, already running with update option') |
| sys.exit(1) |
| else: |
| try: |
| open(TB.update_runfile, 'w').close() |
| except IOError, e: |
| TB.logger.error('Error with %s\n%s', TB.update_runfile, e) |
| if TB.graph: |
| if os.path.isfile(TB.graph_runfile): |
| TB.logger.info('Exiting, already running with graph option') |
| sys.exit(1) |
| else: |
| try: |
| open(TB.graph_runfile, 'w').close() |
| except IOError, e: |
| TB.logger.error('Error with %s\n%s', TB.graph_runfile, e) |
| elif action == 'stop': |
| if TB.update == True: |
| if os.path.isfile(TB.update_runfile): |
| try: |
| os.remove(TB.update_runfile) |
| except IOError, e: |
| TB.logger.error('Error with %s\n%s', TB.update_runfile, e) |
| if TB.graph: |
| if os.path.isfile(TB.graph_runfile): |
| try: |
| os.remove(TB.graph_runfile) |
| except IOError, e: |
| TB.logger.error('Error with %s\n%s', TB.graph_runfile, e) |
| else: |
| TB.logger.error('Unknown option passed to CheckRun(): %s', action) |
| sys.exit(1) |
| |
| |
| def main(argv): |
| options, args = ParseArgs() |
| |
| global TB |
| TB = TestBed(options.logfile, options.log_to_stdout, options.debug, |
| options.graph, options.home, options.html, options.gclient, |
| options.update, options.url) |
| CheckRun('start') |
| sysmon = Monitor() |
| sysmon.UpdateStatus() |
| sysmon.BuildLandingPage() |
| runtime = time() - TB.time |
| endtime = strftime('%H:%M:%S', localtime()) |
| TB.logger.info('End Time: %s', endtime) |
| TB.logger.info('Time of run: %s seconds', runtime) |
| TB.logger.info('Ran with %d threads', sysmon.hostqty) |
| CheckRun('stop') |
| |
| if __name__ == '__main__': |
| main(sys.argv) |
