| # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import fnmatch |
| import glob |
| import logging |
| import os |
| |
| from autotest_lib.client.bin import test, utils |
| from autotest_lib.client.common_lib import error |
| from optparse import OptionParser |
| |
| FILE_CMD="file -m /usr/local/share/misc/magic.mgc" |
| |
| class ToolchainOptionSet: |
| """ |
| Handles a set of hits, along with potential whitelists to ignore. |
| """ |
| def __init__(self, description, bad_files, whitelist_file): |
| self.description = description |
| self.bad_set = set(bad_files.splitlines()) |
| self.whitelist_set = set([]) |
| self.process_whitelist_with_private(whitelist_file) |
| |
| |
| def process_whitelist_with_private(self, whitelist_file): |
| """ |
| Filter out hits found on non-comment lines in the whitelist and |
| and private whitelist. |
| |
| @param whitelist_file: path to whitelist file |
| """ |
| whitelist_files = [whitelist_file] |
| private_file = os.path.join(os.path.dirname(whitelist_file), |
| "private_" + |
| os.path.basename(whitelist_file)) |
| whitelist_files.append(private_file) |
| self.process_whitelists(whitelist_files) |
| |
| |
| def process_whitelist(self, whitelist_file): |
| """ |
| Filter out hits found on non-comment lines in the whitelist. |
| |
| @param whitelist_file: path to whitelist file |
| """ |
| if os.path.isfile(whitelist_file): |
| f = open(whitelist_file) |
| whitelist = [x for x in f.read().splitlines() |
| if not x.startswith('#')] |
| f.close() |
| self.whitelist_set = self.whitelist_set.union(set(whitelist)) |
| |
| filtered_list = [] |
| for bad_file in self.bad_set: |
| # Does |bad_file| match any entry in the whitelist? |
| in_whitelist = any([fnmatch.fnmatch(bad_file, whitelist_entry) |
| for whitelist_entry in self.whitelist_set]) |
| if not in_whitelist: |
| filtered_list.append(bad_file) |
| |
| self.filtered_set = set(filtered_list) |
| # TODO(jorgelo): remove glob patterns from |new_passes|. |
| self.new_passes = self.whitelist_set.difference(self.bad_set) |
| |
| |
| def process_whitelists(self, whitelist_files): |
| """ |
| Filter out hits found in a list of whitelist files. |
| |
| @param whitelist_files: list of paths to whitelist files |
| """ |
| for whitelist_file in whitelist_files: |
| self.process_whitelist(whitelist_file) |
| |
| |
| def get_fail_summary_message(self): |
| m = "Test %s " % self.description |
| m += "%d failures" % len(self.filtered_set) |
| return m |
| |
| |
| def get_fail_message(self): |
| m = self.get_fail_summary_message() |
| sorted_list = list(self.filtered_set) |
| sorted_list.sort() |
| m += "\nFAILED:\n%s\n\n" % "\n".join(sorted_list) |
| return m |
| |
| |
| def __str__(self): |
| m = "Test %s " % self.description |
| m += ("%d failures, %d in whitelist, %d in filtered, %d new passes " % |
| (len(self.bad_set), |
| len(self.whitelist_set), |
| len(self.filtered_set), |
| len(self.new_passes))) |
| |
| if len(self.filtered_set): |
| sorted_list = list(self.filtered_set) |
| sorted_list.sort() |
| m += "FAILED:\n%s" % "\n".join(sorted_list) |
| else: |
| m += "PASSED!" |
| |
| if len(self.new_passes): |
| sorted_list = list(self.new_passes) |
| sorted_list.sort() |
| m += ("\nNew passes (remove these from the whitelist):\n%s" % |
| "\n".join(sorted_list)) |
| logging.debug(m) |
| return m |
| |
| |
| class platform_ToolchainOptions(test.test): |
| """ |
| Tests for various expected conditions on ELF binaries in the image. |
| """ |
| version = 2 |
| |
| def get_cmd(self, test_cmd, find_options=""): |
| base_cmd = ("find '%s' -wholename %s -prune -o " |
| " -wholename /proc -prune -o " |
| " -wholename /dev -prune -o " |
| " -wholename /sys -prune -o " |
| " -wholename /mnt/stateful_partition -prune -o " |
| " -wholename /usr/local -prune -o " |
| # There are files in /home (e.g. encrypted files that look |
| # like they have ELF headers) that cause false positives, |
| # and since that's noexec anyways, it should be skipped. |
| " -wholename '/home' -prune -o " |
| " -wholename " |
| "/opt/google/containers/android/rootfs/root/vendor" |
| " -prune -o " |
| " -wholename " |
| "/run/containers/android_*/root/vendor" |
| " -prune -o " |
| " %s " |
| " -not -name 'libstdc++.so.*' " |
| " -not -name 'libgcc_s.so.*' " |
| " -type f -executable -exec " |
| "sh -c '%s " |
| "{} | grep -q ELF && " |
| "(%s || echo {})' ';'") |
| rootdir = "/" |
| cmd = base_cmd % (rootdir, self.autodir, find_options, FILE_CMD, |
| test_cmd) |
| return cmd |
| |
| |
| def create_and_filter(self, description, cmd, whitelist_file, |
| find_options=""): |
| """ |
| Runs a command, with "{}" replaced (via "find -exec") with the |
| target ELF binary. If the command fails, the file is marked as |
| failing the test. Results are filtered against the provided |
| whitelist file. |
| |
| @param description: text name of the check being done |
| @param cmd: command to run via find's -exec option |
| @param whitelist_file: list of failures to ignore |
| @param find_options: additional options for find to limit the scope |
| """ |
| full_cmd = self.get_cmd(cmd, find_options) |
| bad_files = utils.system_output(full_cmd) |
| cso = ToolchainOptionSet(description, bad_files, whitelist_file) |
| cso.process_whitelist_with_private(whitelist_file) |
| return cso |
| |
| |
| def run_once(self, rootdir="/", args=[]): |
| """ |
| Do a find for all the ELF files on the system. |
| For each one, test for compiler options that should have been used |
| when compiling the file. |
| |
| For missing compiler options, print the files. |
| """ |
| |
| parser = OptionParser() |
| parser.add_option('--hardfp', |
| dest='enable_hardfp', |
| default=False, |
| action='store_true', |
| help='Whether to check for hardfp binaries.') |
| (options, args) = parser.parse_args(args) |
| |
| option_sets = [] |
| |
| libc_glob = "/lib/libc-[0-9]*" |
| |
| readelf_cmd = glob.glob("/usr/local/*/binutils-bin/*/readelf")[0] |
| |
| # We do not test binaries if they are built with Address Sanitizer |
| # because it is a separate testing tool. |
| no_asan_used = utils.system_output("%s -s " |
| "/opt/google/chrome/chrome | " |
| "egrep -q \"__asan_init\" || " |
| "echo no ASAN" % readelf_cmd) |
| if not no_asan_used: |
| logging.debug("ASAN detected on /opt/google/chrome/chrome. " |
| "Will skip all checks.") |
| return |
| |
| # Check that gold was used to build binaries. |
| # TODO(jorgelo): re-enable this check once crbug.com/417912 is fixed. |
| # gold_cmd = ("%s -S {} 2>&1 | " |
| # "egrep -q \".note.gnu.gold-ve\"" % readelf_cmd) |
| # gold_find_options = "" |
| # if utils.get_cpu_arch() == "arm": |
| # # gold is only enabled for Chrome on ARM. |
| # gold_find_options = "-path \"/opt/google/chrome/chrome\"" |
| # gold_whitelist = os.path.join(self.bindir, "gold_whitelist") |
| # option_sets.append(self.create_and_filter("gold", |
| # gold_cmd, |
| # gold_whitelist, |
| # gold_find_options)) |
| |
| # Verify non-static binaries have BIND_NOW in dynamic section. |
| now_cmd = ("(%s {} | grep -q statically) ||" |
| "%s -d {} 2>&1 | " |
| "egrep -q \"BIND_NOW\"" % (FILE_CMD, readelf_cmd)) |
| now_whitelist = os.path.join(self.bindir, "now_whitelist") |
| option_sets.append(self.create_and_filter("-Wl,-z,now", |
| now_cmd, |
| now_whitelist)) |
| |
| # Verify non-static binaries have RELRO program header. |
| relro_cmd = ("(%s {} | grep -q statically) ||" |
| "%s -l {} 2>&1 | " |
| "egrep -q \"GNU_RELRO\"" % (FILE_CMD, readelf_cmd)) |
| relro_whitelist = os.path.join(self.bindir, "relro_whitelist") |
| option_sets.append(self.create_and_filter("-Wl,-z,relro", |
| relro_cmd, |
| relro_whitelist)) |
| |
| # Verify non-static binaries are dynamic (built PIE). |
| pie_cmd = ("(%s {} | grep -q statically) ||" |
| "%s -l {} 2>&1 | " |
| "egrep -q \"Elf file type is DYN\"" % (FILE_CMD, |
| readelf_cmd)) |
| pie_whitelist = os.path.join(self.bindir, "pie_whitelist") |
| option_sets.append(self.create_and_filter("-fPIE", |
| pie_cmd, |
| pie_whitelist)) |
| |
| # Verify ELFs don't include TEXTRELs. |
| # FIXME: Remove the i?86 filter after the bug is fixed. |
| # crbug.com/686926 |
| if (utils.get_current_kernel_arch() not in |
| ('i%d86' % i for i in xrange(3,7))): |
| textrel_cmd = ("(%s {} | grep -q statically) ||" |
| "%s -d {} 2>&1 | " |
| "(egrep -q \"0x0+16..TEXTREL\"; [ $? -ne 0 ])" |
| % (FILE_CMD, readelf_cmd)) |
| textrel_whitelist = os.path.join(self.bindir, "textrel_whitelist") |
| option_sets.append(self.create_and_filter("TEXTREL", |
| textrel_cmd, |
| textrel_whitelist)) |
| |
| # Verify all binaries have non-exec STACK program header. |
| stack_cmd = ("%s -lW {} 2>&1 | " |
| "egrep -q \"GNU_STACK.*RW \"" % readelf_cmd) |
| stack_whitelist = os.path.join(self.bindir, "stack_whitelist") |
| option_sets.append(self.create_and_filter("Executable Stack", |
| stack_cmd, |
| stack_whitelist)) |
| |
| # Verify no binaries have W+X LOAD program headers. |
| loadwx_cmd = ("%s -lW {} 2>&1 | " |
| "grep \"LOAD\" | egrep -v \"(RW |R E|R )\" | " |
| "wc -l | grep -q \"^0$\"" % readelf_cmd) |
| loadwx_whitelist = os.path.join(self.bindir, "loadwx_whitelist") |
| option_sets.append(self.create_and_filter("LOAD Writable and Exec", |
| loadwx_cmd, |
| loadwx_whitelist)) |
| |
| # Verify ARM binaries are all using VFP registers. |
| if (options.enable_hardfp and utils.get_cpu_arch() == 'arm'): |
| hardfp_cmd = ("%s -A {} 2>&1 | " |
| "egrep -q \"Tag_ABI_VFP_args: VFP registers\"" % |
| readelf_cmd) |
| hardfp_whitelist = os.path.join(self.bindir, "hardfp_whitelist") |
| option_sets.append(self.create_and_filter("hardfp", hardfp_cmd, |
| hardfp_whitelist)) |
| |
| # Verify all binaries are not linked with libgcc_s.so. |
| libgcc_cmd = ("%s -dW {} 2>&1 | grep \"NEEDED\" | " |
| "(! grep -q \"libgcc_s.so\")" % readelf_cmd) |
| libgcc_whitelist = os.path.join(self.bindir, "libgcc_whitelist") |
| option_sets.append(self.create_and_filter("Libgcc_s Users", |
| libgcc_cmd, |
| libgcc_whitelist)) |
| |
| # Verify all binaries are not linked with libstdc++.so. |
| libstdcxx_cmd = ("%s -dW {} 2>&1 | grep \"NEEDED\" | " |
| "(! grep -q \"libstdc++.so\")" % readelf_cmd) |
| libstdcxx_whitelist = os.path.join(self.bindir, "libstdcxx_whitelist") |
| option_sets.append(self.create_and_filter("Libstdc++ Users", |
| libstdcxx_cmd, |
| libstdcxx_whitelist)) |
| |
| fail_msg = "" |
| |
| # There is currently no way to clear binary prebuilts for all devs. |
| # Thus, when a new check is added to this test, the test might fail |
| # for users who have old prebuilts which have not been compiled |
| # in the correct manner. |
| fail_summaries = [] |
| full_msg = "Test results:" |
| num_fails = 0 |
| for cos in option_sets: |
| if len(cos.filtered_set): |
| num_fails += 1 |
| fail_msg += cos.get_fail_message() + "\n" |
| fail_summaries.append(cos.get_fail_summary_message()) |
| full_msg += str(cos) + "\n\n" |
| fail_summary_msg = ", ".join(fail_summaries) |
| |
| logging.error(fail_msg) |
| logging.debug(full_msg) |
| if num_fails: |
| raise error.TestFail(fail_summary_msg) |
