Tools to update platform_ToolchainOptions whitelist.
BUG=None
TEST=None
Change-Id: Iabdc1c5515660370921976fe95e50db672412664
Reviewed-on: https://chromium-review.googlesource.com/192028
Reviewed-by: Kees Cook <keescook@chromium.org>
Commit-Queue: Deepak Gopal <deepakg@chromium.org>
Tested-by: Deepak Gopal <deepakg@chromium.org>
Reviewed-by: Kris Rambish <krisr@chromium.org>
diff --git a/provingground/platform_whitelist_tools/README b/provingground/platform_whitelist_tools/README
new file mode 100644
index 0000000..34b27a4
--- /dev/null
+++ b/provingground/platform_whitelist_tools/README
@@ -0,0 +1,15 @@
+Scripts to update platform_ToolchainOptions whitelists.
+
+Download all the debug files for all the platforms into a directory and run
+"fetch_commons.py" against that directory to get a list of all the
+files that are passing currently on multilple platforms.
+This will generate files containing the files that need to be
+removed from the list.
+
+Run "fetch_commons.py" against multilple builds (atleast two).
+Then compare the files generated for various builds using
+"sort file1 file2 | uniq -d" to get a list of files which are common
+accross all platforms and multiple builds.
+
+Run remove_from_whitelist.py" to remove the now useless files from the
+whitelists in platform_ToolchainOptions.
diff --git a/provingground/platform_whitelist_tools/fetch_commons.py b/provingground/platform_whitelist_tools/fetch_commons.py
new file mode 100644
index 0000000..18ebd07
--- /dev/null
+++ b/provingground/platform_whitelist_tools/fetch_commons.py
@@ -0,0 +1,102 @@
+#!/usr/bin/python
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file
+
+__author__ = 'deepakg@google.com'
+
+"""
+Read *.debug files and create seperate files for each whitelist.
+"""
+
+import os
+import re
+import sys
+
+
+class Read_File:
+ """
+ This class should read the debug files from the platform_ToolchainOptions
+ tests and should list out all the whitelist files that are passing and not
+ needed anymore and write them into a file.
+ """
+
+ def __init__(self):
+ """
+ Accept a file name that we should parse.
+ """
+ self.tests = [{'name':'gold', 'match':'Test gold',},
+ {'name':'now', 'match':'Test -Wl,-z,now',},
+ {'name':'relro', 'match':'Test -Wl,-z,relro',},
+ {'name':'pie', 'match':'Test -fPIE',},
+ {'name':'stack', 'match':'Test Executable Stack',}]
+
+
+ def parse_file(self, current_file):
+ """
+ Read the contents and group them.
+
+ current_file: name of the file to be parsed.
+ """
+ parse = False
+ whitelist = None
+ f = open(current_file)
+ for line in f.readlines():
+ if not parse:
+ for item in self.tests:
+ if item['match'] in line:
+ whitelist = item
+
+ if 'New passes' in line:
+ parse = True
+ elif parse:
+ if re.match('\/', line):
+ pathname = line.strip()
+ whitelist.setdefault('files', {})
+ whitelist['files'].setdefault(pathname, 0)
+ whitelist['files'][pathname] += 1
+ else:
+ parse = False
+
+
+ def write_to_file(self, current_path, number_of_files):
+ """
+ Create a file for each whitelist and write the lines into it.
+
+ current_path: Path to the directory containing the debug files.
+ number_of_files: The number of DEBUG files in the directory.
+ """
+ for wl in self.tests:
+ fpath = os.path.join(current_path, wl['name'])
+ fopen = open(fpath, 'w+')
+ for key, value in wl.items():
+ if ((key != 'name' or key != 'match') and
+ value == number_of_files) :
+ fopen.write(key)
+ fopen.write('\n')
+ print ('The whitelist files to be removed are written in ' + fpath)
+ fopen.close()
+
+
+def main():
+ try:
+ dir_path = sys.argv[1]
+ except:
+ dir_path = raw_input('Please enter the complete path to the directory '
+ 'containing all the ToolchainOptions output - \n')
+ dir_path = os.path.expanduser(dir_path)
+ if not os.path.isdir(dir_path):
+ raise RuntimeError('We need a directory with the ToolchainOptions '
+ 'output files to continue.')
+ fread = Read_File()
+ file_count = 0
+ for current_file in os.listdir(dir_path):
+ file_path = os.path.join(dir_path, current_file)
+ if file_path.endswith('.DEBUG'):
+ fread.parse_file(file_path)
+ file_count += 1
+ fread.write_to_file(dir_path, file_count)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/provingground/platform_whitelist_tools/remove_from_whitelist.py b/provingground/platform_whitelist_tools/remove_from_whitelist.py
new file mode 100644
index 0000000..2547dbb
--- /dev/null
+++ b/provingground/platform_whitelist_tools/remove_from_whitelist.py
@@ -0,0 +1,47 @@
+#!/usr/bin/python
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+__author__ = 'deepakg@google.com'
+
+"""
+Remove the filtered lines in temp file from the whitelist files.
+"""
+
+import os
+import sys
+import tempfile
+
+
+def main():
+ """
+ Remove lines from the whitelist.
+ """
+ if len(sys.argv)!=3:
+ print ('InvalidArguments: We need 2 arguments. \n'
+ 'example: remove_from_whitelist.py gold_whitelist gold_temp.')
+ else:
+ whitelist_name = sys.argv[1]
+ tempFile = sys.argv[2]
+ if 'whitelist' in str(sys.argv[2]):
+ tempFile = sys.argv[1]
+ whitelist_name = sys.argv[2]
+ whitelist = open(whitelist_name, 'r')
+ remove = [x.strip() for x in open(tempFile, 'r').readlines()]
+ output = tempfile.NamedTemporaryFile(dir=os.path.dirname(whitelist_name),
+ delete=False)
+ for line in whitelist:
+ if line.startswith('#'):
+ continue
+ if line.strip() in remove:
+ continue
+ output.write(line)
+ output.close()
+ whitelist.close()
+ os.rename(output.name, whitelist_name)
+ print 'We have re-written the file ' + str(whitelist_name)
+
+
+if __name__ == '__main__':
+ main()
