Add a script to delete nightly test data that are old.
TEST=tested on tc-build.hot
Change-Id: I6b6bc298b56c6ac79cbbd66f8168bc234580c884
Reviewed-on: https://chrome-internal-review.googlesource.com/150264
Reviewed-by: Yunlian Jiang <yunlian@google.com>
Commit-Queue: Han Shen <shenhan@google.com>
Tested-by: Han Shen <shenhan@google.com>
diff --git a/auto_delete_nightly_test_data.py b/auto_delete_nightly_test_data.py
new file mode 100755
index 0000000..86f8e55
--- /dev/null
+++ b/auto_delete_nightly_test_data.py
@@ -0,0 +1,80 @@
+#!/usr/bin/python
+
+"""A crontab script to delete night test data."""
+__author__ = 'shenhan@google.com (Han Shen)'
+
+import datetime
+import optparse
+import os
+import sys
+
+from utils import command_executer
+from utils import constants
+from utils import misc
+
+DIR_BY_WEEKDAY = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
+
+
+def CleanNumberedDir(s, dry_run=False):
+ """Deleted directories under each dated_dir."""
+ chromeos_dirs = [os.path.join(s, x) for x in os.listdir(s)
+ if misc.IsChromeOsTree(os.path.join(s, x))]
+ ce = command_executer.GetCommandExecuter()
+ for cd in chromeos_dirs:
+ misc.DeleteChromeOsTree(cd, dry_run=dry_run)
+ ## Now delete the numbered dir
+ cmd = 'rm -fr {0}'.format(s)
+ if dry_run:
+ print cmd
+ else:
+ ce.RunCommand(cmd, return_output=True, terminated_timeout=480)
+
+
+def CleanDatedDir(dated_dir, dry_run=False):
+ # List subdirs under dir
+ subdirs = [os.path.join(dated_dir, x) for x in os.listdir(dated_dir)
+ if os.path.isdir(os.path.join(dated_dir, x))]
+ for s in subdirs:
+ CleanNumberedDir(s, dry_run)
+
+
+def ProcessArguments(argv):
+ """Process arguments."""
+ parser = optparse.OptionParser(
+ description='Automatically delete nightly test data directories.',
+ usage='auto_delete.py options')
+ parser.add_option('-d', '--dry_run', dest='dry_run',
+ default=False, action='store_true',
+ help='Only print command line, do not execute anything.')
+ parser.add_option('--days_to_perserve', dest='days_to_preserve', default=3,
+ help=('Specify the number of days, test data generated '
+ 'on these days will *NOT* be deleted. '
+ 'Defaults to 3.'))
+ options, _ = parser.parse_args(argv)
+ return options
+
+
+def Main(argv):
+ """Delete nightly test data directories."""
+ options = ProcessArguments(argv)
+ # Function 'isoweekday' returns 1(Monday) - 7 (Sunday).
+ d = datetime.datetime.today().isoweekday()
+ # We go back 1 week, delete from that day till we are
+ # options.days_to_preserve away from today.
+ s = d - 7
+ e = d - options.days_to_preserve
+ for i in range(s + 1, e):
+ if i <= 0:
+ ## Wrap around if index is negative. 6 is from i + 7 - 1, because
+ ## DIR_BY_WEEKDAY starts from 0, while isoweekday is from 1-7.
+ dated_dir = DIR_BY_WEEKDAY[i+6]
+ else:
+ dated_dir = DIR_BY_WEEKDAY[i-1]
+ CleanDatedDir(os.path.join(
+ constants.CROSTC_WORKSPACE, dated_dir), options.dry_run)
+ return 0
+
+
+if __name__ == '__main__':
+ retval = Main(sys.argv)
+ sys.exit(retval)
diff --git a/build_tc.py b/build_tc.py
index 02918b6..5419b55 100755
--- a/build_tc.py
+++ b/build_tc.py
@@ -38,7 +38,7 @@
"etc/portage/package.mask/cross-%s" % self._ctarget)
self._new_mask_file = None
- self._chroot_source_path = os.path.join(constants.mounted_toolchain_root,
+ self._chroot_source_path = os.path.join(constants.MOUNTED_TOOLCHAIN_ROOT,
self._name).lstrip("/")
self._incremental = incremental
self._build_env = build_env
diff --git a/dejagnu/run_dejagnu.py b/dejagnu/run_dejagnu.py
index 8943bb2..14bf058 100755
--- a/dejagnu/run_dejagnu.py
+++ b/dejagnu/run_dejagnu.py
@@ -99,7 +99,7 @@
self._chromeos_chroot = path.join(chromeos_root, 'chroot')
if mount:
self._gcc_source_dir_to_mount = mount
- self._gcc_source_dir = path.join(constants.mounted_toolchain_root, 'gcc')
+ self._gcc_source_dir = path.join(constants.MOUNTED_TOOLCHAIN_ROOT, 'gcc')
else:
self._gcc_source_dir = None
diff --git a/utils/constants.py b/utils/constants.py
index fdefeb2..abf9ff1 100644
--- a/utils/constants.py
+++ b/utils/constants.py
@@ -5,6 +5,9 @@
"""Generic constants used accross modules.
"""
-__author__ = "shenhan@google.com (Han Shen)"
+__author__ = 'shenhan@google.com (Han Shen)'
-mounted_toolchain_root='/usr/local/toolchain_root'
+MOUNTED_TOOLCHAIN_ROOT = '/usr/local/toolchain_root'
+
+# Root directory for night testing run.
+CROSTC_WORKSPACE = '/usr/local/google/crostc'
diff --git a/utils/misc.py b/utils/misc.py
index 61a0feb..9db6fd5 100644
--- a/utils/misc.py
+++ b/utils/misc.py
@@ -1,6 +1,6 @@
#!/usr/bin/python
-# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+# Copyright 2013 Google Inc. All Rights Reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
@@ -154,9 +154,9 @@
else:
usepkg_flag = "--nousepkg"
if debug:
- withdebug_flag = '--withdebug'
+ withdebug_flag = "--withdebug"
else:
- withdebug_flag = '--nowithdebug'
+ withdebug_flag = "--nowithdebug"
return ("./build_packages %s --withdev --withtest --withautotest "
"--skip_toolchain_update %s --board=%s "
"--accept_licenses=@CHROMEOS" %
@@ -169,6 +169,7 @@
dev_args = "--noenable_rootfs_verification --disk_layout=2gb-rootfs"
return "./build_image --board=%s %s test" % (board, dev_args)
+
def GetSetupBoardCommand(board, gcc_version=None, binutils_version=None,
usepkg=None, force=None):
"""Get setup_board command."""
@@ -303,8 +304,9 @@
except ValueError:
return False
+
def RemoveChromeBrowserObjectFiles(chromeos_root, board):
- """ Remove any object files from all the posible locations """
+ """Remove any object files from all the posible locations."""
out_dir = os.path.join(
GetChrootPath(chromeos_root),
"var/cache/chromeos-chrome/chrome-src/src/out_%s" % board)
@@ -318,6 +320,7 @@
shutil.rmtree(out_dir)
logger.GetLogger().LogCmd("rm -rf %s" % out_dir)
+
@contextmanager
def WorkingDirectory(new_dir):
"""Get the working directory."""
@@ -332,27 +335,38 @@
logger.GetLogger().LogCmd(msg)
os.chdir(old_dir)
+
def HasGitStagedChanges(git_dir):
"""Return True if git repository has staged changes."""
- command = 'cd {0} && git diff --quiet --cached --exit-code HEAD'.format(
- git_dir)
+ command = "cd {0} && git diff --quiet --cached --exit-code HEAD".format(
+ git_dir)
return command_executer.GetCommandExecuter().RunCommand(
- command, print_to_console=False)
+ command, print_to_console=False)
+
def HasGitUnstagedChanges(git_dir):
"""Return True if git repository has un-staged changes."""
- command = 'cd {0} && git diff --quiet --exit-code HEAD'.format(git_dir)
+ command = "cd {0} && git diff --quiet --exit-code HEAD".format(git_dir)
return command_executer.GetCommandExecuter().RunCommand(
- command, print_to_console=False)
+ command, print_to_console=False)
+
def HasGitUntrackedChanges(git_dir):
"""Return True if git repository has un-tracked changes."""
- command = 'cd {0} && test -z $(git ls-files --exclude-standard --others)' \
- .format(git_dir)
+ command = ("cd {0} && test -z "
+ "$(git ls-files --exclude-standard --others)").format(git_dir)
return command_executer.GetCommandExecuter().RunCommand(
- command,print_to_console=False)
+ command, print_to_console=False)
+
def IsGitTreeClean(git_dir):
+ """Test if git tree has no local changes.
+
+ Args:
+ git_dir: git tree directory.
+ Returns:
+ True if git dir is clean.
+ """
if HasGitStagedChanges(git_dir):
logger.GetLogger().LogWarning("Git tree has staged changes.")
return False
@@ -364,15 +378,67 @@
return False
return True
+
def GetGitChangesAsList(git_dir, path=None, staged=False):
- command = 'cd {0} && git diff --name-only'.format(git_dir)
+ """Get changed files as a list.
+
+ Args:
+ git_dir: git tree directory.
+ path: a relative path that is part of the tree directory, could be null.
+ staged: whether to include staged files as well.
+ Returns:
+ A list containing all the changed files.
+ """
+ command = "cd {0} && git diff --name-only".format(git_dir)
if staged:
- command = command + ' --cached'
+ command += " --cached"
if path:
- command = command + ' -- ' + path
+ command += " -- " + path
_, out, _ = command_executer.GetCommandExecuter().RunCommand(
- command, return_output=True, print_to_console=False)
+ command, return_output=True, print_to_console=False)
rv = []
for line in out.splitlines():
rv.append(line)
return rv
+
+
+def IsChromeOsTree(chromeos_root):
+ return (os.path.isdir(os.path.join(
+ chromeos_root, "src/third_party/chromiumos-overlay")) and
+ os.path.isdir(os.path.join(
+ chromeos_root, "manifest")))
+
+
+def DeleteChromeOsTree(chromeos_root, dry_run=False):
+ """Delete a ChromeOs tree *safely*.
+
+ Args:
+ chromeos_root: dir of the tree, could be a relative one (but be careful)
+ dry_run: only prints out the command if True
+
+ Returns:
+ True if everything is ok.
+ """
+ if not IsChromeOsTree(chromeos_root):
+ logger.GetLogger().LogWarning(
+ '"{0}" does not seem to be a valid chromeos tree, do nothing.'.format(
+ chromeos_root))
+ return False
+ cmd0 = "cd {0} && cros_sdk --delete".format(chromeos_root)
+ if dry_run:
+ print cmd0
+ else:
+ if command_executer.GetCommandExecuter().RunCommand(
+ cmd0, return_output=False, print_to_console=True) != 0:
+ return False
+
+ cmd1 = ('export CHROMEOSDIRNAME="$(dirname $(cd {0} && pwd))" && '
+ 'export CHROMEOSBASENAME="$(basename $(cd {0} && pwd))" && '
+ 'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME').format(
+ chromeos_root)
+ if dry_run:
+ print cmd1
+ return True
+
+ return command_executer.GetCommandExecuter().RunCommand(
+ cmd1, return_output=False, print_to_console=True) == 0
