apply yapf for ninjalog*.py
This is preparation to fix the issue.
Bug: 992282
Change-Id: I64b237670bf460ff765df3eaf069b0e0bdcfb31d
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2237268
Commit-Queue: Ye Kuang <yekuang@google.com>
Reviewed-by: Ye Kuang <yekuang@google.com>
Auto-Submit: Takuto Ikuta <tikuta@chromium.org>
diff --git a/ninjalog_uploader.py b/ninjalog_uploader.py
index e678e16..a56d26f 100755
--- a/ninjalog_uploader.py
+++ b/ninjalog_uploader.py
@@ -2,7 +2,6 @@
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-
"""
This is script to upload ninja_log from googler.
@@ -32,79 +31,80 @@
# These build configs affect build performance a lot.
# TODO(tikuta): Add 'blink_symbol_level', 'closure_compile' and
# 'use_jumbo_build'.
-WHITELISTED_CONFIGS = (
- 'symbol_level', 'use_goma', 'is_debug', 'is_component_build', 'enable_nacl',
- 'host_os', 'host_cpu', 'target_os', 'target_cpu'
-)
+WHITELISTED_CONFIGS = ('symbol_level', 'use_goma', 'is_debug',
+ 'is_component_build', 'enable_nacl', 'host_os',
+ 'host_cpu', 'target_os', 'target_cpu')
+
def IsGoogler(server):
- """Check whether this script run inside corp network."""
- try:
- h = httplib2.Http()
- _, content = h.request('https://'+server+'/should-upload', 'GET')
- return content == 'Success'
- except httplib2.HttpLib2Error:
- return False
+ """Check whether this script run inside corp network."""
+ try:
+ h = httplib2.Http()
+ _, content = h.request('https://' + server + '/should-upload', 'GET')
+ return content == 'Success'
+ except httplib2.HttpLib2Error:
+ return False
+
def ParseGNArgs(gn_args):
- """Parse gn_args as json and return config dictionary."""
- configs = json.loads(gn_args)
- build_configs = {}
+ """Parse gn_args as json and return config dictionary."""
+ configs = json.loads(gn_args)
+ build_configs = {}
- for config in configs:
- key = config["name"]
- if key not in WHITELISTED_CONFIGS:
- continue
- if 'current' in config:
- build_configs[key] = config['current']['value']
- else:
- build_configs[key] = config['default']['value']
+ for config in configs:
+ key = config["name"]
+ if key not in WHITELISTED_CONFIGS:
+ continue
+ if 'current' in config:
+ build_configs[key] = config['current']['value']
+ else:
+ build_configs[key] = config['default']['value']
- return build_configs
+ return build_configs
+
def GetBuildTargetFromCommandLine(cmdline):
- """Get build targets from commandline."""
+ """Get build targets from commandline."""
- # Skip argv0.
- idx = 1
+ # Skip argv0.
+ idx = 1
- # Skipping all args that involve these flags, and taking all remaining args
- # as targets.
- onearg_flags = ('-C', '-f', '-j', '-k', '-l', '-d', '-t', '-w')
- zeroarg_flags = ('--version', '-n', '-v')
+ # Skipping all args that involve these flags, and taking all remaining args
+ # as targets.
+ onearg_flags = ('-C', '-f', '-j', '-k', '-l', '-d', '-t', '-w')
+ zeroarg_flags = ('--version', '-n', '-v')
- targets = []
+ targets = []
- while idx < len(cmdline):
- if cmdline[idx] in onearg_flags:
- idx += 2
- continue
+ while idx < len(cmdline):
+ if cmdline[idx] in onearg_flags:
+ idx += 2
+ continue
- if (cmdline[idx][:2] in onearg_flags or
- cmdline[idx] in zeroarg_flags):
- idx += 1
- continue
+ if (cmdline[idx][:2] in onearg_flags or cmdline[idx] in zeroarg_flags):
+ idx += 1
+ continue
- targets.append(cmdline[idx])
- idx += 1
+ targets.append(cmdline[idx])
+ idx += 1
- return targets
+ return targets
+
def GetJflag(cmdline):
- """Parse cmdline to get flag value for -j"""
+ """Parse cmdline to get flag value for -j"""
- for i in range(len(cmdline)):
- if (cmdline[i] == '-j' and i + 1 < len(cmdline) and
- cmdline[i+1].isdigit()):
- return int(cmdline[i+1])
+ for i in range(len(cmdline)):
+ if (cmdline[i] == '-j' and i + 1 < len(cmdline)
+ and cmdline[i + 1].isdigit()):
+ return int(cmdline[i + 1])
- if (cmdline[i].startswith('-j') and
- cmdline[i][len('-j'):].isdigit()):
- return int(cmdline[i][len('-j'):])
+ if (cmdline[i].startswith('-j') and cmdline[i][len('-j'):].isdigit()):
+ return int(cmdline[i][len('-j'):])
def GetMetadata(cmdline, ninjalog):
- """Get metadata for uploaded ninjalog.
+ """Get metadata for uploaded ninjalog.
Returned metadata has schema defined in
https://cs.chromium.org?q="type+Metadata+struct+%7B"+file:%5Einfra/go/src/infra/appengine/chromium_build_stats/ninjalog/
@@ -112,116 +112,122 @@
TODO(tikuta): Collect GOMA_* env var.
"""
- build_dir = os.path.dirname(ninjalog)
+ build_dir = os.path.dirname(ninjalog)
+ build_configs = {}
+
+ try:
+ args = ['gn', 'args', build_dir, '--list', '--short', '--json']
+ if sys.platform == 'win32':
+ # gn in PATH is bat file in windows environment (except cygwin).
+ args = ['cmd', '/c'] + args
+
+ gn_args = subprocess.check_output(args)
+ build_configs = ParseGNArgs(gn_args)
+ except subprocess.CalledProcessError as e:
+ logging.error("Failed to call gn %s", e)
build_configs = {}
- try:
- args = ['gn', 'args', build_dir, '--list', '--short', '--json']
- if sys.platform == 'win32':
- # gn in PATH is bat file in windows environment (except cygwin).
- args = ['cmd', '/c'] + args
+ # Stringify config.
+ for k in build_configs:
+ build_configs[k] = str(build_configs[k])
- gn_args = subprocess.check_output(args)
- build_configs = ParseGNArgs(gn_args)
- except subprocess.CalledProcessError as e:
- logging.error("Failed to call gn %s", e)
- build_configs = {}
+ metadata = {
+ 'platform': platform.system(),
+ 'cpu_core': multiprocessing.cpu_count(),
+ 'build_configs': build_configs,
+ 'targets': GetBuildTargetFromCommandLine(cmdline),
+ }
- # Stringify config.
- for k in build_configs:
- build_configs[k] = str(build_configs[k])
+ jflag = GetJflag(cmdline)
+ if jflag is not None:
+ metadata['jobs'] = jflag
- metadata = {
- 'platform': platform.system(),
- 'cpu_core': multiprocessing.cpu_count(),
- 'build_configs': build_configs,
- 'targets': GetBuildTargetFromCommandLine(cmdline),
- }
+ return metadata
- jflag = GetJflag(cmdline)
- if jflag is not None:
- metadata['jobs'] = jflag
-
- return metadata
def GetNinjalog(cmdline):
- """GetNinjalog returns the path to ninjalog from cmdline."""
- # ninjalog is in current working directory by default.
- ninjalog_dir = '.'
+ """GetNinjalog returns the path to ninjalog from cmdline."""
+ # ninjalog is in current working directory by default.
+ ninjalog_dir = '.'
- i = 0
- while i < len(cmdline):
- cmd = cmdline[i]
- i += 1
- if cmd == '-C' and i < len(cmdline):
- ninjalog_dir = cmdline[i]
- i += 1
- continue
+ i = 0
+ while i < len(cmdline):
+ cmd = cmdline[i]
+ i += 1
+ if cmd == '-C' and i < len(cmdline):
+ ninjalog_dir = cmdline[i]
+ i += 1
+ continue
- if cmd.startswith('-C') and len(cmd) > len('-C'):
- ninjalog_dir = cmd[len('-C'):]
+ if cmd.startswith('-C') and len(cmd) > len('-C'):
+ ninjalog_dir = cmd[len('-C'):]
- return os.path.join(ninjalog_dir, '.ninja_log')
+ return os.path.join(ninjalog_dir, '.ninja_log')
+
def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--server',
- default='chromium-build-stats.appspot.com',
- help='server to upload ninjalog file.')
- parser.add_argument('--ninjalog', help='ninjalog file to upload.')
- parser.add_argument('--verbose', action='store_true',
- help='Enable verbose logging.')
- parser.add_argument('--cmdline', required=True, nargs=argparse.REMAINDER,
- help='command line args passed to ninja.')
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--server',
+ default='chromium-build-stats.appspot.com',
+ help='server to upload ninjalog file.')
+ parser.add_argument('--ninjalog', help='ninjalog file to upload.')
+ parser.add_argument('--verbose',
+ action='store_true',
+ help='Enable verbose logging.')
+ parser.add_argument('--cmdline',
+ required=True,
+ nargs=argparse.REMAINDER,
+ help='command line args passed to ninja.')
- args = parser.parse_args()
+ args = parser.parse_args()
- if args.verbose:
- logging.basicConfig(level=logging.INFO)
- else:
- # Disable logging.
- logging.disable(logging.CRITICAL)
+ if args.verbose:
+ logging.basicConfig(level=logging.INFO)
+ else:
+ # Disable logging.
+ logging.disable(logging.CRITICAL)
- if not IsGoogler(args.server):
- return 0
-
-
- ninjalog = args.ninjalog or GetNinjalog(args.cmdline)
- if not os.path.isfile(ninjalog):
- logging.warn("ninjalog is not found in %s", ninjalog)
- return 1
-
- # We assume that each ninja invocation interval takes at least 2 seconds.
- # This is not to have duplicate entry in server when current build is no-op.
- if os.stat(ninjalog).st_mtime < time.time() - 2:
- logging.info("ninjalog is not updated recently %s", ninjalog)
- return 0
-
- output = cStringIO.StringIO()
-
- with open(ninjalog) as f:
- with gzip.GzipFile(fileobj=output, mode='wb') as g:
- g.write(f.read())
- g.write('# end of ninja log\n')
-
- metadata = GetMetadata(args.cmdline, ninjalog)
- logging.info('send metadata: %s', json.dumps(metadata))
- g.write(json.dumps(metadata))
-
- h = httplib2.Http()
- resp_headers, content = h.request(
- 'https://'+args.server+'/upload_ninja_log/', 'POST',
- body=output.getvalue(), headers={'Content-Encoding': 'gzip'})
-
- if resp_headers.status != 200:
- logging.warn("unexpected status code for response: %s",
- resp_headers.status)
- return 1
-
- logging.info('response header: %s', resp_headers)
- logging.info('response content: %s', content)
+ if not IsGoogler(args.server):
return 0
+ ninjalog = args.ninjalog or GetNinjalog(args.cmdline)
+ if not os.path.isfile(ninjalog):
+ logging.warn("ninjalog is not found in %s", ninjalog)
+ return 1
+
+ # We assume that each ninja invocation interval takes at least 2 seconds.
+ # This is not to have duplicate entry in server when current build is no-op.
+ if os.stat(ninjalog).st_mtime < time.time() - 2:
+ logging.info("ninjalog is not updated recently %s", ninjalog)
+ return 0
+
+ output = cStringIO.StringIO()
+
+ with open(ninjalog) as f:
+ with gzip.GzipFile(fileobj=output, mode='wb') as g:
+ g.write(f.read())
+ g.write('# end of ninja log\n')
+
+ metadata = GetMetadata(args.cmdline, ninjalog)
+ logging.info('send metadata: %s', json.dumps(metadata))
+ g.write(json.dumps(metadata))
+
+ h = httplib2.Http()
+ resp_headers, content = h.request('https://' + args.server +
+ '/upload_ninja_log/',
+ 'POST',
+ body=output.getvalue(),
+ headers={'Content-Encoding': 'gzip'})
+
+ if resp_headers.status != 200:
+ logging.warn("unexpected status code for response: %s", resp_headers.status)
+ return 1
+
+ logging.info('response header: %s', resp_headers)
+ logging.info('response content: %s', content)
+ return 0
+
+
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
diff --git a/ninjalog_uploader_wrapper.py b/ninjalog_uploader_wrapper.py
index 8cdb5d9..ae45d25 100755
--- a/ninjalog_uploader_wrapper.py
+++ b/ninjalog_uploader_wrapper.py
@@ -19,30 +19,30 @@
def LoadConfig():
- if os.path.isfile(CONFIG):
- with open(CONFIG, 'rb') as f:
- config = json.load(f)
- if config['version'] == VERSION:
- config['countdown'] = max(0, config['countdown'] - 1)
- return config
+ if os.path.isfile(CONFIG):
+ with open(CONFIG, 'rb') as f:
+ config = json.load(f)
+ if config['version'] == VERSION:
+ config['countdown'] = max(0, config['countdown'] - 1)
+ return config
- return {
- 'is-googler': ninjalog_uploader.IsGoogler(
- 'chromium-build-stats.appspot.com'),
- 'countdown': 10,
- 'version': VERSION,
- }
+ return {
+ 'is-googler':
+ ninjalog_uploader.IsGoogler('chromium-build-stats.appspot.com'),
+ 'countdown': 10,
+ 'version': VERSION,
+ }
def SaveConfig(config):
- with open(CONFIG, 'wb') as f:
- json.dump(config, f)
+ with open(CONFIG, 'wb') as f:
+ json.dump(config, f)
def ShowMessage(countdown):
- whitelisted = '\n'.join([' * %s' % config for config in
- ninjalog_uploader.WHITELISTED_CONFIGS])
- print("""
+ whitelisted = '\n'.join(
+ [' * %s' % config for config in ninjalog_uploader.WHITELISTED_CONFIGS])
+ print("""
Your ninjalog will be uploaded to build stats server. The uploaded log will be
used to analyze user side build performance.
@@ -73,46 +73,47 @@
def main():
- config = LoadConfig()
+ config = LoadConfig()
- if len(sys.argv) == 2 and sys.argv[1] == 'opt-in':
- config['opt-in'] = True
- config['countdown'] = 0
- SaveConfig(config)
- print('ninjalog upload is opted in.')
- return 0
+ if len(sys.argv) == 2 and sys.argv[1] == 'opt-in':
+ config['opt-in'] = True
+ config['countdown'] = 0
+ SaveConfig(config)
+ print('ninjalog upload is opted in.')
+ return 0
- if len(sys.argv) == 2 and sys.argv[1] == 'opt-out':
- config['opt-in'] = False
- SaveConfig(config)
- print('ninjalog upload is opted out.')
- return 0
+ if len(sys.argv) == 2 and sys.argv[1] == 'opt-out':
+ config['opt-in'] = False
+ SaveConfig(config)
+ print('ninjalog upload is opted out.')
+ return 0
- if 'opt-in' in config and not config['opt-in']:
- # Upload is opted out.
- return 0
+ if 'opt-in' in config and not config['opt-in']:
+ # Upload is opted out.
+ return 0
- if not config.get("is-googler", False):
- # Not googler.
- return 0
+ if not config.get("is-googler", False):
+ # Not googler.
+ return 0
- if config.get("countdown", 0) > 0:
- # Need to show message.
- ShowMessage(config["countdown"])
- # Only save config if something has meaningfully changed.
- SaveConfig(config)
- return 0
+ if config.get("countdown", 0) > 0:
+ # Need to show message.
+ ShowMessage(config["countdown"])
+ # Only save config if something has meaningfully changed.
+ SaveConfig(config)
+ return 0
- if len(sys.argv) == 1:
- # dry-run for debugging.
- print("upload ninjalog dry-run")
- return 0
+ if len(sys.argv) == 1:
+ # dry-run for debugging.
+ print("upload ninjalog dry-run")
+ return 0
- # Run upload script without wait.
- devnull = open(os.devnull, "w")
- subprocess2.Popen(['vpython', UPLOADER] + sys.argv[1:],
- stdout=devnull, stderr=devnull)
+ # Run upload script without wait.
+ devnull = open(os.devnull, "w")
+ subprocess2.Popen(['vpython', UPLOADER] + sys.argv[1:],
+ stdout=devnull,
+ stderr=devnull)
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
