devserver: move scripts to venv.
There's a dependency error in importing google.protobuf.descriptor.
To avoid further troubles, this CL tends to move monitor scripts for
devserver to venv.
BUG=chromium:708890
TEST=Run bin/apache_log_metrics & bin/tail_until_writer_finished.
Change-Id: Ib1b69e12aea1e09c3d2c6a3d41bc796070d9daa0
Reviewed-on: https://chromium-review.googlesource.com/490707
Commit-Ready: Xixuan Wu <xixuan@chromium.org>
Tested-by: Xixuan Wu <xixuan@chromium.org>
Reviewed-by: Xixuan Wu <xixuan@chromium.org>
diff --git a/apache_log_metrics.py b/apache_log_metrics.py
index 0bd1a4d..9c9d52c 100755
--- a/apache_log_metrics.py
+++ b/apache_log_metrics.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python2
# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
@@ -15,8 +15,12 @@
import re
import sys
-from devserver import MakeLogHandler
+# TODO(ayatane): Fix cros lint pylint to work with virtualenv imports
+# pylint: disable=import-error
+from devserver_lib.devserver import MakeLogHandler
+# only import setup_chromite before chromite import.
+import setup_chromite # pylint: disable=unused-import
from chromite.lib import ts_mon_config
from chromite.lib import metrics
from chromite.lib import cros_logging as logging
@@ -188,7 +192,7 @@
"""Parses command line arguments."""
p = argparse.ArgumentParser(
description='Parses apache logs and emits metrics to Monarch')
- p.add_argument('--logfile')
+ p.add_argument('--logfile', required=True)
return p.parse_args()
@@ -196,6 +200,7 @@
"""Sets up logging and runs matchers against stdin"""
args = ParseArgs()
root = logging.getLogger()
+
root.addHandler(MakeLogHandler(args.logfile))
root.setLevel(logging.DEBUG)
ts_mon_config.SetupTsMonGlobalState('devserver_apache_log_metrics')
diff --git a/bin/apache_log_metrics b/bin/apache_log_metrics
new file mode 100755
index 0000000..09d3df1
--- /dev/null
+++ b/bin/apache_log_metrics
@@ -0,0 +1,14 @@
+#!/bin/bash
+# Copyright 2017 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Run turtle graphics inside virtualenv.
+set -eu
+
+original=$(pwd)
+cd -- "$(readlink -e -- "$(dirname -- "$0")")"
+. ./find_virtualenv.sh
+cd -- "$original"
+
+exec_python_module devserver_lib.apache_log_metrics "$@"
diff --git a/bin/find_virtualenv.sh b/bin/find_virtualenv.sh
new file mode 100644
index 0000000..87972d2
--- /dev/null
+++ b/bin/find_virtualenv.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright 2017 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Find the virtualenv repo to use.
+#
+# This is meant to be sourced in scripts that need virtualenv. The
+# sourcing script should cd into the directory containing this script
+# first. This script defines functions for performing common tasks:
+#
+# exec_python_module -- Execute Python module inside of virtualenv
+set -eu
+
+realpath() {
+ readlink -f -- "$1"
+}
+
+readonly venv_repo=$(realpath ../../../../infra_virtualenv)
+readonly create_script=$(realpath "${venv_repo}/bin/create_venv")
+readonly venv_home=$(realpath ../venv)
+readonly reqs_file=$(realpath "${venv_home}/requirements.txt")
+
+exec_python_module() {
+ venvdir=$("${create_script}" "$reqs_file")
+ export PYTHONPATH=${venv_home}
+ exec "${venvdir}/bin/python" -m "$@"
+}
diff --git a/bin/tail_until_writer_finished b/bin/tail_until_writer_finished
new file mode 100755
index 0000000..bfb3ad3
--- /dev/null
+++ b/bin/tail_until_writer_finished
@@ -0,0 +1,14 @@
+#!/bin/bash
+# Copyright 2017 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Run turtle graphics inside virtualenv.
+set -eu
+
+original=$(pwd)
+cd -- "$(readlink -e -- "$(dirname -- "$0")")"
+. ./find_virtualenv.sh
+cd -- "$original"
+
+exec_python_module devserver_lib.tail_until_writer_finished "$@"
diff --git a/setup_chromite.py b/setup_chromite.py
index 8bd2d38..2d42373 100644
--- a/setup_chromite.py
+++ b/setup_chromite.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2016 The Chromium OS Authors. All rights reserved.
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
@@ -15,4 +15,11 @@
if _path not in sys.path:
sys.path.insert(0, os.path.abspath(_path))
+# TODO(xixuan): crbug.com/717173
+# delete this once there's no third-party packages in chromite.
+try:
+ import chromite # pylint: disable=unused-import
+except ImportError as e:
+ chromite = None
+
del _path
diff --git a/venv/.gitignore b/venv/.gitignore
new file mode 100644
index 0000000..126a1a9
--- /dev/null
+++ b/venv/.gitignore
@@ -0,0 +1 @@
+/.venv*
diff --git a/venv/chromite b/venv/chromite
new file mode 120000
index 0000000..d780b88
--- /dev/null
+++ b/venv/chromite
@@ -0,0 +1 @@
+../../../../chromite
\ No newline at end of file
diff --git a/venv/devserver_lib b/venv/devserver_lib
new file mode 120000
index 0000000..a96aa0e
--- /dev/null
+++ b/venv/devserver_lib
@@ -0,0 +1 @@
+..
\ No newline at end of file
diff --git a/venv/requirements.txt b/venv/requirements.txt
new file mode 100644
index 0000000..47f22d2
--- /dev/null
+++ b/venv/requirements.txt
@@ -0,0 +1,35 @@
+cherrypy==3.2.2
+
+# Copied from chromite on 2016-04-27
+enum34==1.1.6
+funcsigs==1.0.2
+future==0.15.2
+futures==3.0.5
+gax-google-logging-v2==0.8.1
+gax-google-pubsub-v1==0.8.1
+gcloud==0.18.1
+gitdb2==2.0.0
+GitPython==2.1.1
+google-gax==0.12.5
+googleapis-common-protos==1.3.3
+grpc-google-logging-v2==0.8.1
+grpc-google-pubsub-v1==0.8.1
+grpcio==1.0.0
+httplib2==0.9.2
+mock==2.0.0
+mox==0.5.3
+MySQL-python==1.2.5
+numpy==1.11.1
+oauth2client==3.0.0
+pbr==1.10.0
+ply==3.8
+protobuf==3.0.0
+psutil==4.3.1
+pyasn1==0.1.9
+pyasn1-modules==0.0.8
+pyparsing==2.1.8
+PyYAML==3.12
+rsa==3.4.2
+six==1.10.0
+smmap2==2.0.1
+SQLAlchemy==1.0.15
