Set correct HTTP proxy when running gsutil
Gsutil disregards the environment variable http_proxy and only reads
proxy settings from the boto configuration file. This CL allows our GS
library to read http_proxy and then overrides the proxy settings when
running gsutil using '-o'.
BUG=chromium:318478
TEST=unittest, cros lint
Change-Id: I25dff79291fe46eb2852bdf16f25665896dec739
Reviewed-on: https://chromium-review.googlesource.com/179326
Reviewed-by: Yu-Ju Hong <yjhong@chromium.org>
Commit-Queue: Yu-Ju Hong <yjhong@chromium.org>
Tested-by: Yu-Ju Hong <yjhong@chromium.org>
diff --git a/lib/gs.py b/lib/gs.py
index 6d564fc..5e61495 100644
--- a/lib/gs.py
+++ b/lib/gs.py
@@ -177,7 +177,6 @@
GSUTIL_TAR = 'gsutil_3.37.tar.gz'
GSUTIL_URL = PUBLIC_BASE_HTTPS_URL + 'pub/%s' % GSUTIL_TAR
- GSUTIL_FLAGS = ['-o', 'GSUtil:parallel_composite_upload_threshold=0']
RESUMABLE_UPLOAD_ERROR = ('Too many resumable upload attempts failed without '
'progress')
@@ -239,6 +238,28 @@
self._CheckFile('gsutil not found', gsutil_bin)
self.gsutil_bin = gsutil_bin
+ # TODO (yjhong): disable parallel composite upload for now because
+ # it is not backward compatible (older gsutil versions cannot
+ # download files uploaded with this option enabled). Remove this
+ # after all users transition to newer versions (3.37 and above).
+ self.gsutil_flags = ['-o', 'GSUtil:parallel_composite_upload_threshold=0']
+
+ # Set HTTP proxy if environment variable http_proxy is set
+ # (crbug.com/325032).
+ if 'http_proxy' in os.environ:
+ url = urlparse.urlparse(os.environ['http_proxy'])
+ if not url.hostname or (not url.username and url.password):
+ logging.warning('GS_ERROR: Ignoring env variable http_proxy because it '
+ 'is not properly set: %s', os.environ['http_proxy'])
+ else:
+ self.gsutil_flags += ['-o', 'Boto:proxy=%s' % url.hostname]
+ if url.username:
+ self.gsutil_flags += ['-o', 'Boto:proxy_user=%s' % url.username]
+ if url.password:
+ self.gsutil_flags += ['-o', 'Boto:proxy_pass=%s' % url.password]
+ if url.port:
+ self.gsutil_flags += ['-o', 'Boto:proxy_port=%d' % url.port]
+
# Prefer boto_file if specified, else prefer the env then the default.
default_boto = False
if boto_file is None:
@@ -447,11 +468,7 @@
kwargs.setdefault('redirect_stderr', True)
cmd = [self.gsutil_bin]
- # TODO (yjhong): disable parallel composite upload for now because
- # it is not backward compatible (older gsutil versions cannot
- # download files uploaded with this option enbaled). Remove this
- # after all users transition to newer versions (3.37 above).
- cmd += self.GSUTIL_FLAGS
+ cmd += self.gsutil_flags
for header in headers:
cmd += ['-h', header]
cmd.extend(gsutil_cmd)
diff --git a/lib/gs_unittest.py b/lib/gs_unittest.py
index 3937f5f..7186087 100755
--- a/lib/gs_unittest.py
+++ b/lib/gs_unittest.py
@@ -215,6 +215,43 @@
self.assertEqual(gs.GSContext(acl=self.acl_file).acl,
self.acl_file)
+ def _testHTTPProxySettings(self, d):
+ flags = gs.GSContext().gsutil_flags
+ for key in d:
+ flag = 'Boto:%s=%s' % (key, d[key])
+ error_msg = '%s not in %s' % (flag, ' '.join(flags))
+ self.assertTrue(flag in flags, error_msg)
+
+ def testHTTPProxy(self):
+ """Test we set http proxy correctly."""
+ d = {'proxy': 'fooserver', 'proxy_user': 'foouser',
+ 'proxy_pass': 'foopasswd', 'proxy_port': '8080'}
+ os.environ['http_proxy'] = 'http://%s:%s@%s:%s/' % (
+ d['proxy_user'], d['proxy_pass'], d['proxy'], d['proxy_port'])
+ self._testHTTPProxySettings(d)
+
+ def testHTTPProxyNoPort(self):
+ """Test we accept http proxy without port number."""
+ d = {'proxy': 'fooserver', 'proxy_user': 'foouser',
+ 'proxy_pass': 'foopasswd'}
+ os.environ['http_proxy'] = 'http://%s:%s@%s/' % (
+ d['proxy_user'], d['proxy_pass'], d['proxy'])
+ self._testHTTPProxySettings(d)
+
+ def testHTTPProxyNoUserPasswd(self):
+ """Test we accept http proxy without user and password."""
+ d = {'proxy': 'fooserver', 'proxy_port': '8080'}
+ os.environ['http_proxy'] = 'http://%s:%s/' % (d['proxy'], d['proxy_port'])
+ self._testHTTPProxySettings(d)
+
+ def testHTTPProxyNoPasswd(self):
+ """Test we accept http proxy without password."""
+ d = {'proxy': 'fooserver', 'proxy_user': 'foouser',
+ 'proxy_port': '8080'}
+ os.environ['http_proxy'] = 'http://%s@%s:%s/' % (
+ d['proxy_user'], d['proxy'], d['proxy_port'])
+ self._testHTTPProxySettings(d)
+
class GSDoCommandTest(cros_test_lib.TestCase):
"""Tests of gs.DoCommand behavior.
@@ -233,7 +270,7 @@
def _testDoCommand(self, ctx, retries, sleep):
with mock.patch.object(cros_build_lib, 'GenericRetry', autospec=True):
ctx.Copy('/blah', 'gs://foon')
- cmd = [self.ctx.gsutil_bin] + self.ctx.GSUTIL_FLAGS
+ cmd = [self.ctx.gsutil_bin] + self.ctx.gsutil_flags
cmd += ['cp', '--', '/blah', 'gs://foon']
cros_build_lib.GenericRetry.assert_called_once_with(
