project-lakitu: remove token headers in the request - ec2
header redact logic is introduced both in request and logs,
resulting in the request to Ec2 header invalid. To avoid,remove
redaction in request which is fixed in the upstream.
BUG=b/180330997
TEST=presubmit
RELEASE_NOTE=None
Change-Id: I32341fbd5db10ec3cbbbbcccf08ec3fd2e11f526
Reviewed-on: https://cos-review.googlesource.com/c/cos/overlays/board-overlays/+/12430
Reviewed-by: Varsha Teratipally <teratipally@google.com>
Tested-by: Cusky Presubmit Bot <presubmit@cos-infra-prod.iam.gserviceaccount.com>
diff --git a/project-lakitu/app-emulation/cloud-init/cloud-init-20.1-r6.ebuild b/project-lakitu/app-emulation/cloud-init/cloud-init-20.1-r7.ebuild
similarity index 100%
rename from project-lakitu/app-emulation/cloud-init/cloud-init-20.1-r6.ebuild
rename to project-lakitu/app-emulation/cloud-init/cloud-init-20.1-r7.ebuild
diff --git a/project-lakitu/app-emulation/cloud-init/cloud-init-20.1.ebuild b/project-lakitu/app-emulation/cloud-init/cloud-init-20.1.ebuild
index 6ede201..cd2e68d 100644
--- a/project-lakitu/app-emulation/cloud-init/cloud-init-20.1.ebuild
+++ b/project-lakitu/app-emulation/cloud-init/cloud-init-20.1.ebuild
@@ -60,6 +60,10 @@
"${FILESDIR}"/18.4-fix-update_package_sources-function.patch
"${FILESDIR}"/18.4-add-support-for-package_upgrade.patch
+ # Upstream fix for invalid request for ec2.
+ # Remove this after update for versions greater than 20.3
+ "${FILESDIR}"/20.1-ec2-redact-token-in-headers-not-request.patch
+
# For lakitu
"${FILESDIR}/${PV}-stable-uid.patch"
"${FILESDIR}/${PV}-fix-cross-compile.patch"
diff --git a/project-lakitu/app-emulation/cloud-init/files/20.1-ec2-redact-token-in-headers-not-request.patch b/project-lakitu/app-emulation/cloud-init/files/20.1-ec2-redact-token-in-headers-not-request.patch
new file mode 100644
index 0000000..b9a8569
--- /dev/null
+++ b/project-lakitu/app-emulation/cloud-init/files/20.1-ec2-redact-token-in-headers-not-request.patch
@@ -0,0 +1,111 @@
+From: varsha teratipally <teratipally@google.com>
+Date: Mon, 15 Feb 2021 00:07:25 +0000
+Subject:Cherry-pick of an upstream patch
+https://github.com/canonical/cloud-init/pull/230/commits
+/8760e675a6be774e2ed9a8a97934fb3bd463b1c7
+
+From 8760e675a6be774e2ed9a8a97934fb3bd463b1c7 Mon Sep 17 00:00:00 2001
+From: Chad Smith <chad.smith@canonical.com>
+Date: Tue, 3 Mar 2020 06:42:15 -0700
+Subject: [PATCH] ec2: only redact token request headers in logs, avoid
+ altering request
+
+Our header redact logic was redacting both logged request headers and
+the actual source request. This results in DataSourceEc2 sending the
+invalid header "X-aws-ec2-metadata-token-ttl-seconds: REDACTED" which
+gets an HTTP status response of 400.
+
+Cloud-init retries this failed token request for 2 minutes before
+falling back to IMDSv1.
+
+LP: #1865882
+---
+ cloudinit/tests/test_url_helper.py | 34 +++++++++++++++++++++++++++++-
+ cloudinit/url_helper.py | 15 +++++++------
+ 2 files changed, 41 insertions(+), 8 deletions(-)
+
+diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py
+index 1674120fb..29b39374a 100644
+--- a/cloudinit/tests/test_url_helper.py
++++ b/cloudinit/tests/test_url_helper.py
+@@ -1,7 +1,8 @@
+ # This file is part of cloud-init. See LICENSE file for license information.
+
+ from cloudinit.url_helper import (
+- NOT_FOUND, UrlError, oauth_headers, read_file_or_url, retry_on_url_exc)
++ NOT_FOUND, UrlError, REDACTED, oauth_headers, read_file_or_url,
++ retry_on_url_exc)
+ from cloudinit.tests.helpers import CiTestCase, mock, skipIf
+ from cloudinit import util
+ from cloudinit import version
+@@ -50,6 +51,9 @@ def sign(self, url):
+
+
+ class TestReadFileOrUrl(CiTestCase):
++
++ with_logs = True
++
+ def test_read_file_or_url_str_from_file(self):
+ """Test that str(result.contents) on file is text version of contents.
+ It should not be "b'data'", but just "'data'" """
+@@ -71,6 +75,34 @@ def test_read_file_or_url_str_from_url(self):
+ self.assertEqual(result.contents, data)
+ self.assertEqual(str(result), data.decode('utf-8'))
+
++ @httpretty.activate
++ def test_read_file_or_url_str_from_url_redacting_headers_from_logs(self):
++ """Headers are redacted from logs but unredacted in requests."""
++ url = 'http://hostname/path'
++ headers = {'sensitive': 'sekret', 'server': 'blah'}
++ httpretty.register_uri(httpretty.GET, url)
++
++ read_file_or_url(url, headers=headers, headers_redact=['sensitive'])
++ logs = self.logs.getvalue()
++ for k in headers.keys():
++ self.assertEqual(headers[k], httpretty.last_request().headers[k])
++ self.assertIn(REDACTED, logs)
++ self.assertNotIn('sekret', logs)
++
++ @httpretty.activate
++ def test_read_file_or_url_str_from_url_redacts_noheaders(self):
++ """When no headers_redact, header values are in logs and requests."""
++ url = 'http://hostname/path'
++ headers = {'sensitive': 'sekret', 'server': 'blah'}
++ httpretty.register_uri(httpretty.GET, url)
++
++ read_file_or_url(url, headers=headers)
++ for k in headers.keys():
++ self.assertEqual(headers[k], httpretty.last_request().headers[k])
++ logs = self.logs.getvalue()
++ self.assertNotIn(REDACTED, logs)
++ self.assertIn('sekret', logs)
++
+ @mock.patch(M_PATH + 'readurl')
+ def test_read_file_or_url_passes_params_to_readurl(self, m_readurl):
+ """read_file_or_url passes all params through to readurl."""
+diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
+index eeb27aa86..f3c0cf9c0 100644
+--- a/cloudinit/url_helper.py
++++ b/cloudinit/url_helper.py
+@@ -281,13 +281,14 @@ def _cb(url):
+ for (k, v) in req_args.items():
+ if k == 'data':
+ continue
+- filtered_req_args[k] = v
+- if k == 'headers':
+- for hkey, _hval in v.items():
+- if hkey in headers_redact:
+- filtered_req_args[k][hkey] = (
+- copy.deepcopy(req_args[k][hkey]))
+- filtered_req_args[k][hkey] = REDACTED
++ if k == 'headers' and headers_redact:
++ matched_headers = [k for k in headers_redact if v.get(k)]
++ if matched_headers:
++ filtered_req_args[k] = copy.deepcopy(v)
++ for key in matched_headers:
++ filtered_req_args[k][key] = REDACTED
++ else:
++ filtered_req_args[k] = v
+ try:
+
+ if log_req_resp:
