From: varsha teratipally <teratipally@google.com>
Date: Mon, 15 Feb 2021 00:07:25 +0000
Subject:Cherry-pick of an upstream patch
https://github.com/canonical/cloud-init/pull/230/commits
/8760e675a6be774e2ed9a8a97934fb3bd463b1c7

From 8760e675a6be774e2ed9a8a97934fb3bd463b1c7 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Tue, 3 Mar 2020 06:42:15 -0700
Subject: [PATCH] ec2: only redact token request headers in logs, avoid
 altering request

Our header redact logic was redacting both logged request headers and
the actual source request. This results in DataSourceEc2 sending the
invalid header "X-aws-ec2-metadata-token-ttl-seconds: REDACTED" which
gets an HTTP status response of 400.

Cloud-init retries this failed token request for 2 minutes before
falling back to IMDSv1.

LP: #1865882
---
 cloudinit/tests/test_url_helper.py | 34 +++++++++++++++++++++++++++++-
 cloudinit/url_helper.py            | 15 +++++++------
 2 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py
index 1674120fb..29b39374a 100644
--- a/cloudinit/tests/test_url_helper.py
+++ b/cloudinit/tests/test_url_helper.py
@@ -1,7 +1,8 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit.url_helper import (
-    NOT_FOUND, UrlError, oauth_headers, read_file_or_url, retry_on_url_exc)
+    NOT_FOUND, UrlError, REDACTED, oauth_headers, read_file_or_url,
+    retry_on_url_exc)
 from cloudinit.tests.helpers import CiTestCase, mock, skipIf
 from cloudinit import util
 from cloudinit import version
@@ -50,6 +51,9 @@ def sign(self, url):
 
 
 class TestReadFileOrUrl(CiTestCase):
+
+    with_logs = True
+
     def test_read_file_or_url_str_from_file(self):
         """Test that str(result.contents) on file is text version of contents.
         It should not be "b'data'", but just "'data'" """
@@ -71,6 +75,34 @@ def test_read_file_or_url_str_from_url(self):
         self.assertEqual(result.contents, data)
         self.assertEqual(str(result), data.decode('utf-8'))
 
+    @httpretty.activate
+    def test_read_file_or_url_str_from_url_redacting_headers_from_logs(self):
+        """Headers are redacted from logs but unredacted in requests."""
+        url = 'http://hostname/path'
+        headers = {'sensitive': 'sekret', 'server': 'blah'}
+        httpretty.register_uri(httpretty.GET, url)
+
+        read_file_or_url(url, headers=headers, headers_redact=['sensitive'])
+        logs = self.logs.getvalue()
+        for k in headers.keys():
+            self.assertEqual(headers[k],  httpretty.last_request().headers[k])
+        self.assertIn(REDACTED, logs)
+        self.assertNotIn('sekret', logs)
+
+    @httpretty.activate
+    def test_read_file_or_url_str_from_url_redacts_noheaders(self):
+        """When no headers_redact, header values are in logs and requests."""
+        url = 'http://hostname/path'
+        headers = {'sensitive': 'sekret', 'server': 'blah'}
+        httpretty.register_uri(httpretty.GET, url)
+
+        read_file_or_url(url, headers=headers)
+        for k in headers.keys():
+            self.assertEqual(headers[k], httpretty.last_request().headers[k])
+        logs = self.logs.getvalue()
+        self.assertNotIn(REDACTED, logs)
+        self.assertIn('sekret', logs)
+
     @mock.patch(M_PATH + 'readurl')
     def test_read_file_or_url_passes_params_to_readurl(self, m_readurl):
         """read_file_or_url passes all params through to readurl."""
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index eeb27aa86..f3c0cf9c0 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -281,13 +281,14 @@ def _cb(url):
         for (k, v) in req_args.items():
             if k == 'data':
                 continue
-            filtered_req_args[k] = v
-            if k == 'headers':
-                for hkey, _hval in v.items():
-                    if hkey in headers_redact:
-                        filtered_req_args[k][hkey] = (
-                            copy.deepcopy(req_args[k][hkey]))
-                        filtered_req_args[k][hkey] = REDACTED
+            if k == 'headers' and headers_redact:
+                matched_headers = [k for k in headers_redact if v.get(k)]
+                if matched_headers:
+                    filtered_req_args[k] = copy.deepcopy(v)
+                    for key in matched_headers:
+                        filtered_req_args[k][key] = REDACTED
+            else:
+                filtered_req_args[k] = v
         try:
 
             if log_req_resp: