app-admin/stackdriver: rename container log fields
This change modifies the cos specific fluentd config used
by the logging agent so that metadata field keys names
(such as 'stream' and 'container_id') are unique to avoid
potential collisions with user created key names within
json logs. Also adds some clarification comments.
See b/137508822#comment24 for examples of new behavior.
Comment on release note being None: this change
preemptively fixes an issue that was introduced with CL
10520, therefore another release note would be redundant.
BUG=b/137508822
TEST=presubmit, validated behavior on VM
RELEASE_NOTE=None
Change-Id: I21d5aea85dfc2deed071d72909815aa8de49a9ed
Reviewed-on: https://cos-review.googlesource.com/c/cos/overlays/board-overlays/+/10700
Reviewed-by: Roy Yang <royyang@google.com>
Tested-by: Cusky Presubmit Bot <presubmit@cos-infra-prod.iam.gserviceaccount.com>
diff --git a/project-lakitu/app-admin/stackdriver/files/logging_configs/fluentd-lakitu.conf b/project-lakitu/app-admin/stackdriver/files/logging_configs/fluentd-lakitu.conf
index 8445617..08c1b61 100644
--- a/project-lakitu/app-admin/stackdriver/files/logging_configs/fluentd-lakitu.conf
+++ b/project-lakitu/app-admin/stackdriver/files/logging_configs/fluentd-lakitu.conf
@@ -77,22 +77,27 @@
read_from_head true
</source>
-# Adds container_id field in container logs.
+# Revise fields in container log records.
<match reform_containers.**>
@type record_reformer
enable_ruby true
<record>
+ # 1) Add container_id field in container logs.
# tag_parts[] looks like:
# ['reform_containers', 'var', 'lib', 'docker', 'containers', container_id]
- container_id ${tag_parts[5]}
- # Renames field 'log' to a more generic field 'message'. This way Stackdriver
- # will display the log message as the summary of the log entry.
+ cos.googleapis.com/container_id ${tag_parts[5]}
+ # 2) Rename field 'stream' to avoid collisions from container logs where
+ # users may be also using 'stream' as a key
+ cos.googleapis.com/stream ${record['stream']}
+ # 3) Rename field 'log' to a more generic field 'message'. This way Logs
+ # Explorer UI will display the log message as summary of the log entry.
message ${record['log']}
</record>
tag cos_containers
- remove_keys log
+ remove_keys log,stream
</match>
+# Parse message field in container log records as json, if applicable.
<filter cos_containers.**>
@type parser
key_name message
@@ -100,10 +105,14 @@
remove_key_name_field true
<parse>
@type multi_format
+ # multi_format will attempt to parse 'key_name' field from record based
+ # on 'format'. If a parse fails, it will move on to next 'format'.
<pattern>
+ # Attempt parsing as json
format json
</pattern>
<pattern>
+ # If above fails, leave as is (i.e. as text)
format none
</pattern>
</parse>
diff --git a/project-lakitu/app-admin/stackdriver/stackdriver-0.0.1-r20.ebuild b/project-lakitu/app-admin/stackdriver/stackdriver-0.0.1-r21.ebuild
similarity index 100%
rename from project-lakitu/app-admin/stackdriver/stackdriver-0.0.1-r20.ebuild
rename to project-lakitu/app-admin/stackdriver/stackdriver-0.0.1-r21.ebuild
