blob: 962f4bc631d049254af4f63454dfd9e8e33fe4d5 [file] [log] [blame]
#
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
[SERVICE]
# Flush
# =====
# set an interval of seconds before to flush records to a destination
flush 1
# Daemon
# ======
# instruct Fluent Bit to run in foreground or background mode.
daemon Off
# Log_Level
# =========
# Set the verbosity level of the service, values can be:
#
# - error
# - warning
# - info
# - debug
# - trace
#
# by default 'info' is set, that means it includes 'error' and 'warning'.
log_level info
# Parsers File
# ============
# specify an optional 'Parsers' configuration file
parsers_file parsers.conf
# Plugins File
# ============
# specify an optional 'Plugins' configuration file to load external plugins.
#
# plugins_file plugins.conf
# HTTP Server
# ===========
# Enable/Disable the built-in HTTP Server for metrics
http_server Off
# http_listen 0.0.0.0
# http_port 2020
# Storage
# =======
# Fluent Bit can use memory and filesystem buffering based mechanisms
#
# - https://docs.fluentbit.io/manual/administration/buffering-and-storage
#
# storage metrics
# ---------------
# publish storage pipeline metrics in '/api/v1/storage'. The metrics are
# exported only if the 'http_server' option is enabled.
#
storage.metrics on
# storage.path
# ------------
# absolute file system path to store filesystem data buffers (chunks).
#
# storage.path /tmp/storage
# storage.sync
# ------------
# configure the synchronization mode used to store the data into the
# filesystem. It can take the values normal or full.
#
# storage.sync normal
# storage.checksum
# ----------------
# enable the data integrity check when writing and reading data from the
# filesystem. The storage layer uses the CRC32 algorithm.
#
# storage.checksum off
# storage.backlog.mem_limit
# -------------------------
# if storage.path is set, Fluent Bit will look for data chunks that were
# not delivered and are still in the storage layer, these are called
# backlog data. This option configure a hint of maximum value of memory
# to use when processing these records.
#
# storage.backlog.mem_limit 5M
# Collects docker.service logs.
[INPUT]
Name systemd
Tag cos_docker
Systemd_Filter _SYSTEMD_UNIT=docker.service
DB /var/log/google-fluentbit/docker.log.db
Read_From_Tail False
# Collects COS system services logs.
[INPUT]
Name systemd
Tag cos_system
Systemd_Filter _SYSTEMD_UNIT=docker-events-collector-fluent-bit.service
Systemd_Filter _SYSTEMD_UNIT=konlet-startup.service
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Systemd_Filter _SYSTEMD_UNIT=crash-reporter.service
Systemd_Filter _SYSTEMD_UNIT=crash-sender.service
Systemd_Filter _SYSTEMD_UNIT=crash-boot-collect.service
Systemd_Filter _SYSTEMD_UNIT=kdump-load-kernel.service
Systemd_Filter _SYSTEMD_UNIT=kdump-save-dump.service
Systemd_Filter _SYSTEMD_UNIT=cis-level1.service
Systemd_Filter _SYSTEMD_UNIT=cis-level2.service
Systemd_Filter _SYSTEMD_UNIT=cis-compliance-scanner.service
Systemd_Filter SYSLOG_IDENTIFIER=crash-sender.sh
DB /var/log/google-fluentbit/system.log.db
Read_From_Tail False
# Collects COS audit logs.
[INPUT]
Name systemd
Tag cos_audit
Systemd_Filter SYSLOG_IDENTIFIER=audit
DB /var/log/google-fluentbit/audit.log.db
Read_From_Tail False
# Collects all journal logs with priority >= warning
# Change priority levels to make it more/less verbose.
[INPUT]
Name systemd
Tag cos_journal_warning
Systemd_Filter PRIORITY=0
Systemd_Filter PRIORITY=1
Systemd_Filter PRIORITY=2
Systemd_Filter PRIORITY=3
Systemd_Filter PRIORITY=4
DB /var/log/google-fluentbit/journal.db
Read_From_Tail False
# Docker container logs (when not running Kubernetes).
# This will collect logs from all containers using json file logging driver.
# To query logs for specific container, use below filter on GCP logging:
# jsonPayload.container_id=CONTAINER_ID
[INPUT]
Name tail
Tag cos_containers
Path /var/lib/docker/containers/*/*.log
Parser docker
DB /var/log/google-fluentbit/containers.log.db
Path_Key file_name
Read_from_Head True
Mem_Buf_Limit 5MB
Skip_Long_Lines On
# 1) Add container_id field in container logs.
[FILTER]
Name parser
Match cos_containers
Key_Name file_name
Parser container_filename_to_id
Reserve_Data On
# 2) The following two filters save attrs[tag] as container_name field in
# the record.
[FILTER]
Name nest
Match cos_containers
Operation lift
Nested_under attrs
Add_prefix attrs_
[FILTER]
Name modify
Match cos_containers
Condition Key_exists attrs_tag
Rename attrs_tag cos.googleapis.com/container_name
# If attrs[tag] doesn't exist, set container_name field as 'UNKNOWN'
[FILTER]
Name modify
Match cos_containers
Condition Key_does_not_exist attrs_tag
Add cos.googleapis.com/container_name UNKNOWN
[FILTER]
Name modify
Match cos_containers
Rename container_id cos.googleapis.com/container_id
# 3) Rename field 'stream' to avoid collisions from container logs where
# users may be also using 'stream' as a key
Rename stream cos.googleapis.com/stream
# 4) Rename field 'log' to a more generic field 'message'. This way Logs
# Explorer UI will display the log message as summary of the log entry.
Rename log message
# Remove other fields under attrs.
Remove_wildcard attrs_
# match fluentd jsonPayload structure
[FILTER]
Name nest
Match cos_containers
Operation lift
Nested_under message
[OUTPUT]
Name stackdriver
Match *
Resource gce_instance
severity_key severity