From a3fdcdc25b91cffaf9543e170e5d97ddb1e5b05f Mon Sep 17 00:00:00 2001 From: karan k Date: Fri, 7 Jul 2023 15:54:27 +0530 Subject: [PATCH 1/5] Support system layer in service section of fluenbit Signed-off-by: karan k --- .../v1alpha2/clusterfluentbitconfig_types.go | 46 ++++++++++ .../v1alpha2/plugins/input/systemd_types.go | 12 +++ .../v1alpha2/plugins/input/tail_types.go | 12 +++ .../plugins/output/open_search_types.go | 5 ++ ...bit.fluent.io_clusterfluentbitconfigs.yaml | 49 +++++++++++ .../fluentbit.fluent.io_clusterinputs.yaml | 30 +++++++ .../fluentbit.fluent.io_clusteroutputs.yaml | 4 + .../crds/fluentbit.fluent.io_outputs.yaml | 4 + .../fluentbit-clusterinput-systemd.yaml | 4 + .../fluentbit-clusterinput-tail.yaml | 4 + .../fluentbitconfig-fluentBitConfig.yaml | 4 + charts/fluent-operator/values.yaml | 6 ++ ...bit.fluent.io_clusterfluentbitconfigs.yaml | 49 +++++++++++ .../fluentbit.fluent.io_clusterinputs.yaml | 30 +++++++ .../fluentbit.fluent.io_clusteroutputs.yaml | 4 + .../bases/fluentbit.fluent.io_outputs.yaml | 4 + manifests/setup/fluent-operator-crd.yaml | 87 +++++++++++++++++++ manifests/setup/setup.yaml | 87 +++++++++++++++++++ 18 files changed, 441 insertions(+) diff --git a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go index 743852d9c..d3e1aa80e 100644 --- a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go +++ b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go @@ -46,6 +46,27 @@ type FluentBitConfigSpec struct { Namespace *string `json:"namespace,omitempty"` } +type Storage struct { + // Select an optional location in the file system to store streams and chunks of data/ + Path string `json:"path,omitempty"` + // Configure the synchronization mode used to store the data into the file system + // +kubebuilder:validation:Enum:=normal;full + Sync string `json:"sync,omitempty"` + // Enable the data integrity check when writing and reading data from the filesystem + // +kubebuilder:validation:Enum:=on;off + Checksum string `json:"checksum,omitempty"` + // This option configure a hint of maximum value of memory to use when processing these records + BacklogMemLimit string `json:"backlogMemLimit,omitempty"` + // If the input plugin has enabled filesystem storage type, this property sets the maximum number of Chunks that can be up in memory + MaxChunksUp string `json:"maxChunksUp,omitempty"` + // If http_server option has been enabled in the Service section, this option registers a new endpoint where internal metrics of the storage layer can be consumed + // +kubebuilder:validation:Enum:=on;off + Metrics string `json:"metrics,omitempty"` + // When enabled, irrecoverable chunks will be deleted during runtime, and any other irrecoverable chunk located in the configured storage path directory will be deleted when Fluent-Bit starts. + // +kubebuilder:validation:Enum:=on;off + DeleteIrrecoverableChunks string `json:"deleteIrrecoverableChunks,omitempty"` +} + type Service struct { // If true go to background on start Daemon *bool `json:"daemon,omitempty"` @@ -80,6 +101,8 @@ type Service struct { LogLevel string `json:"logLevel,omitempty"` // Optional 'parsers' config file (can be multiple) ParsersFile string `json:"parsersFile,omitempty"` + // Configure a global environment for the storage layer in Service + Storage *Storage `json:"storage,omitempty"` } // +kubebuilder:object:root=true @@ -149,6 +172,29 @@ func (s *Service) Params() *params.KVs { if s.ParsersFile != "" { m.Insert("Parsers_File", s.ParsersFile) } + if s.Storage != nil { + if s.Storage.Path != "" { + m.Insert("storage.path", s.Storage.Path) + } + if s.Storage.Sync != "" { + m.Insert("storage.sync", s.Storage.Sync) + } + if s.Storage.Checksum != "" { + m.Insert("storage.checksum", s.Storage.Checksum) + } + if s.Storage.BacklogMemLimit != "" { + m.Insert("storage.backlog.mem_limit", s.Storage.BacklogMemLimit) + } + if s.Storage.Metrics != "" { + m.Insert("storage.metrics", s.Storage.Metrics) + } + if s.Storage.MaxChunksUp != "" { + m.Insert("storage.max_chunks_up", s.Storage.MaxChunksUp) + } + if s.Storage.DeleteIrrecoverableChunks != "" { + m.Insert("storage.delete_irrecoverable_chunks", s.Storage.DeleteIrrecoverableChunks) + } + } return m } diff --git a/apis/fluentbit/v1alpha2/plugins/input/systemd_types.go b/apis/fluentbit/v1alpha2/plugins/input/systemd_types.go index 5d4ffac6b..a62cc3b60 100644 --- a/apis/fluentbit/v1alpha2/plugins/input/systemd_types.go +++ b/apis/fluentbit/v1alpha2/plugins/input/systemd_types.go @@ -44,6 +44,12 @@ type Systemd struct { // Remove the leading underscore of the Journald field (key). For example the Journald field _PID becomes the key PID. // +kubebuilder:validation:Enum:=on;off StripUnderscores string `json:"stripUnderscores,omitempty"` + // Specify the buffering mechanism to use. It can be memory or filesystem + // +kubebuilder:validation:Enum:=filesystem;memory + StorageType string `json:"storageType,omitempty"` + // Specifies if the input plugin should be paused (stop ingesting new data) when the storage.max_chunks_up value is reached. + // +kubebuilder:validation:Enum:=on;off + PauseOnChunksOverlimit string `json:"PauseOnChunksOverlimit,omitempty"` } func (_ *Systemd) Name() string { @@ -85,6 +91,12 @@ func (s *Systemd) Params(_ plugins.SecretLoader) (*params.KVs, error) { if s.StripUnderscores != "" { kvs.Insert("Strip_Underscores", s.StripUnderscores) } + if s.StorageType != "" { + kvs.Insert("storage.type", s.StorageType) + } + if s.PauseOnChunksOverlimit != "" { + kvs.Insert("storage.pause_on_chunks_overlimit", s.PauseOnChunksOverlimit) + } return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/input/tail_types.go b/apis/fluentbit/v1alpha2/plugins/input/tail_types.go index 149af9546..2176213b0 100644 --- a/apis/fluentbit/v1alpha2/plugins/input/tail_types.go +++ b/apis/fluentbit/v1alpha2/plugins/input/tail_types.go @@ -93,6 +93,12 @@ type Tail struct { // This will help to reassembly multiline messages originally split by Docker or CRI //Specify one or Multiline Parser definition to apply to the content. MultilineParser string `json:"multilineParser,omitempty"` + // Specify the buffering mechanism to use. It can be memory or filesystem + // +kubebuilder:validation:Enum:=filesystem;memory + StorageType string `json:"storageType,omitempty"` + // Specifies if the input plugin should be paused (stop ingesting new data) when the storage.max_chunks_up value is reached. + // +kubebuilder:validation:Enum:=on;off + PauseOnChunksOverlimit string `json:"PauseOnChunksOverlimit,omitempty"` } func (_ *Tail) Name() string { @@ -179,5 +185,11 @@ func (t *Tail) Params(_ plugins.SecretLoader) (*params.KVs, error) { if t.MultilineParser != "" { kvs.Insert("multiline.parser", t.MultilineParser) } + if t.StorageType != "" { + kvs.Insert("storage.type", t.StorageType) + } + if t.PauseOnChunksOverlimit != "" { + kvs.Insert("storage.pause_on_chunks_overlimit", t.PauseOnChunksOverlimit) + } return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/output/open_search_types.go b/apis/fluentbit/v1alpha2/plugins/output/open_search_types.go index 124af0e42..e5a3e3cd7 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/open_search_types.go +++ b/apis/fluentbit/v1alpha2/plugins/output/open_search_types.go @@ -104,6 +104,8 @@ type OpenSearch struct { // Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0. Workers *int32 `json:"Workers,omitempty"` *plugins.TLS `json:"tls,omitempty"` + // Limit the maximum number of Chunks in the filesystem for the current output logical destination. + TotalLimitSize string `json:"totalLimitSize,omitempty"` } // Name implement Section() method @@ -240,5 +242,8 @@ func (o *OpenSearch) Params(sl plugins.SecretLoader) (*params.KVs, error) { } kvs.Merge(tls) } + if o.TotalLimitSize != "" { + kvs.Insert("storage.total_limit_size", o.TotalLimitSize) + } return kvs, nil } diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml index cffdbe35f..86596342b 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -296,6 +296,55 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + storage: + description: Configure a global environment for the storage layer + in Service + properties: + backlogMemLimit: + description: This option configure a hint of maximum value + of memory to use when processing these records + type: string + checksum: + description: Enable the data integrity check when writing + and reading data from the filesystem + enum: + - "on" + - "off" + type: string + deleteIrrecoverableChunks: + description: When enabled, irrecoverable chunks will be deleted + during runtime, and any other irrecoverable chunk located + in the configured storage path directory will be deleted + when Fluent-Bit starts. + enum: + - "on" + - "off" + type: string + maxChunksUp: + description: If the input plugin has enabled filesystem storage + type, this property sets the maximum number of Chunks that + can be up in memory + type: string + metrics: + description: If http_server option has been enabled in the + Service section, this option registers a new endpoint where + internal metrics of the storage layer can be consumed + enum: + - "on" + - "off" + type: string + path: + description: Select an optional location in the file system + to store streams and chunks of data/ + type: string + sync: + description: Configure the synchronization mode used to store + the data into the file system + enum: + - normal + - full + type: string + type: object type: object type: object type: object diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml index d76be3455..982bcfe7d 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml @@ -134,6 +134,14 @@ spec: systemd: description: Systemd defines Systemd Input configuration. properties: + PauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string db: description: Specify the database file to keep track of monitored files and offsets. @@ -174,6 +182,13 @@ spec: - "on" - "off" type: string + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string stripUnderscores: description: Remove the leading underscore of the Journald field (key). For example the Journald field _PID becomes the key PID. @@ -209,6 +224,14 @@ spec: tail: description: Tail defines Tail Input configuration. properties: + PauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string bufferChunkSize: description: Set the initial buffer size to read files data. This value is used too to increase buffer size. The value must be @@ -341,6 +364,13 @@ spec: behavior and instruct Fluent Bit to skip long lines and continue processing other lines that fits into the buffer size. type: boolean + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string tag: description: Set a tag (with regex-extract fields) that will be placed on lines read. E.g. kube... diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml index c8c209c38..ee079e14a 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -1573,6 +1573,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml index 1dc2b500d..eecd9d870 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml @@ -1573,6 +1573,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error diff --git a/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml b/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml index 6e27b641e..c8ab19984 100644 --- a/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml +++ b/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml @@ -25,6 +25,10 @@ spec: {{- toYaml .Values.fluentbit.input.systemd.systemdFilter.filters | nindent 6 }} {{- end }} {{- end }} + storageType: {{ .Values.fluentbit.input.systemd.storageType }} + {{- if eq .Values.fluentbit.input.systemd.storageType "filesystem" }} + pauseOnChunksOverlimit: {{ .Values.fluentbit.input.systemd.pauseOnChunksOverlimit }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml b/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml index d2c36e323..0d870ed13 100644 --- a/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml +++ b/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml @@ -25,6 +25,10 @@ spec: skipLongLines: {{ .Values.fluentbit.input.tail.skipLongLines }} db: /fluent-bit/tail/pos.db dbSync: Normal + storageType: {{ .Values.fluentbit.input.tail.storageType }} + {{- if eq .Values.fluentbit.input.tail.storageType "filesystem" }} + pauseOnChunksOverlimit: {{ .Values.fluentbit.input.tail.pauseOnChunksOverlimit }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml b/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml index c7abf3faf..f5d44908c 100644 --- a/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml +++ b/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml @@ -9,6 +9,10 @@ metadata: spec: service: parsersFile: parsers.conf +{{- if .Values.fluentbit.service.storage }} + storage: +{{ toYaml .Values.fluentbit.service.storage | indent 6 }} +{{- end }} inputSelector: matchLabels: fluentbit.fluent.io/enabled: "true" diff --git a/charts/fluent-operator/values.yaml b/charts/fluent-operator/values.yaml index e405773b0..edd18b6f6 100644 --- a/charts/fluent-operator/values.yaml +++ b/charts/fluent-operator/values.yaml @@ -130,6 +130,8 @@ fluentbit: path: "/var/log/containers/*.log" skipLongLines: true readFromHead: false + storageType: memory + pauseOnChunksOverlimit: off systemd: enable: true systemdFilter: @@ -138,6 +140,8 @@ fluentbit: path: "/var/log/journal" includeKubelet: true stripUnderscores: "off" + storageType: memory + pauseOnChunksOverlimit: off nodeExporterMetrics: {} # uncomment below nodeExporterMetrics section if you want to collect node exporter metrics # nodeExporterMetrics: @@ -183,6 +187,8 @@ fluentbit: # You can configure the opensearch-related configuration here stdout: enable: false + service: + storage: {} # Configure the default filters in FluentBit. # The `filter` will filter and parse the collected log information and output the logs into a uniform format. You can choose whether to turn this on or not. diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml index cffdbe35f..86596342b 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -296,6 +296,55 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + storage: + description: Configure a global environment for the storage layer + in Service + properties: + backlogMemLimit: + description: This option configure a hint of maximum value + of memory to use when processing these records + type: string + checksum: + description: Enable the data integrity check when writing + and reading data from the filesystem + enum: + - "on" + - "off" + type: string + deleteIrrecoverableChunks: + description: When enabled, irrecoverable chunks will be deleted + during runtime, and any other irrecoverable chunk located + in the configured storage path directory will be deleted + when Fluent-Bit starts. + enum: + - "on" + - "off" + type: string + maxChunksUp: + description: If the input plugin has enabled filesystem storage + type, this property sets the maximum number of Chunks that + can be up in memory + type: string + metrics: + description: If http_server option has been enabled in the + Service section, this option registers a new endpoint where + internal metrics of the storage layer can be consumed + enum: + - "on" + - "off" + type: string + path: + description: Select an optional location in the file system + to store streams and chunks of data/ + type: string + sync: + description: Configure the synchronization mode used to store + the data into the file system + enum: + - normal + - full + type: string + type: object type: object type: object type: object diff --git a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml index d76be3455..982bcfe7d 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml @@ -134,6 +134,14 @@ spec: systemd: description: Systemd defines Systemd Input configuration. properties: + PauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string db: description: Specify the database file to keep track of monitored files and offsets. @@ -174,6 +182,13 @@ spec: - "on" - "off" type: string + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string stripUnderscores: description: Remove the leading underscore of the Journald field (key). For example the Journald field _PID becomes the key PID. @@ -209,6 +224,14 @@ spec: tail: description: Tail defines Tail Input configuration. properties: + PauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string bufferChunkSize: description: Set the initial buffer size to read files data. This value is used too to increase buffer size. The value must be @@ -341,6 +364,13 @@ spec: behavior and instruct Fluent Bit to skip long lines and continue processing other lines that fits into the buffer size. type: boolean + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string tag: description: Set a tag (with regex-extract fields) that will be placed on lines read. E.g. kube... diff --git a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml index c8c209c38..ee079e14a 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml @@ -1573,6 +1573,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error diff --git a/config/crd/bases/fluentbit.fluent.io_outputs.yaml b/config/crd/bases/fluentbit.fluent.io_outputs.yaml index 1dc2b500d..eecd9d870 100644 --- a/config/crd/bases/fluentbit.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_outputs.yaml @@ -1573,6 +1573,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 78dd14220..b110258d0 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -1449,6 +1449,55 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + storage: + description: Configure a global environment for the storage layer + in Service + properties: + backlogMemLimit: + description: This option configure a hint of maximum value + of memory to use when processing these records + type: string + checksum: + description: Enable the data integrity check when writing + and reading data from the filesystem + enum: + - "on" + - "off" + type: string + deleteIrrecoverableChunks: + description: When enabled, irrecoverable chunks will be deleted + during runtime, and any other irrecoverable chunk located + in the configured storage path directory will be deleted + when Fluent-Bit starts. + enum: + - "on" + - "off" + type: string + maxChunksUp: + description: If the input plugin has enabled filesystem storage + type, this property sets the maximum number of Chunks that + can be up in memory + type: string + metrics: + description: If http_server option has been enabled in the + Service section, this option registers a new endpoint where + internal metrics of the storage layer can be consumed + enum: + - "on" + - "off" + type: string + path: + description: Select an optional location in the file system + to store streams and chunks of data/ + type: string + sync: + description: Configure the synchronization mode used to store + the data into the file system + enum: + - normal + - full + type: string + type: object type: object type: object type: object @@ -1771,6 +1820,14 @@ spec: systemd: description: Systemd defines Systemd Input configuration. properties: + PauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string db: description: Specify the database file to keep track of monitored files and offsets. @@ -1811,6 +1868,13 @@ spec: - "on" - "off" type: string + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string stripUnderscores: description: Remove the leading underscore of the Journald field (key). For example the Journald field _PID becomes the key PID. @@ -1846,6 +1910,14 @@ spec: tail: description: Tail defines Tail Input configuration. properties: + PauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string bufferChunkSize: description: Set the initial buffer size to read files data. This value is used too to increase buffer size. The value must be @@ -1978,6 +2050,13 @@ spec: behavior and instruct Fluent Bit to skip long lines and continue processing other lines that fits into the buffer size. type: boolean + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string tag: description: Set a tag (with regex-extract fields) that will be placed on lines read. E.g. kube... @@ -3564,6 +3643,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error @@ -23001,6 +23084,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index b06783bad..5e3e9f989 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -1449,6 +1449,55 @@ spec: parsersFile: description: Optional 'parsers' config file (can be multiple) type: string + storage: + description: Configure a global environment for the storage layer + in Service + properties: + backlogMemLimit: + description: This option configure a hint of maximum value + of memory to use when processing these records + type: string + checksum: + description: Enable the data integrity check when writing + and reading data from the filesystem + enum: + - "on" + - "off" + type: string + deleteIrrecoverableChunks: + description: When enabled, irrecoverable chunks will be deleted + during runtime, and any other irrecoverable chunk located + in the configured storage path directory will be deleted + when Fluent-Bit starts. + enum: + - "on" + - "off" + type: string + maxChunksUp: + description: If the input plugin has enabled filesystem storage + type, this property sets the maximum number of Chunks that + can be up in memory + type: string + metrics: + description: If http_server option has been enabled in the + Service section, this option registers a new endpoint where + internal metrics of the storage layer can be consumed + enum: + - "on" + - "off" + type: string + path: + description: Select an optional location in the file system + to store streams and chunks of data/ + type: string + sync: + description: Configure the synchronization mode used to store + the data into the file system + enum: + - normal + - full + type: string + type: object type: object type: object type: object @@ -1771,6 +1820,14 @@ spec: systemd: description: Systemd defines Systemd Input configuration. properties: + PauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string db: description: Specify the database file to keep track of monitored files and offsets. @@ -1811,6 +1868,13 @@ spec: - "on" - "off" type: string + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string stripUnderscores: description: Remove the leading underscore of the Journald field (key). For example the Journald field _PID becomes the key PID. @@ -1846,6 +1910,14 @@ spec: tail: description: Tail defines Tail Input configuration. properties: + PauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string bufferChunkSize: description: Set the initial buffer size to read files data. This value is used too to increase buffer size. The value must be @@ -1978,6 +2050,13 @@ spec: behavior and instruct Fluent Bit to skip long lines and continue processing other lines that fits into the buffer size. type: boolean + storageType: + description: Specify the buffering mechanism to use. It can be + memory or filesystem + enum: + - filesystem + - memory + type: string tag: description: Set a tag (with regex-extract fields) that will be placed on lines read. E.g. kube... @@ -3564,6 +3643,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error @@ -23001,6 +23084,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + totalLimitSize: + description: Limit the maximum number of Chunks in the filesystem + for the current output logical destination. + type: string traceError: description: When enabled print the elasticsearch API calls to stdout when elasticsearch returns an error From 30d9df8c2cbb57c2b980137752ca0c6cc16d1156 Mon Sep 17 00:00:00 2001 From: karan k Date: Fri, 7 Jul 2023 17:50:56 +0530 Subject: [PATCH 2/5] fix pauseOnChunksOverlimit --- .../v1alpha2/plugins/input/systemd_types.go | 2 +- .../v1alpha2/plugins/input/tail_types.go | 2 +- .../fluentbit.fluent.io_clusterinputs.yaml | 32 +++++++++---------- .../crds/fluentbit.fluent.io_fluentbits.yaml | 1 + .../fluentbit.fluent.io_clusterinputs.yaml | 32 +++++++++---------- manifests/setup/fluent-operator-crd.yaml | 32 +++++++++---------- manifests/setup/setup.yaml | 32 +++++++++---------- 7 files changed, 67 insertions(+), 66 deletions(-) diff --git a/apis/fluentbit/v1alpha2/plugins/input/systemd_types.go b/apis/fluentbit/v1alpha2/plugins/input/systemd_types.go index a62cc3b60..5e86575b2 100644 --- a/apis/fluentbit/v1alpha2/plugins/input/systemd_types.go +++ b/apis/fluentbit/v1alpha2/plugins/input/systemd_types.go @@ -49,7 +49,7 @@ type Systemd struct { StorageType string `json:"storageType,omitempty"` // Specifies if the input plugin should be paused (stop ingesting new data) when the storage.max_chunks_up value is reached. // +kubebuilder:validation:Enum:=on;off - PauseOnChunksOverlimit string `json:"PauseOnChunksOverlimit,omitempty"` + PauseOnChunksOverlimit string `json:"pauseOnChunksOverlimit,omitempty"` } func (_ *Systemd) Name() string { diff --git a/apis/fluentbit/v1alpha2/plugins/input/tail_types.go b/apis/fluentbit/v1alpha2/plugins/input/tail_types.go index 2176213b0..6ca911470 100644 --- a/apis/fluentbit/v1alpha2/plugins/input/tail_types.go +++ b/apis/fluentbit/v1alpha2/plugins/input/tail_types.go @@ -98,7 +98,7 @@ type Tail struct { StorageType string `json:"storageType,omitempty"` // Specifies if the input plugin should be paused (stop ingesting new data) when the storage.max_chunks_up value is reached. // +kubebuilder:validation:Enum:=on;off - PauseOnChunksOverlimit string `json:"PauseOnChunksOverlimit,omitempty"` + PauseOnChunksOverlimit string `json:"pauseOnChunksOverlimit,omitempty"` } func (_ *Tail) Name() string { diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml index 982bcfe7d..3549e1ff8 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml @@ -134,14 +134,6 @@ spec: systemd: description: Systemd defines Systemd Input configuration. properties: - PauseOnChunksOverlimit: - description: Specifies if the input plugin should be paused (stop - ingesting new data) when the storage.max_chunks_up value is - reached. - enum: - - "on" - - "off" - type: string db: description: Specify the database file to keep track of monitored files and offsets. @@ -175,6 +167,14 @@ spec: not set, the plugin will use default paths to read local-only logs. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromTail: description: Start reading new entries. Skip entries already stored in Journald. @@ -224,14 +224,6 @@ spec: tail: description: Tail defines Tail Input configuration. properties: - PauseOnChunksOverlimit: - description: Specifies if the input plugin should be paused (stop - ingesting new data) when the storage.max_chunks_up value is - reached. - enum: - - "on" - - "off" - type: string bufferChunkSize: description: Set the initial buffer size to read files data. This value is used too to increase buffer size. The value must be @@ -342,6 +334,14 @@ spec: file as part of the record. The value assigned becomes the key in the map. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromHead: description: For new discovered files on start (without a database offset/position), read the content from the head of the file, diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_fluentbits.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_fluentbits.yaml index 180e1c216..a4a8714e1 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_fluentbits.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_fluentbits.yaml @@ -4,6 +4,7 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null name: fluentbits.fluentbit.fluent.io spec: group: fluentbit.fluent.io diff --git a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml index 982bcfe7d..3549e1ff8 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml @@ -134,14 +134,6 @@ spec: systemd: description: Systemd defines Systemd Input configuration. properties: - PauseOnChunksOverlimit: - description: Specifies if the input plugin should be paused (stop - ingesting new data) when the storage.max_chunks_up value is - reached. - enum: - - "on" - - "off" - type: string db: description: Specify the database file to keep track of monitored files and offsets. @@ -175,6 +167,14 @@ spec: not set, the plugin will use default paths to read local-only logs. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromTail: description: Start reading new entries. Skip entries already stored in Journald. @@ -224,14 +224,6 @@ spec: tail: description: Tail defines Tail Input configuration. properties: - PauseOnChunksOverlimit: - description: Specifies if the input plugin should be paused (stop - ingesting new data) when the storage.max_chunks_up value is - reached. - enum: - - "on" - - "off" - type: string bufferChunkSize: description: Set the initial buffer size to read files data. This value is used too to increase buffer size. The value must be @@ -342,6 +334,14 @@ spec: file as part of the record. The value assigned becomes the key in the map. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromHead: description: For new discovered files on start (without a database offset/position), read the content from the head of the file, diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index b110258d0..e8d480cec 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -1820,14 +1820,6 @@ spec: systemd: description: Systemd defines Systemd Input configuration. properties: - PauseOnChunksOverlimit: - description: Specifies if the input plugin should be paused (stop - ingesting new data) when the storage.max_chunks_up value is - reached. - enum: - - "on" - - "off" - type: string db: description: Specify the database file to keep track of monitored files and offsets. @@ -1861,6 +1853,14 @@ spec: not set, the plugin will use default paths to read local-only logs. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromTail: description: Start reading new entries. Skip entries already stored in Journald. @@ -1910,14 +1910,6 @@ spec: tail: description: Tail defines Tail Input configuration. properties: - PauseOnChunksOverlimit: - description: Specifies if the input plugin should be paused (stop - ingesting new data) when the storage.max_chunks_up value is - reached. - enum: - - "on" - - "off" - type: string bufferChunkSize: description: Set the initial buffer size to read files data. This value is used too to increase buffer size. The value must be @@ -2028,6 +2020,14 @@ spec: file as part of the record. The value assigned becomes the key in the map. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromHead: description: For new discovered files on start (without a database offset/position), read the content from the head of the file, diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 5e3e9f989..55c822e02 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -1820,14 +1820,6 @@ spec: systemd: description: Systemd defines Systemd Input configuration. properties: - PauseOnChunksOverlimit: - description: Specifies if the input plugin should be paused (stop - ingesting new data) when the storage.max_chunks_up value is - reached. - enum: - - "on" - - "off" - type: string db: description: Specify the database file to keep track of monitored files and offsets. @@ -1861,6 +1853,14 @@ spec: not set, the plugin will use default paths to read local-only logs. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromTail: description: Start reading new entries. Skip entries already stored in Journald. @@ -1910,14 +1910,6 @@ spec: tail: description: Tail defines Tail Input configuration. properties: - PauseOnChunksOverlimit: - description: Specifies if the input plugin should be paused (stop - ingesting new data) when the storage.max_chunks_up value is - reached. - enum: - - "on" - - "off" - type: string bufferChunkSize: description: Set the initial buffer size to read files data. This value is used too to increase buffer size. The value must be @@ -2028,6 +2020,14 @@ spec: file as part of the record. The value assigned becomes the key in the map. type: string + pauseOnChunksOverlimit: + description: Specifies if the input plugin should be paused (stop + ingesting new data) when the storage.max_chunks_up value is + reached. + enum: + - "on" + - "off" + type: string readFromHead: description: For new discovered files on start (without a database offset/position), read the content from the head of the file, From fb893a3b02f0dfa8fe614628da33a7454800d574 Mon Sep 17 00:00:00 2001 From: karan k Date: Fri, 7 Jul 2023 18:06:29 +0530 Subject: [PATCH 3/5] fix --- apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go | 6 +++--- .../crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml | 3 ++- .../bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml | 3 ++- manifests/setup/fluent-operator-crd.yaml | 3 ++- manifests/setup/setup.yaml | 3 ++- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go index d3e1aa80e..9312541a2 100644 --- a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go +++ b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go @@ -58,7 +58,7 @@ type Storage struct { // This option configure a hint of maximum value of memory to use when processing these records BacklogMemLimit string `json:"backlogMemLimit,omitempty"` // If the input plugin has enabled filesystem storage type, this property sets the maximum number of Chunks that can be up in memory - MaxChunksUp string `json:"maxChunksUp,omitempty"` + MaxChunksUp *int64 `json:"maxChunksUp,omitempty"` // If http_server option has been enabled in the Service section, this option registers a new endpoint where internal metrics of the storage layer can be consumed // +kubebuilder:validation:Enum:=on;off Metrics string `json:"metrics,omitempty"` @@ -188,8 +188,8 @@ func (s *Service) Params() *params.KVs { if s.Storage.Metrics != "" { m.Insert("storage.metrics", s.Storage.Metrics) } - if s.Storage.MaxChunksUp != "" { - m.Insert("storage.max_chunks_up", s.Storage.MaxChunksUp) + if s.Storage.MaxChunksUp != nil { + m.Insert("storage.max_chunks_up", fmt.Sprint(*s.Storage.MaxChunksUp)) } if s.Storage.DeleteIrrecoverableChunks != "" { m.Insert("storage.delete_irrecoverable_chunks", s.Storage.DeleteIrrecoverableChunks) diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml index 86596342b..4d8bae71e 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -324,7 +324,8 @@ spec: description: If the input plugin has enabled filesystem storage type, this property sets the maximum number of Chunks that can be up in memory - type: string + format: int64 + type: integer metrics: description: If http_server option has been enabled in the Service section, this option registers a new endpoint where diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml index 86596342b..4d8bae71e 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterfluentbitconfigs.yaml @@ -324,7 +324,8 @@ spec: description: If the input plugin has enabled filesystem storage type, this property sets the maximum number of Chunks that can be up in memory - type: string + format: int64 + type: integer metrics: description: If http_server option has been enabled in the Service section, this option registers a new endpoint where diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index e8d480cec..2f05bd78d 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -1477,7 +1477,8 @@ spec: description: If the input plugin has enabled filesystem storage type, this property sets the maximum number of Chunks that can be up in memory - type: string + format: int64 + type: integer metrics: description: If http_server option has been enabled in the Service section, this option registers a new endpoint where diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 55c822e02..a48d4f59c 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -1477,7 +1477,8 @@ spec: description: If the input plugin has enabled filesystem storage type, this property sets the maximum number of Chunks that can be up in memory - type: string + format: int64 + type: integer metrics: description: If http_server option has been enabled in the Service section, this option registers a new endpoint where From f710d9dbe0339a14f3c6da05066b78504b430c1d Mon Sep 17 00:00:00 2001 From: karan k Date: Fri, 7 Jul 2023 19:45:17 +0530 Subject: [PATCH 4/5] fix --- charts/fluent-operator/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/fluent-operator/values.yaml b/charts/fluent-operator/values.yaml index edd18b6f6..3858288b4 100644 --- a/charts/fluent-operator/values.yaml +++ b/charts/fluent-operator/values.yaml @@ -131,7 +131,7 @@ fluentbit: skipLongLines: true readFromHead: false storageType: memory - pauseOnChunksOverlimit: off + pauseOnChunksOverlimit: "off" systemd: enable: true systemdFilter: @@ -141,7 +141,7 @@ fluentbit: includeKubelet: true stripUnderscores: "off" storageType: memory - pauseOnChunksOverlimit: off + pauseOnChunksOverlimit: "off" nodeExporterMetrics: {} # uncomment below nodeExporterMetrics section if you want to collect node exporter metrics # nodeExporterMetrics: From 63862cdaacdc856b5f40160d61d4ddf6e5e15385 Mon Sep 17 00:00:00 2001 From: karan k Date: Fri, 7 Jul 2023 19:55:24 +0530 Subject: [PATCH 5/5] fix --- .../templates/fluentbit-clusterinput-systemd.yaml | 2 +- .../fluent-operator/templates/fluentbit-clusterinput-tail.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml b/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml index c8ab19984..5ed37d645 100644 --- a/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml +++ b/charts/fluent-operator/templates/fluentbit-clusterinput-systemd.yaml @@ -27,7 +27,7 @@ spec: {{- end }} storageType: {{ .Values.fluentbit.input.systemd.storageType }} {{- if eq .Values.fluentbit.input.systemd.storageType "filesystem" }} - pauseOnChunksOverlimit: {{ .Values.fluentbit.input.systemd.pauseOnChunksOverlimit }} + pauseOnChunksOverlimit: {{ .Values.fluentbit.input.systemd.pauseOnChunksOverlimit | quote }} {{- end }} {{- end }} {{- end }} diff --git a/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml b/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml index 0d870ed13..76fd07387 100644 --- a/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml +++ b/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml @@ -27,7 +27,7 @@ spec: dbSync: Normal storageType: {{ .Values.fluentbit.input.tail.storageType }} {{- if eq .Values.fluentbit.input.tail.storageType "filesystem" }} - pauseOnChunksOverlimit: {{ .Values.fluentbit.input.tail.pauseOnChunksOverlimit }} + pauseOnChunksOverlimit: {{ .Values.fluentbit.input.tail.pauseOnChunksOverlimit | quote }} {{- end }} {{- end }} {{- end }}