Merge pull request #232 from m-lab/sandbox-roberto-fluentd-mem-limits

Update fluentd and simplify configuration
m-lab · Jul 22, 2019 · 4402386 · 4402386
2 parents 8ceacd3 + 8bf3cc3
commit 4402386
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 187 deletions.
diff --git a/config/fluentd.jsonnet b/config/fluentd.jsonnet
@@ -12,6 +12,5 @@ local outputConf = std.strReplace(outputConfMissingZone, '{{GCE_ZONE}}', std.ext
     'containers.input.conf': importstr 'fluentd/containers.input.conf',
     'monitoring.conf': importstr 'fluentd/monitoring.conf',
     'output.conf': outputConf,
-    'system.input.conf': importstr 'fluentd/system.input.conf',
   },
 }
diff --git a/config/fluentd/containers.input.conf b/config/fluentd/containers.input.conf
@@ -66,19 +66,19 @@
 
 # Detect exceptions in the log output and forward them as one log entry.
 <match raw.kubernetes.**>
-  @type copy
+@type copy
 
   <store>
-    @type prometheus
+  @type prometheus
 
     <metric>
-      type counter
-      name logging_line_count
-      desc Total number of lines generated by application containers
-      <labels>
-        tag ${tag}
-      </labels>
-    </metric>
+    type counter
+    name logging_line_count
+    desc Total number of lines generated by application containers
+    <labels>
+      tag ${tag}
+    </labels>
+  </metric>
   </store>
   <store>
     @type detect_exceptions
@@ -90,4 +90,3 @@
     max_bytes 500000
     max_lines 1000
   </store>
-</match>
diff --git a/config/fluentd/output.conf.template b/config/fluentd/output.conf.template
@@ -1,7 +1,4 @@
-# We use 2 output stanzas - one to handle the container logs and one to handle
-# the node daemon logs. This allows us to label Prometheus metrics differently
-# depending on the component ("container" or "system").
-<match kubernetes.**>
+<match **>
   @type copy
 
   <store>
@@ -53,41 +50,3 @@
     </metric>
   </store>
 </match>
-
-<match **>
-  @type copy
-
-  <store>
-    @type google_cloud
-    use_metadata_service false
-
-    project_id {{PROJECT_ID}}
-    zone {{GCE_ZONE}}
-    vm_id NODE_HOSTNAME
-    vm_name NODE_HOSTNAME
-
-    detect_subservice false
-    buffer_type file
-    buffer_path /var/log/fluentd-buffers/kubernetes.system.buffer
-    buffer_queue_full_action block
-    buffer_chunk_limit 2M
-    buffer_queue_limit 2
-    flush_interval 5s
-    max_retry_wait 30
-    disable_retry_limit
-    num_threads 2
-  </store>
-  <store>
-    @type prometheus
-
-    <metric>
-      type counter
-      name logging_entry_count
-      desc Total number of log entries generated by either an application container or a system component
-      <labels>
-        tag ${tag}
-        component system
-      </labels>
-    </metric>
-  </store>
-</match>
diff --git a/config/fluentd/system.input.conf b/config/fluentd/system.input.conf
diff --git a/k8s/daemonsets/core/fluentd.jsonnet b/k8s/daemonsets/core/fluentd.jsonnet
@@ -46,7 +46,7 @@
             command: [
               '/bin/sh',
               '-c',
-              'mkdir /etc/fluent/config.d && cp /config/* /etc/fluent/config.d/ && sed -i "s/NODE_HOSTNAME/$NODE_HOSTNAME/" /etc/fluent/config.d/output.conf && /run.sh $FLUENTD_ARGS',
+              'mkdir /etc/fluent/config.d && cp /config/* /etc/fluent/config.d/ && sed -i "s/NODE_HOSTNAME/$NODE_HOSTNAME/" /etc/fluent/config.d/output.conf && /run.sh $FLUENTD_ARGS 2>&1 >>/var/log/fluentd.log',
             ],
             env: [
               {
@@ -66,7 +66,7 @@
                 },
               },
             ],
-            image: 'k8s.gcr.io/fluentd-gcp:2.0.2',
+            image: 'k8s.gcr.io/fluentd-gcp:2.1.1',
             name: 'fluentd',
             ports: [
               {