Nginx monitoring showcase. (#153)

deploy/platform/docker/Makefile.in

@@ -17,4 +17,4 @@
 #
 
 .EXPORT_ALL_VARIABLES:
-FEATURE_FLAGS ?= agent,cluster,so11y,vm-monitor,event,mysql-monitor,apisix-monitor,grafana,elasticsearch-monitor
+FEATURE_FLAGS ?= agent,cluster,so11y,vm-monitor,event,mysql-monitor,nginx-monitor,apisix-monitor,grafana,elasticsearch-monitor

deploy/platform/docker/config/nginx/Dockerfile.nginx

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM openresty/openresty:1.17.8.2-5-alpine-fat
+
+RUN luarocks install nginx-lua-prometheus
+RUN mkdir -p /var/log/nginx/

deploy/platform/docker/config/nginx/fluent-bit-script.lua

@@ -0,0 +1,36 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+function rewrite_access_log(tag, timestamp, record)
+    local newRecord = {}
+    newRecord["layer"] = "NGINX"
+    newRecord["service"] = "nginx::nginx"
+    newRecord["serviceInstance"] = "nginx-instance"
+    newRecord["body"] = { text = { text = record.log } }
+    newRecord["tags"] = { data = {{ key = "LOG_KIND", value = "NGINX_ACCESS_LOG"}}}
+    return 1, timestamp, newRecord
+end
+
+function rewrite_error_log(tag, timestamp, record)
+    local newRecord = {}
+    newRecord["layer"] = "NGINX"
+    newRecord["service"] = "nginx::nginx"
+    newRecord["serviceInstance"] = "nginx-instance"
+    newRecord["body"] = { text = { text = record.log } }
+    newRecord["tags"] = { data = {{ key = "LOG_KIND", value = "NGINX_ERROR_LOG" }}}
+    return 1, timestamp, newRecord
+end

deploy/platform/docker/config/nginx/fluent-bit.conf

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[SERVICE]
+    Flush          5
+    Daemon         Off
+    Log_Level      warn
+[INPUT]
+    Name           tail
+    Tag            access
+    Path           /var/log/nginx/access.log
+[INPUT]
+    Name           tail
+    Tag            error
+    Path           /var/log/nginx/error.log
+[FILTER]
+    Name           lua
+    Match          access
+    Script         fluent-bit-script.lua
+    Call           rewrite_access_log
+[FILTER]
+    Name           lua
+    Match          error
+    Script         fluent-bit-script.lua
+    Call           rewrite_error_log
+[OUTPUT]
+    Name            stdout
+    Match           *
+    Format          json
+[OUTPUT]
+    Name            http
+    Match           *
+    Host            oap
+    Port            12800
+    URI             /v3/logs
+    Format          json

deploy/platform/docker/config/nginx/nginx.conf

@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+worker_processes  1;
+daemon off;
+error_log  /var/log/nginx/error.log notice;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    lua_shared_dict prometheus_metrics 10M;
+    # lua_package_path "/path/to/nginx-lua-prometheus/?.lua;;";
+
+    init_worker_by_lua_block {
+      prometheus = require("prometheus").init("prometheus_metrics")
+
+      metric_bytes = prometheus:counter(
+        "nginx_http_size_bytes", "Total size of HTTP", {"type", "route"})
+      metric_requests = prometheus:counter(
+        "nginx_http_requests_total", "Number of HTTP requests", {"status", "route"})
+      metric_latency = prometheus:histogram(
+        "nginx_http_latency", "HTTP request latency", {"route"})
+      metric_connections = prometheus:gauge(
+        "nginx_http_connections", "Number of HTTP connections", {"state"})
+    }
+
+    server {
+        listen 8080;
+
+        location /test {
+          default_type application/json;
+          return 200  '{"code": 200, "message": "success"}';
+
+          log_by_lua_block {
+            metric_bytes:inc(tonumber(ngx.var.request_length), {"request", "/test/**"})
+            metric_bytes:inc(tonumber(ngx.var.bytes_send), {"response", "/test/**"})
+            metric_requests:inc(1, {ngx.var.status, "/test/**"})
+            metric_latency:observe(tonumber(ngx.var.request_time), {"/test/**"})
+          }
+        }
+
+        location /test_404 {
+          default_type application/json;
+          return 404  '{"code": 404, "message": "404 NOT Found"}';
+
+          log_by_lua_block {
+            metric_bytes:inc(tonumber(ngx.var.request_length), {"request", "/test_404/**"})
+            metric_bytes:inc(tonumber(ngx.var.bytes_send), {"response", "/test_404/**"})
+            metric_requests:inc(1, {ngx.var.status, "/test_404/**"})
+            metric_latency:observe(tonumber(ngx.var.request_time), {"/test_404/**"})
+          }
+        }
+
+        location /test_500 {
+          default_type application/json;
+          return 500  '{"code": 500, "message": "500 Internal Server Error"}';
+
+          log_by_lua_block {
+            metric_bytes:inc(tonumber(ngx.var.request_length), {"request", "/test_500/**"})
+            metric_bytes:inc(tonumber(ngx.var.bytes_send), {"response", "/test_500/**"})
+            metric_requests:inc(1, {ngx.var.status, "/test_500/**"})
+            metric_latency:observe(tonumber(ngx.var.request_time), {"/test_500/**"})
+          }
+        }
+    }
+
+    server {
+      listen 9145;
+      location /metrics {
+        content_by_lua_block {
+          metric_connections:set(ngx.var.connections_reading, {"reading"})
+          metric_connections:set(ngx.var.connections_waiting, {"waiting"})
+          metric_connections:set(ngx.var.connections_writing, {"writing"})
+          prometheus:collect()
+        }
+      }
+    }
+}

deploy/platform/docker/config/nginx/otel-collector-config.yaml

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+receivers:
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: 'nginx-monitoring'
+          scrape_interval: 5s
+          metrics_path: "/metrics"
+          static_configs:
+            - targets: ['nginx:9145']
+              labels:
+                service: nginx
+                service_instance_id: nginx-instance
+processors:
+  batch:
+
+exporters:
+  otlp:
+    endpoint: oap:11800
+    tls:
+      insecure: true
+service:
+  pipelines:
+    metrics:
+      receivers:
+        - prometheus
+      processors:
+        - batch
+      exporters:
+        - otlp

deploy/platform/docker/docker-compose.nginx-monitor.yaml

@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '2.1'
+
+services:
+  nginx:
+    build:
+      context: ./config/nginx
+      dockerfile: Dockerfile.nginx
+    volumes:
+      - ./config/nginx/nginx.conf:/var/nginx/conf.d/nginx.conf
+      - /tmp/skywalking-logs/nginx:/var/log/nginx
+    entrypoint: ['bash', '-c', 'openresty -c /var/nginx/conf.d/nginx.conf']
+    depends_on:
+      oap:
+        condition: service_healthy
+    ports:
+      - 8080
+    expose:
+      - 9145
+    networks:
+      - sw
+
+  nginx-caller:
+    image: openresty/openresty:1.17.8.2-5-alpine-fat
+    depends_on:
+      - otel-collector
+    networks:
+      - sw
+    entrypoint: sh
+    command:
+      - -c
+      - "while true; do curl -o /dev/null -s http://nginx:8080/test; curl -o /dev/null -s http://nginx:8080/test_404; curl -o /dev/null -s http://nginx:8080/test_500; sleep 1s; done"
+
+  fluent-bit:
+    image: fluent/fluent-bit:1.9
+    volumes:
+      - /tmp/skywalking-logs/nginx:/var/log/nginx
+      - ./config/nginx/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
+      - ./config/nginx/fluent-bit-script.lua:/fluent-bit/etc/fluent-bit-script.lua
+    depends_on:
+      - oap
+      - nginx
+    networks:
+      - sw
+
+  otel-collector:
+    image: ${OTEL_COLLECTOR_IMAGE}:${OTEL_COLLECTOR_IMAGE_TAG}
+    command: [ "--config=/etc/otel-collector-config.yaml" ]
+    volumes:
+      - ./config/nginx/otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    depends_on:
+      - oap
+      - nginx
+    expose:
+      - 55678
+    networks:
+      sw:
+
+networks:
+  sw:

deploy/platform/kubernetes/Makefile

@@ -126,6 +126,11 @@ feature-postgresql-monitor:
 	$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.postgresqlMonitor.enabled=true)
 	$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set opentelemetry.enabled=true)
 
+.PHONY: feature-nginx-monitor
+feature-nginx-monitor:
+	$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.nginxMonitor.enabled=true)
+	$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set opentelemetry.enabled=true)
+
 .PHONY: feature-apisix-monitor
 feature-apisix-monitor:
 	$(eval HELM_OPTIONS := $(HELM_OPTIONS) --set features.apisixMonitor.enabled=true)

deploy/platform/kubernetes/Makefile.in

@@ -23,4 +23,4 @@ SAMPLE_SERVICES_NAMESPACE ?= sample-services
 RELEASE ?= demo
 AGENTLESS ?= false
 
-FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm-monitor,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql-monitor,postgresql-monitor,apisix-monitor,grafana,elasticsearch-monitor,rabbitmq-monitor,mongodb-monitor
+FEATURE_FLAGS ?= java-agent-injector,cluster,elasticsearch,kubernetes-monitor,so11y,vm-monitor,als,event,istiod-monitor,satellite,rover,trace-profiling,mysql-monitor,postgresql-monitor,nginx-monitor,apisix-monitor,grafana,elasticsearch-monitor,rabbitmq-monitor,mongodb-monitor

deploy/platform/kubernetes/templates/feature-nginx-monitor/opentelemetry-config.yaml

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+{{- define "opentelemetry-config-nginx" }}
+{{- if .Values.features.nginxMonitor.enabled }}
+
+- job_name: 'nginx-monitoring'
+  scrape_interval: 5s
+  metrics_path: "/metrics"
+  kubernetes_sd_configs:
+    - role: pod
+  relabel_configs:
+    - source_labels: [ __meta_kubernetes_pod_container_name, __meta_kubernetes_pod_container_port_name ]
+      action: keep
+      regex: nginx;metrics
+    - source_labels: [ __meta_kubernetes_pod_name ]
+      target_label: service_instance_id
+      regex: (.+)
+      replacement: $$1
+    - target_label: service
+      replacement: nginx
+
+{{- end }}
+{{- end }}