Skip to content

Commit

Permalink
Bump sdk to last version and Integration tests added (#129)
Browse files Browse the repository at this point in the history
- integration tests covering bootstrap and zookeeper

- Add new SDK 3.6.8 and get the JMX Host and Port when a query fails
  • Loading branch information
alvarocabanas authored Jun 2, 2021
1 parent 460eb62 commit 45b6885
Show file tree
Hide file tree
Showing 16 changed files with 20,432 additions and 7 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).

## 2.16 (2021-06-02)
### Changed
- Upgraded github.com/newrelic/infra-integrations-sdk to v3.6.7
- Added integration tests
- JMX logging shows host and port

## 2.15.1 (2021-05-26)
### Changed
- Add debug broker logs from Zookeeper
Expand Down
9 changes: 9 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,15 @@ test:
@echo "=== $(INTEGRATION) === [ test ]: running unit tests..."
@go test -race ./... -count=1

integration-test:
@echo "=== $(INTEGRATION) === [ test ]: running integration tests..."
@if [ "$(NRJMX_VERSION)" = "" ]; then \
echo "Error: missing required env-var: NRJMX_VERSION\n" ;\
exit 1 ;\
fi
@docker-compose -f tests/integration/docker-compose.yml up -d --build
@go test -v -tags=integration ./tests/integration/. -count=1 ; (ret=$$?; docker-compose -f tests/integration/docker-compose.yml down && exit $$ret)

# Include thematic Makefiles
include $(CURDIR)/build/ci.mk
include $(CURDIR)/build/release.mk
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@ require (
github.com/eapache/go-resiliency v1.2.1-0.20190716143035-b98ce2825f72 // indirect
github.com/eapache/queue v1.1.1-0.20180227141424-093482f3f8ce // indirect
github.com/golangci/golangci-lint v1.40.0
github.com/newrelic/infra-integrations-sdk v3.6.6+incompatible
github.com/newrelic/infra-integrations-sdk v3.6.8+incompatible
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da
github.com/stretchr/objx v0.3.0 // indirect
github.com/stretchr/testify v1.7.0
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
github.com/xdg/stringprep v1.0.1-0.20180714160509-73f8eece6fdc // indirect
github.com/xeipuuv/gojsonschema v1.2.0
)
10 changes: 8 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -482,8 +482,8 @@ github.com/nakabonne/nestif v0.3.0 h1:+yOViDGhg8ygGrmII72nV9B/zGxY188TYpfolntsaP
github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c=
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 h1:4kuARK6Y6FxaNu/BnU2OAaLF86eTVhP2hjTB6iMvItA=
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8=
github.com/newrelic/infra-integrations-sdk v3.6.6+incompatible h1:ThGyTlLKvSM0hiTW1dj+yITUH4U1o4fHQ4vwV0WLp6c=
github.com/newrelic/infra-integrations-sdk v3.6.6+incompatible/go.mod h1:tMUHRMq6mJS0YyBnbWrTXAnREnQqC1AGO6Lu45u5xAM=
github.com/newrelic/infra-integrations-sdk v3.6.8+incompatible h1:EBFBmBxNXpsTbDb5qpMC9QahgQrBWc+El+1JqwQVk24=
github.com/newrelic/infra-integrations-sdk v3.6.8+incompatible/go.mod h1:tMUHRMq6mJS0YyBnbWrTXAnREnQqC1AGO6Lu45u5xAM=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nishanths/exhaustive v0.1.0 h1:kVlMw8h2LHPMGUVqUj6230oQjjTMFjwcZrnkhXzFfl8=
Expand Down Expand Up @@ -678,6 +678,12 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xdg/stringprep v1.0.1-0.20180714160509-73f8eece6fdc h1:vIp1tjhVogU0yBy7w96P027ewvNPeH6gzuNcoc+NReU=
github.com/xdg/stringprep v1.0.1-0.20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yeya24/promlinter v0.1.0 h1:goWULN0jH5Yajmu/K+v1xCqIREeB+48OiJ2uu2ssc7U=
Expand Down
11 changes: 10 additions & 1 deletion src/broker/broker_collection_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"testing"
"time"

"github.com/newrelic/nri-kafka/src/metrics"

"github.com/Shopify/sarama"
"github.com/newrelic/infra-integrations-sdk/data/attribute"
"github.com/newrelic/infra-integrations-sdk/data/inventory"
Expand Down Expand Up @@ -186,7 +188,12 @@ func TestCollectBrokerTopicMetrics(t *testing.T) {
}

i, _ := integration.New("test", "1.0.0")
e, _ := i.Entity("testEntity", "testNamespace", integration.IDAttribute{Key: "clusterName", Value: ""})
e, _ := i.Entity(
"kafkabroker:9090",
"ka-broker",
integration.IDAttribute{Key: "clusterName", Value: ""},
integration.IDAttribute{Key: "brokerID", Value: "0"},
)

mockBroker := &mocks.SaramaBroker{}
mockBroker.On("Addr").Return("kafkabroker:9090")
Expand Down Expand Up @@ -214,5 +221,7 @@ func TestCollectBrokerTopicMetrics(t *testing.T) {

out := collectBrokerTopicMetrics(testBroker, []string{"topic"}, i)

metrics.CollectMetricDefinitions(sample, metrics.BrokerTopicMetricDefs, metrics.ApplyTopicName("topic"))

assert.Equal(t, expected, out)
}
6 changes: 3 additions & 3 deletions src/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func CollectBrokerRequestMetrics(sample *metric.Set, metricSets []*JMXMetricSet)
results, err := jmxwrapper.JMXQuery(beanName, args.GlobalArgs.Timeout)
// If we fail we don't want a total failure as other metrics can be collected even if a single failure/timout occurs
if err != nil && err == jmx.ErrConnection {
log.Error("Connection error: %s", err)
log.Error("Connection error for %s:%s : %s", jmx.HostName(), jmx.Port(), err)
os.Exit(1)
} else if err != nil {
log.Error("Unable to execute JMX query for MBean '%s': %s", beanName, err)
Expand Down Expand Up @@ -134,7 +134,7 @@ func CollectMetricDefinitions(sample *metric.Set, metricSets []*JMXMetricSet, be
results, err := jmxwrapper.JMXQuery(beanName, args.GlobalArgs.Timeout)
// If we fail we don't want a total failure as other metrics can be collected even if a single failure/timout occurs
if err != nil && err == jmx.ErrConnection {
log.Error("Connection error: %s", err)
log.Error("Connection error for %s:%s : %s", jmx.HostName(), jmx.Port(), err)
os.Exit(1)
} else if err != nil {
log.Error("Unable to execute JMX query for MBean '%s': %s", beanName, err.Error())
Expand Down Expand Up @@ -208,7 +208,7 @@ func getTopicListFromJMX(producer string) ([]string, error) {
func getAllTopicsFromJMX(producer string) ([]string, error) {
result, err := jmxwrapper.JMXQuery(fmt.Sprintf("kafka.producer:type=producer-topic-metrics,client-id=%s,topic=*", producer), args.GlobalArgs.Timeout)
if err != nil && err == jmx.ErrConnection {
log.Error("Connection error: %s", err)
log.Error("Connection error for %s:%s : %s", jmx.HostName(), jmx.Port(), err)
os.Exit(1)
} else if err != nil {
return nil, err
Expand Down
17 changes: 17 additions & 0 deletions tests/integration/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
FROM golang:1.16 as builder
ARG CGO_ENABLED=0
ARG NRJMX_VERSION
WORKDIR /go/src/github.com/newrelic/nri-kafka
COPY . .
RUN make clean compile
WORKDIR /
ADD http://download.newrelic.com/infrastructure_agent/binaries/linux/noarch/nrjmx_linux_${NRJMX_VERSION}_noarch.tar.gz ./
RUN tar -xzf nrjmx_linux_${NRJMX_VERSION}_noarch.tar.gz

FROM alpine:latest
RUN apk --update add openjdk8-jre
COPY --from=builder /usr/bin/nrjmx /usr/bin/nrjmx
COPY --from=builder /usr/bin/nrjmx.jar /usr/bin/nrjmx.jar
COPY --from=builder /usr/bin/jmxterm.jar /usr/bin/jmxterm.jar
COPY --from=builder /go/src/github.com/newrelic/nri-kafka/bin /
CMD ["sleep", "1h"]
81 changes: 81 additions & 0 deletions tests/integration/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
version: '2'
services:
zookeeper:
image: wurstmeister/zookeeper
ports:
- "2181:2181"
networks:
- kfk

kafka1:
image: wurstmeister/kafka
ports:
- "19092:9092"
- "1099"
environment:
KAFKA_ADVERTISED_HOST_NAME: kafka1
KAFKA_ADVERTISED_PORT: 9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_LOG_DIRS: /kafka/logs
KAFKA_BROKER_ID: 500
KAFKA_offsets_topic_replication_factor: 3
KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka1 -Dcom.sun.management.jmxremote.rmi.port=1099"
JMX_PORT: 1099
volumes:
- /var/run/docker.sock:/var/run/docker.sock
networks:
- kfk

kafka2:
image: wurstmeister/kafka
ports:
- "9092"
- "1099"
environment:
KAFKA_ADVERTISED_HOST_NAME: kafka2
KAFKA_ADVERTISED_PORT: 9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_LOG_DIRS: /kafka/logs
KAFKA_BROKER_ID: 501
KAFKA_offsets_topic_replication_factor: 3
KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka2 -Dcom.sun.management.jmxremote.rmi.port=1099"
JMX_PORT: 1099
volumes:
- /var/run/docker.sock:/var/run/docker.sock
networks:
- kfk

kafka3:
image: wurstmeister/kafka
ports:
- "9092"
- "1099"
environment:
KAFKA_ADVERTISED_HOST_NAME: kafka3
KAFKA_ADVERTISED_PORT: 9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_LOG_DIRS: /kafka/logs
KAFKA_BROKER_ID: 502
KAFKA_offsets_topic_replication_factor: 3
KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka3 -Dcom.sun.management.jmxremote.rmi.port=1099"
JMX_PORT: 1099
volumes:
- /var/run/docker.sock:/var/run/docker.sock
networks:
- kfk

nri-kafka:
container_name: integration_nri_kafka_1
build:
context: ../../
dockerfile: tests/integration/Dockerfile
args:
- NRJMX_VERSION=${NRJMX_VERSION}
dns:
- 8.8.8.8
networks:
- kfk

networks:
kfk:
driver: bridge
36 changes: 36 additions & 0 deletions tests/integration/helpers/helpers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package helpers

import (
"bytes"
"os/exec"
"strings"

"github.com/newrelic/infra-integrations-sdk/log"
)

const cmdMinLength = 3

// ExecInContainer executes the given command inside the specified container. It returns three values:
// 1st - Standard Output
// 2nd - Standard Error
// 3rd - Runtime error, if any
func ExecInContainer(container string, command []string) (string, string, error) {
cmdLine := make([]string, 0, cmdMinLength+len(command))
cmdLine = append(cmdLine, "exec", "-i")
cmdLine = append(cmdLine, container)
cmdLine = append(cmdLine, command...)

log.Debug("executing: docker %s", strings.Join(cmdLine, " "))

cmd := exec.Command("docker", cmdLine...)

var outbuf, errbuf bytes.Buffer
cmd.Stdout = &outbuf
cmd.Stderr = &errbuf

err := cmd.Run()
stdout := outbuf.String()
stderr := errbuf.String()

return stdout, stderr, err
}
Loading

0 comments on commit 45b6885

Please sign in to comment.