Merge pull request #128 from gocardless/dyson-etcd-certificate-expiry

Add etcd store certificate expiry metric

Dockerfile

@@ -2,7 +2,7 @@
 # build
 ################################################################################
 
-FROM golang:1.14.1 AS build
+FROM golang:1.14.2 AS build
 COPY . /go/src/github.com/gocardless/stolon-pgbouncer
 WORKDIR /go/src/github.com/gocardless/stolon-pgbouncer
 
@@ -19,7 +19,7 @@ RUN set -x \
 # release
 ################################################################################
 
-FROM gocardless/stolon-pgbouncer-base:2019100101 AS release
+FROM gocardless/stolon-pgbouncer-base:2020050701 AS release
 COPY --from=build /go/src/github.com/gocardless/stolon-pgbouncer/stolon-pgbouncer /usr/local/bin/stolon-pgbouncer
 USER postgres
 ENTRYPOINT ["/usr/local/bin/stolon-pgbouncer"]

Makefile

@@ -3,9 +3,9 @@ PROJECT=github.com/gocardless/stolon-pgbouncer
 VERSION=$(shell git rev-parse --short HEAD)-dev
 BUILD_COMMAND=go build -ldflags "-X main.Version=$(VERSION)"
 
-BASE_TAG=2019100101
-CIRCLECI_TAG=2019100101
-STOLON_DEVELOPMENT_TAG=2019100101
+BASE_TAG=2020051101
+CIRCLECI_TAG=2020051101
+STOLON_DEVELOPMENT_TAG=2020051101
 
 .PHONY: all darwin linux test clean test-acceptance docker-compose
 

VERSION

@@ -1 +1 @@
-2.0.1
+2.1.0

circle.yml

@@ -5,7 +5,7 @@ references:
   docker_build_image: &docker_build_image
     working_directory: /go/src/github.com/gocardless/stolon-pgbouncer
     docker:
-      - image: &image gocardless/stolon-pgbouncer-circleci:2019100101
+      - image: &image gocardless/stolon-pgbouncer-circleci:2020050701
   docker_postgres_build_image: &docker_postgres_build_image
     working_directory: /go/src/github.com/gocardless/stolon-pgbouncer
     docker:

cmd/stolon-pgbouncer/main.go

@@ -3,8 +3,12 @@ package main
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
+	"encoding/pem"
+	"errors"
 	"fmt"
+	"io/ioutil"
 	stdlog "log"
 	"net"
 	"net/http"
@@ -31,7 +35,7 @@ import (
 	"github.com/coreos/etcd/mvcc/mvccpb"
 	kitlog "github.com/go-kit/kit/log"
 	level "github.com/go-kit/kit/log/level"
-	"github.com/grpc-ecosystem/go-grpc-middleware"
+	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	"github.com/oklog/run"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -191,6 +195,12 @@ var (
 		},
 		[]string{"keeper"},
 	)
+	storeCertificateExpirySeconds = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Name: "stolon_store_certificate_expiry_seconds",
+			Help: "Time in unix epoch seconds at which the store certificate expires",
+		},
+	)
 )
 
 func init() {
@@ -201,6 +211,7 @@ func init() {
 	prometheus.MustRegister(storeLastUpdateSeconds)
 	prometheus.MustRegister(lastKeeperSeconds)
 	prometheus.MustRegister(lastReloadSeconds)
+	prometheus.MustRegister(storeCertificateExpirySeconds)
 }
 
 type exitError struct {
@@ -706,6 +717,15 @@ func mustTLS(opt *stolonOptions) *tls.Config {
 			kingpin.Fatalf("failed to load client certs: %s", err)
 		}
 
+		expiry, err := parseX509Expiry(opt.CertFile)
+		if err != nil {
+			storeCertificateExpirySeconds.Set(float64(-1))
+		} else {
+			storeCertificateExpirySeconds.Set(
+				float64(expiry.UnixNano()) / 1e9,
+			)
+		}
+
 		cfg.Certificates = []tls.Certificate{*cert}
 	}
 
@@ -723,6 +743,24 @@ func mustTLS(opt *stolonOptions) *tls.Config {
 	return cfg
 }
 
+func parseX509Expiry(certFile string) (time.Time, error) {
+	data, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return time.Now(), err
+	}
+
+	block, _ := pem.Decode(data)
+	if block == nil {
+		return time.Now(), errors.New("failed to decode PEM block from certificate")
+	}
+	cert, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		return time.Now(), err
+	}
+
+	return cert.NotAfter, nil
+}
+
 // setupSignalHandler is similar to the community provided functions, but follows a more
 // modern pattern using contexts. If the caller desires a channel that will be closed on
 // completion, they can simply use ctx.Done()

docker-compose.yml

@@ -21,7 +21,7 @@ services:
           - etcd-store
 
   sentinel:
-    image: &stolonDevelopmentImage gocardless/stolon-development:2019100101
+    image: &stolonDevelopmentImage gocardless/stolon-development:2020050701
     restart: on-failure
     depends_on:
       - etcd-store

docker/base/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:bionic-20190807
+FROM ubuntu:bionic-20200403
 RUN set -x \
       && apt-get update -y \
       && apt-get install -y curl gpg \

docker/circleci/Dockerfile

@@ -1,6 +1,6 @@
 # In addition to our base install of pgbouncer and postgresql-client, add CI
 # dependencies that we require during our builds.
-FROM gocardless/stolon-pgbouncer-base:2019100101
+FROM gocardless/stolon-pgbouncer-base:2020050701
 
 # General test utilities
 RUN set -x \
@@ -12,7 +12,7 @@ RUN set -x \
 ENV GOPATH=/go GOROOT=/usr/local/go PATH=$PATH:/usr/local/go/bin:/go/bin:/usr/sbin
 RUN set -x \
       && mkdir -p /usr/local/go /go \
-      && curl -L https://dl.google.com/go/go1.13.linux-amd64.tar.gz -o /tmp/go.tar.gz \
+      && curl -L https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz -o /tmp/go.tar.gz \
       && tar xfvz /tmp/go.tar.gz -C /usr/local/go --strip-components=1 \
       && go version \
       && go get -v -u github.com/onsi/ginkgo/ginkgo \

docker/observability/prometheus/rules.yml

@@ -183,3 +183,61 @@ groups:
             stolon-keeper manages Postgres configuration, along with Postgres
             reload and restarts. This alert is firing when a keeper is reporting
             a required restart that it's been unable to automatically apply.
+
+  - name: stolon-etcd.rules
+    rules:
+      - alert: StolonEtcdClientExpiryImminent
+        # We want to know if we imminently need to roll the client certificates
+        # used by stolon-pgbouncer to access etcd. This will page when a
+        # certificate will expire within 3 days.
+        expr: >
+          count by (cluster_name) (
+            stolon_cluster_identifier * ignoring(cluster_name, component) group_right(cluster_name) (
+              stolon_store_certificate_expiry_seconds - time()
+            ) < (60 * 60 * 24 * 3)
+          )
+        for: 5m
+        labels:
+          severity: page
+        annotations:
+          summary: Etcd client certificate will expire imminently
+          description: |
+
+            Our stolon infrastructure accesses etcd using client certs.
+            The validity of these certificates is ending imminently, and should be rolled.
+            If not done, postgres will become unavailable.
+
+      - alert: StolonEtcdClientExpiryUrgent
+        expr: >
+          count by (cluster_name) (
+            stolon_cluster_identifier * ignoring(cluster_name, component) group_right(cluster_name) (
+              stolon_store_certificate_expiry_seconds - time()
+            ) < (60 * 60 * 24 * 14)
+          )
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: Etcd client certificate will expire urgently
+          description: |
+
+            Our stolon infrastructure accesses etcd using client certs.
+            The validity of these certificates is ending soon, and should be rolled.
+
+      - alert: StolonEtcdClientExpiryWarning
+        expr: >
+          count by (cluster_name) (
+            stolon_cluster_identifier * ignoring(cluster_name, component) group_right(cluster_name) (
+              stolon_store_certificate_expiry_seconds - time()
+            ) < (60 * 60 * 24 * 28)
+          )
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: Etcd client certificate will expire soon
+          description: |
+
+            Our stolon infrastructure accesses etcd using client certs.
+            The validity of these certificates is ending soon, and should be rolled.
+

docker/stolon-development/Dockerfile

@@ -1,28 +1,29 @@
 # Temporarily use the GoCardless stolon fork to install the keeper. This enables
 # us to verify the metrics we're adding to the binaries.
-FROM golang:1.12 AS stolon-fork
+FROM golang:1.14.2 AS stolon-fork
 RUN set -x \
       && go get -d -v github.com/sorintlab/stolon/cmd \
       && cd "${GOPATH}/src/github.com/sorintlab/stolon" \
       && git remote add gocardless https://github.com/gocardless/stolon \
       && git fetch gocardless \
-      && git checkout 35b235b58abc64a3ace7486710b7817f0bf7d06f \
+      && git checkout b2e9e04434e0350356e682dc725bb1551b33784a \
       && ./build
 
 # GoCardless runs this fork for PgBouncer metrics. We'll likely change this in
 # future but include it for now so the dashboards in this repo can match what we
 # have deployed internally.
-FROM golang:1.12 AS pgbouncer-exporter-fork
+FROM golang:1.14.2 AS pgbouncer-exporter-fork
 RUN set -x \
       && go get -d -v github.com/gocardless/pgbouncer_exporter \
       && cd "${GOPATH}/src/github.com/gocardless/pgbouncer_exporter" \
-      && git checkout a4ec94990b18f76dfd872b2a9214e827c52220a5 \
+      && git checkout 7bbdf2573e6b6fe4bc6fe3981173ccd338763771 \
       && make
 
 # In addition to our base install of pgbouncer and postgresql-client, configure
 # all the dependencies we'll need across our docker-compose setup along with
 # convenience env vars to make stolon tooling function correctly.
-FROM gocardless/stolon-pgbouncer-base:2019100101
+FROM gocardless/stolon-pgbouncer-base:2020050701
+ENV DEBIAN_FRONTEND noninteractive
 
 RUN set -x \
       && apt-get update -y \

go.mod

@@ -1,6 +1,6 @@
 module github.com/gocardless/stolon-pgbouncer
 
-go 1.12
+go 1.14
 
 require (
 	github.com/alecthomas/kingpin v2.2.6+incompatible
@@ -15,15 +15,13 @@ require (
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/go-kit/kit v0.10.0
 	github.com/gogo/protobuf v1.2.1 // indirect
-	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
 	github.com/golang/protobuf v1.4.1
 	github.com/google/btree v1.0.0 // indirect
 	github.com/google/certificate-transparency-go v1.0.21 // indirect
 	github.com/google/uuid v1.1.1 // indirect
 	github.com/gorilla/websocket v1.4.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway v1.9.0 // indirect
 	github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect
 	github.com/jackc/pgx v3.6.2+incompatible
 	github.com/jonboulle/clockwork v0.1.0 // indirect
@@ -39,13 +37,6 @@ require (
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
 	go.etcd.io/bbolt v1.3.3 // indirect
-	go.uber.org/atomic v1.4.0 // indirect
-	go.uber.org/multierr v1.1.0 // indirect
-	go.uber.org/zap v1.10.0 // indirect
-	golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd // indirect
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
-	google.golang.org/genproto v0.0.0-20190327125643-d831d65fe17d // indirect
 	google.golang.org/grpc v1.26.0
-	gopkg.in/yaml.v2 v2.2.2 // indirect
 	sigs.k8s.io/yaml v1.1.0 // indirect
 )

go.sum

@@ -11,9 +11,11 @@ github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvr
 github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
@@ -98,6 +100,7 @@ github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgO
 github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
@@ -345,6 +348,7 @@ github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNG
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI=
 github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -489,6 +493,7 @@ golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8=
 golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -545,7 +550,9 @@ google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLY
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0 h1:cJv5/xdbk1NnMPR1VP9+HU6gupuG9MLBoH1r6RHZ2MY=
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=