Skip to content

Commit

Permalink
Merge pull request #96 from justwatchcom/feat/add_cloudsql_client
Browse files Browse the repository at this point in the history
Add CloudSQL support
  • Loading branch information
dewey authored Sep 21, 2023
2 parents 8d391a3 + 8cd377c commit 1b8fd18
Show file tree
Hide file tree
Showing 926 changed files with 258,917 additions and 2,221 deletions.
75 changes: 75 additions & 0 deletions cloudsql.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package main

import (
"fmt"
"net/url"
"regexp"
"strings"
)

const (
CLOUDSQL_POSTGRES = "cloudsql-postgres"
CLOUDSQL_MYSQL = "cloudsql-mysql"
)

func isValidCloudSQLDriver(conn string) (bool, string) {
switch {
case strings.HasPrefix(conn, CLOUDSQL_POSTGRES):
return true, CLOUDSQL_POSTGRES
case strings.HasPrefix(conn, CLOUDSQL_MYSQL):
return true, CLOUDSQL_MYSQL
default:
return false, ""
}
}

var cloudSQLHostRegex = regexp.MustCompile(`(.*@)(.*?)(/.*)`)

type CloudSQLUrl struct {
*url.URL
Project string
Region string
Instance string
}

func ParseCloudSQLUrl(u string) (*CloudSQLUrl, error) {
parts := cloudSQLHostRegex.FindStringSubmatch(u)
if len(parts) != 4 {
return nil, fmt.Errorf("did get invalid part count from regex expected 4, got %d. %v", len(parts), parts)
}

sanitizedUrl := fmt.Sprintf("%shost%s", parts[1], parts[3])
urlParsed, err := url.Parse(sanitizedUrl)
if err != nil {
return nil, fmt.Errorf("could not parse sanized url %q: %w", sanitizedUrl, err)
}

hostParts := strings.Split(parts[2], ":")
if len(hostParts) != 3 {
return nil, fmt.Errorf("could not parse cloudsql host. Expected 3 elements, but got %d: %v", len(hostParts), hostParts)
}
urlParsed.Host = parts[2]

cloudSQLUrl := &CloudSQLUrl{
URL: urlParsed,
Project: hostParts[0],
Region: hostParts[1],
Instance: hostParts[2],
}
return cloudSQLUrl, nil
}

func (u *CloudSQLUrl) GetConnectionURL(driver, instance, database string) (string, error) {
pass, isSet := u.User.Password()
if !isSet {
return "", fmt.Errorf("invalid url: cannot find password")
}

switch driver {
case CLOUDSQL_POSTGRES:
return fmt.Sprintf("host=%s user=%s password=%s dbname=%s sslmode=disable", instance, u.User.Username(), pass, database), nil
case CLOUDSQL_MYSQL:
return fmt.Sprintf("%s@cloudsql-mysql(%s)/%s", u.User, instance, database), nil
}
return "", fmt.Errorf("driver not supported: %s", driver)
}
13 changes: 11 additions & 2 deletions config.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,19 @@ func Read(path string) (File, error) {
return f, nil
}

// CloudSQLConfig is required for configuring the cloudsql connections.
//
// If it is not set, no CloudSQL connection will be created
type CloudSQLConfig struct {
// If KeyFile is set, then we load the IAM key from there
KeyFile string `yaml:"key_file"`
}

// File is a collection of jobs
type File struct {
Jobs []*Job `yaml:"jobs"`
Queries map[string]string `yaml:"queries"`
Jobs []*Job `yaml:"jobs"`
Queries map[string]string `yaml:"queries"`
CloudSQLConfig *CloudSQLConfig `yaml:"cloudsql_config"`
}

type cronConfig struct {
Expand Down
44 changes: 41 additions & 3 deletions exporter.go
Original file line number Diff line number Diff line change
@@ -1,17 +1,25 @@
package main

import (
"cloud.google.com/go/cloudsqlconn"
"cloud.google.com/go/cloudsqlconn/mysql/mysql"
"cloud.google.com/go/cloudsqlconn/postgres/pgxv4"
"context"
"fmt"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/prometheus/client_golang/prometheus"
"github.com/robfig/cron/v3"
"google.golang.org/api/option"
sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

// Exporter collects SQL metrics. It implements prometheus.Collector.
type Exporter struct {
jobs []*Job
logger log.Logger
cronScheduler *cron.Cron
jobs []*Job
logger log.Logger
cronScheduler *cron.Cron
sqladminService *sqladmin.Service
}

// NewExporter returns a new SQL Exporter for the provided config.
Expand All @@ -32,6 +40,36 @@ func NewExporter(logger log.Logger, configFile string) (*Exporter, error) {
cronScheduler: cron.New(),
}

if cfg.CloudSQLConfig != nil {
if cfg.CloudSQLConfig.KeyFile == "" {
return nil, fmt.Errorf("as cloudsql_config is not empty, then cloudsql_config.key_file must be set")
}

// We currently only support keyfile. Additional authentication options would be via automatic IAM
// with cloudsqlconn.WithIAMAuthN()
cloudsqlconnection := cloudsqlconn.WithCredentialsFile(cfg.CloudSQLConfig.KeyFile)
sqladminService, err := sqladmin.NewService(context.Background(), option.WithAPIKey(cfg.CloudSQLConfig.KeyFile))
if err != nil {
return nil, fmt.Errorf("could not create new cloud sqladmin service: %w", err)
}
exp.sqladminService = sqladminService

//
// Register all possible cloudsql drivers

// drop cleanup as we don't really know when to end this
_, err = pgxv4.RegisterDriver(CLOUDSQL_POSTGRES, cloudsqlconnection)
if err != nil {
return nil, fmt.Errorf("could not register cloudsql-postgres driver: %w", err)
}

// drop cleanup as we don't really know when to end this
_, err = mysql.RegisterDriver(CLOUDSQL_MYSQL, cloudsqlconnection)
if err != nil {
return nil, fmt.Errorf("could not register cloudsql-mysql driver: %w", err)
}
}

// dispatch all jobs
for _, job := range cfg.Jobs {
if job == nil {
Expand Down
44 changes: 34 additions & 10 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/justwatchcom/sql_exporter
go 1.20

require (
cloud.google.com/go/cloudsqlconn v1.4.0
github.com/ClickHouse/clickhouse-go/v2 v2.10.1
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/denisenkom/go-mssqldb v0.12.3
Expand All @@ -20,10 +21,12 @@ require (
)

require (
cloud.google.com/go/compute v1.20.1 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
github.com/ClickHouse/ch-go v0.52.1 // indirect
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
Expand Down Expand Up @@ -55,21 +58,35 @@ require (
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.6.1 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/goccy/go-json v0.10.0 // indirect
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v23.1.21+incompatible // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/googleapis/gax-go/v2 v2.11.0 // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgconn v1.14.0 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.2 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgtype v1.14.0 // indirect
github.com/jackc/pgx/v4 v4.18.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.15.15 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/microsoft/go-mssqldb v1.3.0 // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/mtibben/percent v0.2.1 // indirect
Expand All @@ -83,19 +100,26 @@ require (
github.com/shopspring/decimal v1.3.1 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.13.0 // indirect
go.opentelemetry.io/otel/trace v1.13.0 // indirect
golang.org/x/crypto v0.10.0 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.11.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sys v0.9.0 // indirect
golang.org/x/term v0.9.0 // indirect
golang.org/x/text v0.10.0 // indirect
golang.org/x/net v0.12.0 // indirect
golang.org/x/oauth2 v0.10.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.6.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/protobuf v1.30.0 // indirect
google.golang.org/api v0.130.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect
google.golang.org/grpc v1.56.1 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
)
Loading

0 comments on commit 1b8fd18

Please sign in to comment.