Skip to content

Commit

Permalink
feat: optionally load extra CA certs for HTTP statuscheck
Browse files Browse the repository at this point in the history
  • Loading branch information
grahambrereton-form3 authored and miketonks-form3 committed Dec 15, 2023
1 parent 374536a commit 271f5ce
Show file tree
Hide file tree
Showing 11 changed files with 196 additions and 15 deletions.
2 changes: 2 additions & 0 deletions controllers/fx.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/chaos-mesh/chaos-mesh/controllers/podnetworkchaos"
"github.com/chaos-mesh/chaos-mesh/controllers/schedule"
"github.com/chaos-mesh/chaos-mesh/controllers/statuscheck"
"github.com/chaos-mesh/chaos-mesh/controllers/utils/catrust"
"github.com/chaos-mesh/chaos-mesh/controllers/utils/chaosdaemon"
"github.com/chaos-mesh/chaos-mesh/controllers/utils/recorder"
wfcontrollers "github.com/chaos-mesh/chaos-mesh/pkg/workflow/controllers"
Expand All @@ -39,6 +40,7 @@ var Module = fx.Options(
recorder.NewRecorderBuilder,
common.AllSteps,
clusterregistry.New,
catrust.New,
),
fx.Invoke(common.Bootstrap),
fx.Invoke(podhttpchaos.Bootstrap),
Expand Down
3 changes: 2 additions & 1 deletion controllers/statuscheck/common_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package statuscheck

import (
"crypto/x509"
"time"

"github.com/go-logr/logr"
Expand All @@ -40,7 +41,7 @@ func (e *fakeHTTPExecutor) Type() string {
return "Fake-HTTP"
}

func newFakeExecutor(logger logr.Logger, statusCheck v1alpha1.StatusCheck) (Executor, error) {
func newFakeExecutor(logger logr.Logger, _ *x509.CertPool, statusCheck v1alpha1.StatusCheck) (Executor, error) {
var executor Executor
switch statusCheck.Spec.Type {
case v1alpha1.TypeHTTP:
Expand Down
29 changes: 24 additions & 5 deletions controllers/statuscheck/fx.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,43 @@ package statuscheck

import (
"github.com/go-logr/logr"
"github.com/pkg/errors"
"go.uber.org/fx"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
"github.com/chaos-mesh/chaos-mesh/controllers/config"
"github.com/chaos-mesh/chaos-mesh/controllers/utils/builder"
"github.com/chaos-mesh/chaos-mesh/controllers/utils/catrust"
"github.com/chaos-mesh/chaos-mesh/controllers/utils/recorder"
)

func Bootstrap(mgr ctrl.Manager, client client.Client, logger logr.Logger, recorderBuilder *recorder.RecorderBuilder) error {
type Params struct {
fx.In

Mgr ctrl.Manager
CertLoader *catrust.CACertLoader
KubeClient client.Client
Logger logr.Logger
RecorderBuilder *recorder.RecorderBuilder
}

func Bootstrap(params Params) error {
if !config.ShouldSpawnController("statuscheck") {
return nil
}
eventRecorder := recorderBuilder.Build("statuscheck")
manager := NewManager(logger.WithName("statuscheck-manager"), eventRecorder, newExecutor)

return builder.Default(mgr).
certPool, err := params.CertLoader.Load()
if err != nil {
return errors.Wrap(err, "loading CA certs")
}

eventRecorder := params.RecorderBuilder.Build("statuscheck")
manager := NewManager(params.Logger.WithName("statuscheck-manager"), eventRecorder, certPool, newExecutor)

return builder.Default(params.Mgr).
For(&v1alpha1.StatusCheck{}).
Named("statuscheck").
Complete(NewReconciler(logger.WithName("statuscheck-reconciler"), client, eventRecorder, manager))
Complete(NewReconciler(params.Logger.WithName("statuscheck-reconciler"), params.KubeClient, eventRecorder, manager))
}
17 changes: 14 additions & 3 deletions controllers/statuscheck/http/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ package http

import (
"bytes"
"crypto/tls"
"crypto/x509"
"fmt"
"io"
"net/http"
Expand All @@ -31,14 +33,15 @@ import (
)

type httpExecutor struct {
logger logr.Logger
logger logr.Logger
certPool *x509.CertPool

timeoutSeconds int
httpStatusCheck v1alpha1.HTTPStatusCheck
}

func NewExecutor(logger logr.Logger, timeoutSeconds int, httpStatusCheck v1alpha1.HTTPStatusCheck) *httpExecutor {
return &httpExecutor{logger: logger, timeoutSeconds: timeoutSeconds, httpStatusCheck: httpStatusCheck}
func NewExecutor(logger logr.Logger, certPool *x509.CertPool, timeoutSeconds int, httpStatusCheck v1alpha1.HTTPStatusCheck) *httpExecutor {
return &httpExecutor{logger: logger, certPool: certPool, timeoutSeconds: timeoutSeconds, httpStatusCheck: httpStatusCheck}
}

type response struct {
Expand All @@ -55,6 +58,14 @@ func (e *httpExecutor) Do() (bool, string, error) {
Timeout: time.Duration(e.timeoutSeconds) * time.Second,
}

if e.certPool != nil {
client.Transport = &http.Transport{
TLSClientConfig: &tls.Config{
RootCAs: e.certPool,
},
}
}

httpStatusCheck := e.httpStatusCheck
return e.DoHTTPRequest(client,
httpStatusCheck.RequestUrl,
Expand Down
16 changes: 11 additions & 5 deletions controllers/statuscheck/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package statuscheck

import (
"crypto/x509"
"sync"

"github.com/go-logr/logr"
Expand Down Expand Up @@ -48,17 +49,19 @@ type manager struct {
workers workerCache
results resultCache
newExecutor newExecutorFunc
certPool *x509.CertPool
}

type newExecutorFunc func(logger logr.Logger, statusCheck v1alpha1.StatusCheck) (Executor, error)
type newExecutorFunc func(logger logr.Logger, certPool *x509.CertPool, statusCheck v1alpha1.StatusCheck) (Executor, error)

func NewManager(logger logr.Logger, eventRecorder recorder.ChaosRecorder, newExecutorFunc newExecutorFunc) Manager {
func NewManager(logger logr.Logger, eventRecorder recorder.ChaosRecorder, certPool *x509.CertPool, newExecutorFunc newExecutorFunc) Manager {
return &manager{
logger: logger,
eventRecorder: eventRecorder,
workers: workerCache{workers: sync.Map{}},
results: resultCache{results: make(map[types.NamespacedName]Result)},
newExecutor: newExecutorFunc,
certPool: certPool,
}
}

Expand All @@ -74,7 +77,7 @@ func (m *manager) Add(statusCheck v1alpha1.StatusCheck) error {
return errors.New("status check is completed")
}

executor, err := m.newExecutor(m.logger, statusCheck)
executor, err := m.newExecutor(m.logger, m.certPool, statusCheck)
if err != nil {
return errors.Wrap(err, "new executor")
}
Expand Down Expand Up @@ -192,7 +195,7 @@ func limitRecords(records []v1alpha1.StatusCheckRecord, limit uint) []v1alpha1.S
return records[length-int(limit):]
}

func newExecutor(logger logr.Logger, statusCheck v1alpha1.StatusCheck) (Executor, error) {
func newExecutor(logger logr.Logger, certPool *x509.CertPool, statusCheck v1alpha1.StatusCheck) (Executor, error) {
var executor Executor
switch statusCheck.Spec.Type {
case v1alpha1.TypeHTTP:
Expand All @@ -202,7 +205,10 @@ func newExecutor(logger logr.Logger, statusCheck v1alpha1.StatusCheck) (Executor
}
executor = http.NewExecutor(
logger.WithName("http-executor").WithValues("url", statusCheck.Spec.HTTPStatusCheck.RequestUrl),
statusCheck.Spec.TimeoutSeconds, *statusCheck.Spec.HTTPStatusCheck)
certPool,
statusCheck.Spec.TimeoutSeconds,
*statusCheck.Spec.HTTPStatusCheck,
)
default:
return nil, errors.Errorf("unsupported type '%s'", statusCheck.Spec.Type)
}
Expand Down
2 changes: 1 addition & 1 deletion controllers/statuscheck/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ func testBootstrap(mgr ctrl.Manager, client client.Client, logger logr.Logger, r
return nil
}
eventRecorder := recorderBuilder.Build("statuscheck")
manager := NewManager(logger.WithName("statuscheck-manager"), eventRecorder, newFakeExecutor)
manager := NewManager(logger.WithName("statuscheck-manager"), eventRecorder, nil, newFakeExecutor)

return builder.Default(mgr).
For(&v1alpha1.StatusCheck{}).
Expand Down
104 changes: 104 additions & 0 deletions controllers/utils/catrust/catrust.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
// Copyright 2023 Chaos Mesh Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package catrust

import (
"crypto/x509"
"os"
"path/filepath"

"github.com/pkg/errors"
)

func loadExtraCACerts(caCerts *x509.CertPool) error {
extraCAPath, ok := os.LookupEnv("EXTRA_CA_TRUST_PATH")
if !ok {
return nil
}

err := filepath.WalkDir(extraCAPath, func(path string, d os.DirEntry, err error) error {
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}

info, err := d.Info()
if err != nil {
return errors.Wrapf(err, "get info for path %q", path)
}

// follow symlinks - kubernetes secret volume mounts contain symlinks to the secret value files and
// a directory containing the raw data. Following symlinks allows us to process only links to regular
// files.
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
newPath, err := filepath.EvalSymlinks(path)
if err != nil {
return errors.Wrapf(err, "read symlink %q", path)
}

path = newPath
info, err = os.Stat(path)
if err != nil {
return errors.Wrapf(err, "cannot stat %q", path)
}
}

// filter directories, pipes, other irregular files
if !info.Mode().IsRegular() {
return nil
}

bytes, err := os.ReadFile(path)
if err != nil {
return errors.Wrapf(err, "read cert file %q", path)
}

ok := caCerts.AppendCertsFromPEM(bytes)
if !ok {
return errors.Errorf("parse PEM file %q", path)
}
return nil
})

if err != nil {
return errors.Wrap(err, "load extra CA trust certificates")
}

return nil
}

type CACertLoader struct{}

func (cac CACertLoader) Load() (*x509.CertPool, error) {
caCerts, _ := x509.SystemCertPool()

if caCerts == nil {
caCerts = x509.NewCertPool()
}

err := loadExtraCACerts(caCerts)
if err != nil {
return nil, errors.Wrap(err, "load extra CA certificates")
}

return caCerts, nil
}

func New() *CACertLoader {
return &CACertLoader{}
}
17 changes: 17 additions & 0 deletions helm/chaos-mesh/templates/controller-manager-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,8 @@ spec:
- name: BURST
value: "50"
{{- end }}
- name: EXTRA_CA_TRUST_PATH
value: /etc/extra-ca-trust
{{- if .Values.controllerManager.chaosdSecurityMode }}
- name: CHAOSD_CA_CERT
value: /etc/chaosd/cert/ca.crt
Expand Down Expand Up @@ -163,6 +165,11 @@ spec:
mountPath: /etc/chaos-daemon/cert
readOnly: true
{{- end }}
{{- range .Values.controllerManager.extraCATrust }}
- name: extra-ca-trust-certs-{{ .secret }}
mountPath: /etc/extra-ca-trust/{{ .secret }}
readOnly: true
{{- end }}
{{- if .Values.controllerManager.chaosdSecurityMode }}
- name: chaosd-client-cert
mountPath: /etc/chaosd/cert
Expand Down Expand Up @@ -210,6 +217,16 @@ spec:
secret:
secretName: {{ template "chaos-mesh.daemon-client.certs" . }}
{{- end }}
{{- range .Values.controllerManager.extraCATrust }}
- name: extra-ca-trust-certs-{{ .secret }}
secret:
secretName: {{ .secret }}
items:
{{- range .items }}
- key: {{ . }}
path: {{ . }}
{{- end }}
{{- end }}
{{- if .Values.controllerManager.chaosdSecurityMode }}
- name: chaosd-client-cert
secret:
Expand Down
17 changes: 17 additions & 0 deletions helm/chaos-mesh/values.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,23 @@
}
}
},
"extraCATrust": {
"type": "array",
"items": {
"type": "object",
"properties": {
"secret": {
"type": "string"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
},
"hostNetwork": {
"type": "boolean"
},
Expand Down
2 changes: 2 additions & 0 deletions helm/chaos-mesh/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ controllerManager:
# Image pull policy
imagePullPolicy: IfNotPresent

extraCATrust: []

# The keys within the "env" map are mounted as environment variables on the pod.
env:
# WEBHOOK_PORT is configured the port for chaos-controller-manager provides webhooks.
Expand Down
2 changes: 2 additions & 0 deletions install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1885,6 +1885,8 @@ spec:
value: "false"
- name: CHAOSD_SECURITY_MODE
value: "false"
- name: EXTRA_CA_TRUST_PATH
value: /etc/extra-ca-trust
- name: POD_FAILURE_PAUSE_IMAGE
value: gcr.io/google-containers/pause:latest
- name: ENABLE_LEADER_ELECTION
Expand Down

0 comments on commit 271f5ce

Please sign in to comment.