From 35afd41b45d45faf5e37e48c8595bf2ff928694f Mon Sep 17 00:00:00 2001 From: Kumbirai Tanekha Date: Wed, 6 May 2020 18:21:36 +0100 Subject: [PATCH] POC: Auto generation of troubleshooting guide Creating the proxy service in memory makes it very easy to have a troubleshooting generation as code approach. This commit provides an example using the HTTP connector. A similar approac h would work for other connectors. It manages to capture client output and relevant Secretless logs all in one fell swoop. --- .../generic/troubleshooting/example_test.go | 228 ++++++++++++++++++ .../troubleshooting/http_client_request.go | 42 ++++ .../troubleshooting/http_proxy_service.go | 128 ++++++++++ .../troubleshooting/troubleshooting.md | 112 +++++++++ 4 files changed, 510 insertions(+) create mode 100644 test/connector/http/generic/troubleshooting/example_test.go create mode 100644 test/connector/http/generic/troubleshooting/http_client_request.go create mode 100644 test/connector/http/generic/troubleshooting/http_proxy_service.go create mode 100644 test/connector/http/generic/troubleshooting/troubleshooting.md diff --git a/test/connector/http/generic/troubleshooting/example_test.go b/test/connector/http/generic/troubleshooting/example_test.go new file mode 100644 index 000000000..cf35a848b --- /dev/null +++ b/test/connector/http/generic/troubleshooting/example_test.go @@ -0,0 +1,228 @@ +package troubleshooting + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "os/exec" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +type client func (url, proxy string) (string, error) + +func Code(text string) string { + return "```\n" + text + "\n```" +} + +func Indent(text, indent string) string { + return _Indent(Code(strings.TrimSpace(text)), indent) +} + +// indents a block of text with an indent string +func _Indent(text, indent string) string { + if text[len(text)-1:] == "\n" { + result := "" + for _, j := range strings.Split(text[:len(text)-1], "\n") { + result += indent + j + "\n" + } + return result + } + result := "" + for _, j := range strings.Split(strings.TrimRight(text, "\n"), "\n") { + result += indent + j + "\n" + } + return result[:len(result)-1] +} + +func httpCurl(url, proxy string) (string, error) { + args := []string{ + //"-k", + "-sS", + "-v", + "-X", "GET", + url, + } + + cmd := exec.Command( + "curl", + args..., + ) + + proxyVarName := "http_proxy" + if strings.HasPrefix(url, "https") { + proxyVarName = "https_proxy" + } + + cmd.Env = append(cmd.Env, proxyVarName+"="+proxy) + out, err := cmd.CombinedOutput() + + if err != nil { + return "", fmt.Errorf("%s", string(out)) + } + + return string(out), nil +} + +func httpGo(url, proxy string) (string, error) { + res, err := proxyGet( + url, + proxy, + ) + if err != nil { + return "", err + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + + return fmt.Sprintf(` +Status: +%s + +Body: +%s`, res.Status, string(body)), nil +} + +func TestHTTPS(t *testing.T) { + _, logs, err1 := troubleShooting(t, "https://httpbin.org/anything", httpCurl) + if !assert.Error(t, err1) { + return + } + + _, _, err2 := troubleShooting(t, "https://httpbin.org/anything", httpGo) + if !assert.Error(t, err2) { + return + } + + res := fmt.Sprintf(` +### Connecting to Secretless as an HTTPS proxy +#### Symptoms +- You see a CONNECT request in your Secretless logs that looks something like: + +%s + +- Sample client log output messages + + curl: +%s + + Go client: +%s + +#### Known Causes +This type of error occurs when the client attempts to use Secretless as an HTTPS proxy. +Secretless can only act as an HTTP proxy. +This error can happen for a few reasons: +1. An explicit attempt to use Secretless as an HTTPS proxy +1. Providing the client an HTTPS target when intending to proxy the connection through Secretless might result in the client attempting to use Secretless as an HTTPS proxy, as is the case with Go's standard library HTTP client. + +#### Resolution +- Ensure that target of your request is HTTP only e.g. http://httpbin.org. +- Secretless does not support HTTPS between the client and Secretless, it does support it between Secretless and the target. Do not use Secretless as an HTTPS proxy. +- To make connection between Secretless and the target an HTTPS connection you must set "forceSSL: true" on the Secretless service connector config. + +`, Indent(logs.String(), " "), Indent(err1.Error(), " "), Indent(err2.Error(), " ")) + + writeToTroubleshooting(res) +} + +func TestSelfSigned(t *testing.T) { + out1, logs, err := troubleShooting(t, "http://self-signed.badssl.com", httpCurl) + if !assert.NoError(t, err) { + return + } + + out2, _, err := troubleShooting(t, "http://wrong.host.badssl.com/", httpGo) + if !assert.NoError(t, err) { + return + } + + res := fmt.Sprintf(` +### HTTPS certificate verification failure when forceSSL is set +#### Symptoms +- You see an x509 certificate error in your Secretless logs that looks something like: + +%s + +- Sample client log output messages + + curl: +%s + + Go client: +%s + +#### Known Causes +This type of error occurs when the client attempts to connect to a target with a self-signed certificate, and there is some failure on verification. Secretless verifies all HTTPS connections to the target. + +There are several reasons why verification might fail including: +1. The signer of the target's certificate is not a trusted CA +1. The target's certificate is expired or is not yet valid +1. The target's certificate is not valid for the host + +#### Resolution + +This type of error can be broken into 2 categories. +1. The signer of the target's certificate is not a trusted CA +1. The rest + +For the rest (2), you must ensure that the target's certificate is valid for the target. + +For (1) you will need to ensure that Secretless is aware of the root certificate authority (CA) it should use to verify the server certificates when proxying requests. To do this, ensure that the environment variable **SECRETLESS_HTTP_CA_BUNDLE** is set. **SECRETLESS_HTTP_CA_BUNDLE** is a path to the bundle of CA certificates that are appended to the certificate pool that Secretless uses for server certificate verification of all HTTP service connectors. +`, Indent(logs.String(), " "), Indent(out1, " "), Indent(out2, " ")) + + + writeToTroubleshooting(res) +} + +func writeToTroubleshooting(out string) { + f, err := os.OpenFile("troubleshooting.md", + os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + panic(err) + } + defer f.Close() + if _, err := f.WriteString(out); err != nil { + panic(err) + } +} +func troubleShooting(t *testing.T, url string, client2 client) (string, bytes.Buffer, error) { + var buf bytes.Buffer + // Create in-process proxy service + proxyService, err := newInProcessProxyService( + []byte(` +credentialPatterns: + username: '[^:]+' +headers: + Authorization: ' Basic {{ printf "%s:%s" .username .password | base64 }}' +authenticateURLsMatching: + - ^http +forceSSL: true +`), + map[string][]byte{}, + &buf, + ) + if !assert.NoError(t, err) { + return "", buf, nil + } + + // Ensure the proxy service is stopped + defer proxyService.Stop() + // Start the proxyService service. Note + proxyService.Start() + + // Avoid all the unnecessary initial logs + buf.Reset(); + + proxyURL := "http://" + proxyService.host + ":" + proxyService.port + + // Make the client request to the proxy service + out, err := client2(url, proxyURL) + return out, buf, err +} diff --git a/test/connector/http/generic/troubleshooting/http_client_request.go b/test/connector/http/generic/troubleshooting/http_client_request.go new file mode 100644 index 000000000..7aa900ac4 --- /dev/null +++ b/test/connector/http/generic/troubleshooting/http_client_request.go @@ -0,0 +1,42 @@ +package troubleshooting + +import ( + "net" + "net/http" + "net/url" +) + +// ephemeralListenerOnPort creates a net.Listener (with a short deadline) on some given +// port. Note that passing in a port of "0" will result in a random port being used. +func ephemeralListenerOnPort(port string) (net.Listener, error) { + listener, err := net.Listen("tcp", "127.0.0.1:"+port) + if err != nil { + return nil, err + } + + // We generally don't want to wait forever for a connection to come in + //err = listener.(*net.TCPListener).SetDeadline(time.Now().Add(10 * time.Second)) + //if err != nil { + // return nil, err + //} + + return listener, nil +} + +// proxyGet is a convenience method that makes an HTTP GET request using a proxy +func proxyGet(endpoint, proxy string) (*http.Response, error) { + req, err := http.NewRequest( + "GET", + endpoint, + nil, + ) + if err != nil { + return nil, err + } + + transport := &http.Transport{Proxy: func(req *http.Request) (proxyURL *url.URL, err error) { + return url.Parse(proxy) + }} + client := &http.Client{Transport: transport} + return client.Do(req) +} diff --git a/test/connector/http/generic/troubleshooting/http_proxy_service.go b/test/connector/http/generic/troubleshooting/http_proxy_service.go new file mode 100644 index 000000000..3fa076458 --- /dev/null +++ b/test/connector/http/generic/troubleshooting/http_proxy_service.go @@ -0,0 +1,128 @@ +package troubleshooting + +import ( + "io" + "net" + + "github.com/cyberark/secretless-broker/internal/log" + httpInternal "github.com/cyberark/secretless-broker/internal/plugin/connectors/http" + "github.com/cyberark/secretless-broker/internal/plugin/connectors/http/generic" + v2 "github.com/cyberark/secretless-broker/pkg/secretless/config/v2" + "github.com/cyberark/secretless-broker/pkg/secretless/plugin/connector" +) + +// proxyService is a Secretless proxy service endpoint. The start logic of this service +// endpoint is abstracted so that the user can decide how it is implemented, common +// examples are in-process as a TCP proxy service or out-of-process as the Secretless +// binary. +type proxyService struct { + host string + port string + start func() // start runs the logic to start the proxy service + stop func() // stop runs the logic to stop the proxy service +} + +// Start concurrently runs the start logic for a proxy service +func (s *proxyService) Start() { + s.start() +} + +// Stop delegates the cleanup logic for a proxy service +func (s *proxyService) Stop() { + s.stop() +} + + +// cloneCredentials creates an independent clone of a credentials map. The resulting +// clone will not be affected by any mutations to the original, and vice-versa. The clone +// is useful for passing to a proxyService service, to avoid zeroization of the original. +func cloneCredentials(original map[string][]byte) map[string][]byte { + credsClone := make(map[string][]byte) + + for key, value := range original { + // Clone the value + valueClone := make([]byte, len(value)) + copy(valueClone, value) + + // Set the key, value pair on the credentials clone + credsClone[key] = valueClone + } + + return credsClone +} + +// newInProcessProxyService creates an HTTP proxy service. +// 1. Create the net.Listener +// 2. Create the HTTP Service Connector +// 3. Create a TCP proxy service that uses (1) and (2) +func newInProcessProxyService( + config []byte, + credentials map[string][]byte, + writer io.Writer, +) (*proxyService, error) { + logger := log.NewWithOptions(writer, "", true) + + // Create a net.Listener on a random port + listener, err := ephemeralListenerOnPort("0") + if err != nil { + return nil, err + } + + // Extract the host and port from the net.Listener + host, port, err := net.SplitHostPort( + listener.Addr().String(), + ) + if err != nil { + return nil, err + } + + // Create HTTP service connector + svcConnector := generic.NewConnector( + connector.NewResources(config, logger), + ) + + httpCfg, err := v2.NewHTTPConfig(config) + if err != nil { + return nil, err + } + + // Create the TCP proxy service + tcpProxySvc, err := httpInternal.NewProxyService( + []httpInternal.Subservice{ + { + Connector: svcConnector, + ConnectorID: "test", + RetrieveCredentials: func() (bytes map[string][]byte, e error) { + // Clone credentials to prevents any mutation or zeroization + return cloneCredentials(credentials), nil + }, + AuthenticateURLsMatching: httpCfg.AuthenticateURLsMatching, + }, + }, + listener, + logger, + ) + if err != nil { + return nil, err + } + + return &proxyService{ + host: host, + port: port, + // Starts the TCP proxy service + start: func() { + err := tcpProxySvc.Start() + if err != nil { + logger.Warnf("proxyService#start: %s", err) + } + }, + // Stops the TCP proxy service, and cleans up + stop: func() { + // Ensure the proxyService service is stopped + err = tcpProxySvc.Stop() + if err != nil { + logger.Warnf("proxyService#stop: %s", err) + } + }, + }, nil +} diff --git a/test/connector/http/generic/troubleshooting/troubleshooting.md b/test/connector/http/generic/troubleshooting/troubleshooting.md new file mode 100644 index 000000000..3ec7dffcc --- /dev/null +++ b/test/connector/http/generic/troubleshooting/troubleshooting.md @@ -0,0 +1,112 @@ + +### Connecting to Secretless as an HTTPS proxy +#### Symptoms +- You see a CONNECT request in your Secretless logs that looks something like: + + ``` + 2020/05/06 18:39:19 [DEBUG] Got request httpbin.org:443 CONNECT //httpbin.org:443 + ``` + +- Sample client log output messages + + curl: + ``` + * Trying 127.0.0.1... + * TCP_NODELAY set + * Connected to 127.0.0.1 (127.0.0.1) port 62160 (#0) + * Establish HTTP proxy tunnel to httpbin.org:443 + > CONNECT httpbin.org:443 HTTP/1.1 + > Host: httpbin.org:443 + > User-Agent: curl/7.54.0 + > Proxy-Connection: Keep-Alive + > + < HTTP/1.1 405 Method Not Allowed + < Content-Type: text/plain; charset=utf-8 + < X-Content-Type-Options: nosniff + < Date: Wed, 06 May 2020 17:39:19 GMT + < Content-Length: 26 + < + * Received HTTP code 405 from proxy after CONNECT + * Closing connection 0 + curl: (56) Received HTTP code 405 from proxy after CONNECT + ``` + + Go client: + ``` + Get https://httpbin.org/anything: Method Not Allowed + ``` + +#### Known Causes +This type of error occurs when the client attempts to use Secretless as an HTTPS proxy. +Secretless can only act as an HTTP proxy. +This error can happen for a few reasons: +1. An explicit attempt to use Secretless as an HTTPS proxy +1. Providing the client an HTTPS target when intending to proxy the connection through Secretless might result in the client attempting to use Secretless as an HTTPS proxy, as is the case with Go's standard library HTTP client. + +#### Resolution +- Ensure that target of your request is HTTP only e.g. http://httpbin.org. +- Secretless does not support HTTPS between the client and Secretless, it does support it between Secretless and the target. Do not use Secretless as an HTTPS proxy. +- To make connection between Secretless and the target an HTTPS connection you must set "forceSSL: true" on the Secretless service connector config. + + +### HTTPS certificate verification failure when forceSSL is set +#### Symptoms +- You see an x509 certificate error in your Secretless logs that looks something like: + + ``` + 2020/05/06 18:39:19 [DEBUG] Got request / self-signed.badssl.com GET http://self-signed.badssl.com/ + 2020/05/06 18:39:19 [DEBUG] Using connector 'test' for request http://self-signed.badssl.com/ + 2020/05/06 18:39:20 [DEBUG] Error: x509: certificate signed by unknown authority + ``` + +- Sample client log output messages + + curl: + ``` + * Rebuilt URL to: http://self-signed.badssl.com/ + * Trying 127.0.0.1... + * TCP_NODELAY set + * Connected to 127.0.0.1 (127.0.0.1) port 62165 (#0) + > GET http://self-signed.badssl.com/ HTTP/1.1 + > Host: self-signed.badssl.com + > User-Agent: curl/7.54.0 + > Accept: */* + > Proxy-Connection: Keep-Alive + > + < HTTP/1.1 503 Service Unavailable + < Content-Type: text/plain; charset=utf-8 + < X-Content-Type-Options: nosniff + < Date: Wed, 06 May 2020 17:39:20 GMT + < Content-Length: 46 + < + { [46 bytes data] + * Connection #0 to host 127.0.0.1 left intact + x509: certificate signed by unknown authority + ``` + + Go client: + ``` + Status: + 503 Service Unavailable + + Body: + x509: certificate is valid for *.badssl.com, badssl.com, not wrong.host.badssl.com + ``` + +#### Known Causes +This type of error occurs when the client attempts to connect to a target with a self-signed certificate, and there is some failure on verification. Secretless verifies all HTTPS connections to the target. + +There are several reasons why verification might fail including: +1. The signer of the target's certificate is not a trusted CA +1. The target's certificate is expired or is not yet valid +1. The target's certificate is not valid for the host + +#### Resolution + +This type of error can be broken into 2 categories. +1. The signer of the target's certificate is not a trusted CA +1. The rest + +For the rest (2), you must ensure that the target's certificate is valid for the target. + +For (1) you will need to ensure that Secretless is aware of the root certificate authority (CA) it should use to verify the server certificates when proxying requests. To do this, ensure that the environment variable **SECRETLESS_HTTP_CA_BUNDLE** is set. **SECRETLESS_HTTP_CA_BUNDLE** is a path to the bundle of CA certificates that are appended to the certificate pool that Secretless uses for server certificate verification of all HTTP service connectors.