Fix bug so slurp can pipe to either http/s hoarder

.travis.yml

@@ -5,5 +5,5 @@ go: 1.6
 install:
   - go get github.com/nanopack/hoarder
   - go get -t -v .
-  - hoarder --server -b file:///tmp/hoarder --insecure=false &
+  - hoarder --server -b file:///tmp/hoarder -H "https://127.0.0.1:7410" &
 script: go test -v ./...

README.md

@@ -64,7 +64,7 @@ Usage:
   slurp [flags]
 
 Flags:
-  -a, --api-address="127.0.0.1:1566": Listen address for the API
+  -a, --api-address="127.0.0.1:1566": Listen uri for the API (scheme defaults to https)
   -t, --api-token="secret": Token for API Access
   -b, --build-dir="/var/db/slurp/build/": Build staging directory
   -c, --config-file="": Configuration file to load

api/api.go

@@ -4,8 +4,10 @@ package api
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 
 	"github.com/gorilla/pat"
 	"github.com/nanobox-io/golang-nanoauth"
@@ -29,12 +31,17 @@ type (
 
 // start the web server
 func StartApi() error {
+	uri, err := url.Parse(config.ApiAddress)
+	if err != nil {
+		return fmt.Errorf("Failed to parse 'api-address' - %v", err)
+	}
+
 	var auth nanoauth.Auth
 	auth.Header = "X-AUTH-TOKEN"
 
-	if config.Insecure {
-		config.Log.Info("Api listening at http://%s...", config.ApiAddress)
-		return auth.ListenAndServe(config.ApiAddress, config.ApiToken, routes(), "/ping")
+	if uri.Scheme == "http" {
+		config.Log.Info("Api listening at http://%s...", uri.Host)
+		return auth.ListenAndServe(uri.Host, config.ApiToken, routes(), "/ping")
 	}
 
 	cert, err := nanoauth.Generate("slurp.nanobox.io")
@@ -43,8 +50,8 @@ func StartApi() error {
 	}
 	auth.Certificate = cert
 
-	config.Log.Info("Api listening at https://%s...", config.ApiAddress)
-	return auth.ListenAndServeTLS(config.ApiAddress, config.ApiToken, routes(), "/ping")
+	config.Log.Info("Api listening at https://%s...", uri.Host)
+	return auth.ListenAndServeTLS(uri.Host, config.ApiToken, routes(), "/ping")
 }
 
 // api routes

api/api_test.go

@@ -118,7 +118,7 @@ func initialize() {
 func rest(method, route, data string) ([]byte, error) {
 	body := bytes.NewBuffer([]byte(data))
 
-	req, _ := http.NewRequest(method, fmt.Sprintf("https://%s%s", config.ApiAddress, route), body)
+	req, _ := http.NewRequest(method, fmt.Sprintf("%s%s", config.ApiAddress, route), body)
 	req.Header.Add("X-AUTH-TOKEN", "")
 
 	res, err := http.DefaultClient.Do(req)

backend/backend.go

@@ -29,10 +29,12 @@ func Initialize() error {
 		return fmt.Errorf("Failed to parse backend connection - %v", err)
 	}
 	switch u.Scheme {
-	case "hoarder":
-		backend = &hoarder{}
+	case "hoarder": // insecure hoarder
+		backend = &hoarder{proto: "http"}
+	case "hoarders": // secure hoarder
+		backend = &hoarder{proto: "https"}
 	default:
-		backend = &hoarder{}
+		backend = &hoarder{proto: "https"}
 	}
 	storeAddr = u.Host
 	return backend.initialize()

backend/hoarder.go

@@ -9,7 +9,9 @@ import (
 	"github.com/nanobox-io/slurp/config"
 )
 
-type hoarder struct{}
+type hoarder struct {
+	proto string
+}
 
 // ensure hoarder is up
 func (self hoarder) initialize() error {
@@ -37,7 +39,7 @@ func (self hoarder) rest(method, path string, body io.Reader) (*http.Response, e
 	config.Log.Trace("[client] - %v hoarder/%v", method, path)
 	var client *http.Client
 	client = http.DefaultClient
-	uri := fmt.Sprintf("https://%s/%s", storeAddr, path)
+	uri := fmt.Sprintf("%s://%s/%s", self.proto, storeAddr, path)
 
 	// if store-ssl is true, verify cert
 	if !config.StoreSSL {
@@ -51,19 +53,8 @@ func (self hoarder) rest(method, path string, body io.Reader) (*http.Response, e
 	req.Header.Add("X-AUTH-TOKEN", config.StoreToken)
 	res, err := client.Do(req)
 	if err != nil {
-		// if requesting `https://` failed, server may have been started with `-i`, try `http://`
-		uri = fmt.Sprintf("http://%s/%s", storeAddr, path)
-		req, er := http.NewRequest(method, uri, body)
-		if er != nil {
-			panic(er)
-		}
-		req.Header.Add("X-AUTH-TOKEN", config.StoreToken)
-		var err2 error
-		res, err2 = client.Do(req)
-		if err2 != nil {
-			// return original error to client
-			return nil, err
-		}
+		// return original error to client
+		return nil, err
 	}
 	if res.StatusCode == 401 {
 		return nil, fmt.Errorf("401 Unauthorized. Please specify backend api token (-T 'backend-token')")

config/config.go

@@ -12,26 +12,26 @@ import (
 )
 
 var (
-	ApiToken   = "secret"                   // Token for API Access
-	ApiAddress = "127.0.0.1:1566"           // Listen address for the API
-	BuildDir   = "/var/db/slurp/build/"     // Build staging directory
-	ConfigFile = ""                         // Configuration file to load
-	Insecure   = false                      // Disable tls key checking (client) and listen on http (server)
-	LogLevel   = "info"                     // Log level to output [fatal|error|info|debug|trace]
-	SshAddr    = "127.0.0.1:1567"           // Address ssh server will listen on (ip:port combo)
-	SshHostKey = "/var/db/slurp/slurp_rsa"  // SSH host (private) key file
-	StoreAddr  = "hoarder://127.0.0.1:7410" // Storage host address
-	StoreSSL   = false                      // Disable tls key checking (client) and listen on http (server)
-	StoreToken = ""                         // Storage auth token
-	Version    = false                      // Print version info and exit
+	ApiToken   = "secret"                    // Token for API Access
+	ApiAddress = "https://127.0.0.1:1566"    // Listen uri for the API (scheme defaults to https)
+	BuildDir   = "/var/db/slurp/build/"      // Build staging directory
+	ConfigFile = ""                          // Configuration file to load
+	Insecure   = false                       // Disable tls key checking (client)
+	LogLevel   = "info"                      // Log level to output [fatal|error|info|debug|trace]
+	SshAddr    = "127.0.0.1:1567"            // Address ssh server will listen on (ip:port combo)
+	SshHostKey = "/var/db/slurp/slurp_rsa"   // SSH host (private) key file
+	StoreAddr  = "hoarders://127.0.0.1:7410" // Storage host address
+	StoreSSL   = false                       // Disable tls key checking (client) and listen on http (server)
+	StoreToken = ""                          // Storage auth token
+	Version    = false                       // Print version info and exit
 
 	Log lumber.Logger // Central logger for slurp
 )
 
 // AddFlags adds the available cli flags
 func AddFlags(cmd *cobra.Command) {
 	cmd.PersistentFlags().StringVarP(&ApiToken, "api-token", "t", ApiToken, "Token for API Access")
-	cmd.PersistentFlags().StringVarP(&ApiAddress, "api-address", "a", ApiAddress, "Listen address for the API")
+	cmd.PersistentFlags().StringVarP(&ApiAddress, "api-address", "a", ApiAddress, "Listen uri for the API (scheme defaults to https)")
 	cmd.PersistentFlags().StringVarP(&BuildDir, "build-dir", "b", BuildDir, "Build staging directory")
 	cmd.PersistentFlags().StringVarP(&ConfigFile, "config-file", "c", ConfigFile, "Configuration file to load")
 	cmd.PersistentFlags().BoolVarP(&Insecure, "insecure", "i", Insecure, "Disable tls key checking (client) and listen on http (server)")

core/slurp.go

@@ -125,7 +125,7 @@ func CommitStage(buildId string) error {
 	if err != nil {
 		return fmt.Errorf("Failed to compress build - %v", err)
 		// the error `io: read/write on closed pipe` here is likely due to
-		// mismatched hoarder/slurp protocols (http/https)
+		// wrong backend protocol (http/https)
 	}
 
 	config.Log.Trace("Compressed build")

main.go

@@ -12,7 +12,7 @@
 //    slurp [flags]
 //
 //  Flags:
-//    -a, --api-address="127.0.0.1:1566": Listen address for the API
+//    -a, --api-address="127.0.0.1:1566": Listen uri for the API (scheme defaults to https)
 //    -t, --api-token="secret": Token for API Access
 //    -b, --build-dir="/var/db/slurp/build/": Build staging directory
 //    -c, --config-file="": Configuration file to load

main_test.go

@@ -24,7 +24,7 @@ func TestMain(m *testing.M) {
 	// manually configure
 	initialize()
 
-	args := strings.Split("-b /tmp/slurpMain/ -l fatal -k /tmp/slurp_rsa -s 127.0.0.1:1568 -a 127.0.0.1:1564", " ")
+	args := strings.Split("-b /tmp/slurpMain/ -l fatal -k /tmp/slurp_rsa -s 127.0.0.1:1568 -a https://127.0.0.1:1564", " ")
 	slurp.SetArgs(args)
 
 	// start api
@@ -70,7 +70,7 @@ func initialize() {
 func rest(method, route, data string) ([]byte, error) {
 	body := bytes.NewBuffer([]byte(data))
 
-	req, _ := http.NewRequest(method, fmt.Sprintf("https://%s%s", config.ApiAddress, route), body)
+	req, _ := http.NewRequest(method, fmt.Sprintf("%s%s", config.ApiAddress, route), body)
 	req.Header.Add("X-AUTH-TOKEN", "")
 
 	res, err := http.DefaultClient.Do(req)