Minor cleanup. Refactor to use build-id as user

.gitignore

@@ -1,2 +1,3 @@
 slurp
+build/
 

README.md

@@ -0,0 +1,122 @@
+<!-- [![Build Status](https://travis-ci.org/nanobox-io/slurp.svg)](https://travis-ci.org/nanobox-io/slurp) -->
+[![GoDoc](https://godoc.org/github.com/nanobox-io/slurp?status.svg)](https://godoc.org/github.com/nanobox-io/slurp)
+
+# Slurp
+Intermediary to the stored build/blob, used specifically to speed up publishing nanobox builds.
+
+## Quickstart:
+```sh
+# Once hoarder is running, slurp can be quickly started by running ()
+slurp -b /tmp/build
+
+# register a new build
+curl -k https://localhost:1566/stages -d '{"new-id": "test"}'
+# sync up your build (current directory)
+rsync -v --delete -aR . -e 'ssh -p 1567' [email protected]:test
+# tell slurp you are done syncing
+curl -k https://localhost:1566/stages/test -X PUT
+# Congratulations!
+```
+**Part II:**
+```sh
+# after modifying your code, register a new build
+curl -k https://localhost:1566/stages -d '{"old-id": "test", "new-id": "test2"}'
+# sync up your build (current directory)
+rsync -v --delete -aR . -e 'ssh -p 1567' [email protected]:this-location-really-doesnt-matter
+# tell slurp you are done syncing
+curl -k https://localhost:1566/stages/test2 -X PUT
+# Congratulations!
+```
+
+## Usage:
+
+### As a Server
+To start slurp as a server, run:
+
+`slurp`
+
+An optional config file can also be passed on startup:
+
+`slurp -c /path/to/config.json`
+
+>config.json
+>```json
+{
+  "api-token": "secret",
+  "api-address": "127.0.0.1:1566",
+  "build-dir": "/var/db/slurp/build/",
+  "config-file": "",
+  "insecure": false,
+  "log-level": "info",
+  "ssh-addr": "127.0.0.1:1567",
+  "ssh-host": "/var/db/slurp/slurp_rsa",
+  "store-addr": "hoarder://127.0.0.1:7410",
+  "store-token": ""
+}
+```
+
+`slurp -h` will show usage and a list of commands:
+
+```
+slurp - build intermediary
+
+Usage:
+  slurp [flags]
+
+Flags:
+  -a, --api-address="127.0.0.1:1566": Listen address for the API
+  -t, --api-token="secret": Token for API Access
+  -b, --build-dir="/var/db/slurp/build/": Build staging directory
+  -c, --config-file="": Configuration file to load
+  -i, --insecure[=false]: Disable tls key checking (client) and listen on http (server)
+  -l, --log-level="info": Log level to output [fatal|error|info|debug|trace]
+  -s, --ssh-addr="127.0.0.1:1567": Address ssh server will listen on (ip:port combo)
+  -k, --ssh-host="/var/db/slurp/slurp_rsa": SSH host (private) key file
+  -S, --store-addr="hoarder://127.0.0.1:7410": Storage host address
+  -T, --store-token="": Storage auth token
+```
+
+## API:
+
+| Route | Description | Payload | Output |
+| --- | --- | --- | --- |
+| **POST** | /stages | Stage a new build | json stage object | json auth object |
+| **PUT** | /stages/:id | Commit a new build | nil | success/err message |
+| **DELETE** | /stages/:id | Delete a build | nil | success/err message |
+- Commit will clean up the staged build *after* pushing it to storage
+- Delete will clean up the staged build *without* pushing it to storage
+
+## Data types:
+
+### Stage
+json:
+```json
+{
+  "old-id": "abc123",
+  "new-id": "def456"
+}
+```
+Fields:
+- **old-id**: ID (in storage) of build to update
+- **new-id**: ID for the new build (required)
+
+### Auth
+json:
+```json
+{
+  "secret": "def456"
+}
+```
+Fields:
+- **secret**: Contains the username to ssh with (ID of new build)
+
+## Todo
+- tests
+- rebuild auth user list on reboot
+- routinely clean up undeleted builds
+
+## Changelog
+- v0.0.1 (April 25, 2016)
+  - slurp is born
+
+[![nanobox oss logo](http://nano-assets.gopagoda.io/open-src/nanobox-open-src.png)](http://nanobox.io/open-source)

api/README.md

@@ -0,0 +1,62 @@
+## Sample Usage Demo
+This demo has [Hoarder](github.com/nanopack/hoarder) running locally
+
+### create some thing
+```sh
+dd if=/dev/urandom of=thing bs=1M count=10
+```
+
+#### tell slurp about new build
+```sh
+curl -k -H "X-AUTH-TOKEN: secret" https://localhost:1566/stages -d '{"new-id": "test"}'
+```
+
+#### sync files
+```sh
+rsync -v --delete -aR . -e 'ssh -p 1567' [email protected]:test/
+```
+
+#### commit update (done syncing)
+```sh
+curl -k -H "X-AUTH-TOKEN: secret" https://localhost:1566/stages/test -X PUT
+```
+
+#### delete temp build dir
+```sh
+curl -k -H "X-AUTH-TOKEN: secret" https://localhost:1566/stages/test -X DELETE
+```
+
+#### *get build from hoarder
+```sh
+curl localhost:7410/blobs/test | tar -zxf -
+du -h thing
+# 10M thing
+```
+
+### make changes
+```sh
+echo '{things"}' > file
+```
+
+#### tell slurp about new build
+```sh
+curl -k -H "X-AUTH-TOKEN: secret" https://localhost:1566/stages -d '{"old-id": "test", "new-id": "test2"}'
+```
+
+#### sync files
+```sh
+rsync --delete -aR . -e 'ssh -p 1567' [email protected]:this-location-really-doesnt-matter
+```
+
+#### commit update (done syncing)
+```sh
+curl -k -H "X-AUTH-TOKEN: secret" https://localhost:1566/stages/test2 -X PUT
+```
+
+#### *get build from hoarder
+```sh
+curl localhost:7410/blobs/test2 | tar -zxf -
+ls
+# file thing
+```
+- *new directory

api/api.go

@@ -1,10 +1,9 @@
+// Package "api" defines the routes accessible and the logic when they are hit.
 package api
 
 import (
-	"crypto/tls"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 
@@ -15,8 +14,8 @@ import (
 )
 
 var (
-	BadJson      = errors.New("Bad JSON Syntax Received in Body")
-	BodyReadFail = errors.New("Body Read Failed")
+	badJson      = errors.New("Bad JSON Syntax Received in Body")
+	bodyReadFail = errors.New("Body Read Failed")
 )
 
 type (
@@ -31,34 +30,30 @@ type (
 // start the web server
 func StartApi() error {
 	if config.Insecure {
-		config.Log.Info("Api listening at http://%s:%s...", config.ApiHost, config.ApiPort)
-		return http.ListenAndServe(fmt.Sprintf("%s:%s", config.ApiHost, config.ApiPort), routes())
+		config.Log.Info("Api listening at http://%s...", config.ApiAddress)
+		return http.ListenAndServe(config.ApiAddress, routes())
 	}
+
 	var auth nanoauth.Auth
-	var cert *tls.Certificate
-	var err error
-	if config.ApiCert == "" {
-		cert, err = nanoauth.Generate("slurp.nanobox.io")
-	} else {
-		cert, err = nanoauth.Load(config.ApiCert, config.ApiKey, config.ApiKeyPassword)
-	}
+	cert, err := nanoauth.Generate("slurp.nanobox.io")
 	if err != nil {
 		return err
 	}
 	auth.Certificate = cert
 	auth.Header = "X-AUTH-TOKEN"
 
-	config.Log.Info("Api listening at https://%s:%s...", config.ApiHost, config.ApiPort)
-	return auth.ListenAndServeTLS(fmt.Sprintf("%s:%s", config.ApiHost, config.ApiPort), config.ApiToken, routes())
+	config.Log.Info("Api listening at https://%s...", config.ApiAddress)
+	return auth.ListenAndServeTLS(config.ApiAddress, config.ApiToken, routes())
 }
 
 // api routes
 func routes() *pat.Router {
 	router := pat.New()
 
-	router.Post("/", addStage)
-	router.Put("/{buildId}", commitStage)
-	router.Delete("/{buildId}", deleteStage)
+	// keep "/stages" so a build named "ping" won't break anything
+	router.Post("/stages", addStage)
+	router.Put("/stages/{buildId}", commitStage)
+	router.Delete("/stages/{buildId}", deleteStage)
 
 	router.Get("/ping", pong)
 
@@ -97,14 +92,14 @@ func parseBody(req *http.Request, v interface{}) error {
 	b, err := ioutil.ReadAll(req.Body)
 	if err != nil {
 		config.Log.Error(err.Error())
-		return BodyReadFail
+		return bodyReadFail
 	}
 	defer req.Body.Close()
 
 	// parse body and store in v
 	err = json.Unmarshal(b, v)
 	if err != nil {
-		return BadJson
+		return badJson
 	}
 
 	return nil

api/stages.go

@@ -6,16 +6,18 @@ import (
 	"github.com/nanopack/slurp/core"
 )
 
+// for whatever reason, these need to be exported so json.[un]marshal can utilize it
 type build struct {
-	// for whatever reason, these need to be exported so json.[un]marshal can utilize it
-	OldId string `json:"old-id"`
-	NewId string `json:"new-id"`
+	OldId string `json:"old-id"` // build to fetch from storage
+	NewId string `json:"new-id"` // build to stage and store
 }
 
 type auth struct {
 	AuthSecret string `json:"secret"`
 }
 
+// addStage prepares a directory for receiving the new build. If an old build is specified,
+// that build is fetched from hoarder, otherwise a new directory is created.
 func addStage(rw http.ResponseWriter, req *http.Request) {
 	var stage build
 	err := parseBody(req, &stage)
@@ -30,15 +32,18 @@ func addStage(rw http.ResponseWriter, req *http.Request) {
 	}
 
 	// stage the build
-	secret, err := slurp.AddStage(stage.OldId, stage.NewId)
+	err = slurp.AddStage(stage.OldId, stage.NewId)
 	if err != nil {
 		writeBody(rw, req, apiError{err.Error()}, http.StatusInternalServerError)
 		return
 	}
 
-	writeBody(rw, req, auth{secret}, http.StatusOK)
+	writeBody(rw, req, auth{stage.NewId}, http.StatusOK)
 }
 
+// commitStage is called once the local build is synced with the staged build. It will
+// compress and upload the staged build to hoarder. CommitStage will also remove the
+// user for security.
 func commitStage(rw http.ResponseWriter, req *http.Request) {
 	// PUT /stages/{buildId}
 	buildId := req.URL.Query().Get(":buildId")
@@ -50,9 +55,17 @@ func commitStage(rw http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	// delete the staged build
+	err = slurp.DeleteStage(buildId)
+	if err != nil {
+		writeBody(rw, req, apiError{err.Error()}, http.StatusInternalServerError)
+		return
+	}
+
 	writeBody(rw, req, apiMsg{"Success"}, http.StatusOK)
 }
 
+// deleteStage removes the staged build directory
 func deleteStage(rw http.ResponseWriter, req *http.Request) {
 	// DELETE /stages/{buildId}
 	buildId := req.URL.Query().Get(":buildId")

backend/backend.go

@@ -15,32 +15,34 @@ type blobReadWriter interface {
 	writeBlob(id string, blob io.Reader) error
 }
 
-var Backend blobReadWriter // exportable for testing.. todo: if needed
+// the pluggable (future) backend
+var backend blobReadWriter
 
+// Initialize prepares the backend and ensures it is available
 func Initialize() error {
 	var err error
 	var u *url.URL
 	u, err = url.Parse(config.StoreAddr)
 	if err != nil {
-		return fmt.Errorf("Failed to parse db connection - %v", err)
+		return fmt.Errorf("Failed to parse backend connection - %v", err)
 	}
 	switch u.Scheme {
 	case "hoarder":
-		Backend = &hoarder{}
+		backend = &hoarder{}
 	default:
-		Backend = &hoarder{}
+		backend = &hoarder{}
 	}
 
 	config.StoreAddr = u.Host
-	return Backend.initialize()
+	return backend.initialize()
 }
 
 // ReadBlob reads a blob from a storage backend
 func ReadBlob(id string) (io.ReadCloser, error) {
-	return Backend.readBlob(id)
+	return backend.readBlob(id)
 }
 
 // WriteBlob writes a blob to a storage backend
 func WriteBlob(id string, blob io.Reader) error {
-	return Backend.writeBlob(id, blob)
+	return backend.writeBlob(id, blob)
 }