Implement updated Writable spec

CONTRIBUTING.md

@@ -28,6 +28,10 @@ It should always be safe to run `go get -u=patch` to pickup patches.
 
 When making changes to Wash's APIs, remember to update the inline swagger documentation. Instructions for regenerating the API docs are [here](./website/README.md#regenerate-swagger-docs).
 
+### Local filesystem testing
+
+A developer-only plugin is included in Wash that references the local filesystem. It can be enabled by setting the `WASH_LOCALFS` environment variable to the local file path to mount before starting Wash. This can be useful for testing changes to core functionality like the `fuse` or `api` modules.
+
 ## Submitting Changes
 Fork the repo, make changes, file a Pull Request.
 

README.md

@@ -37,9 +37,9 @@ Project maintainers are not actively working on all of these things, but any of
 ### Primitives
 
 * [ ] file/directory upload _(prereq for executing commands that aren't just one-liners)_
-* [ ] edit a resource _(e.g. edit a file representing a k8s ConfigMap, and upon write save it via the k8s api)_
-* [ ] delete a resource _(e.g. `rm`-ing a file in an S3 bucket deletes it)_
-* [ ] signal handling to represent basic verbs _(e.g. sending a TERM to an EC2 instance will terminate it)_
+* [x] edit a resource _(e.g. edit a file representing a k8s ConfigMap, and upon write save it via the k8s api)_
+* [x] delete a resource _(e.g. `rm`-ing a file in an S3 bucket deletes it)_
+* [x] signal handling to represent basic verbs _(e.g. sending a TERM to an EC2 instance will terminate it)_
 * [ ] copy / move / rename _(how should this work?)_
 * [ ] make `stream` able to "go back in time" _(e.g. support `tail -100 -f` style of "look-back")_
 
@@ -52,7 +52,7 @@ Project maintainers are not actively working on all of these things, but any of
 ### CLI tools
 
 * [ ] colorized output for `ls`, similar to `exa -l`
-* [ ] make `ls` emit something useful when used against non-`wash` resources
+* [x] make `ls` emit something useful when used against non-`wash` resources
 * [ ] `exec` should work in parallel across multiple target resources
 * [ ] build an interactive shell that works over `exec` _(need to update plugins API to support this, most likely)_
 * [ ] a version of `top` that works using `wash` primitives to get information to display from multiple targets

api/fs/dir.go

@@ -38,13 +38,14 @@ func (d *dir) List(ctx context.Context) ([]plugin.Entry, error) {
 }
 
 func (d *dir) ChildSchemas() []*plugin.EntrySchema {
-	return nil
+	return []*plugin.EntrySchema{
+		(&dir{}).Schema(),
+		(&file{}).Schema(),
+	}
 }
 
 func (d *dir) Schema() *plugin.EntrySchema {
-	// Schema only makes sense for core plugins. apifs isn't a core
-	// plugin.
-	return nil
+	return plugin.NewEntrySchema(d, "dir")
 }
 
 var _ = plugin.Parent(&dir{})

api/fs/file.go

@@ -29,7 +29,12 @@ func (f *file) Read(ctx context.Context) ([]byte, error) {
 }
 
 func (f *file) Schema() *plugin.EntrySchema {
-	return nil
+	return plugin.NewEntrySchema(f, "file")
+}
+
+func (f *file) Write(ctx context.Context, p []byte) error {
+	return ioutil.WriteFile(f.path, p, 0640)
 }
 
 var _ = plugin.Readable(&file{})
+var _ = plugin.Writable(&file{})

api/fs/root.go

@@ -0,0 +1,52 @@
+package apifs
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/puppetlabs/wash/plugin"
+)
+
+// Root of the local filesystem plugin
+type Root struct {
+	dir
+}
+
+// Init for root
+func (r *Root) Init(cfg map[string]interface{}) error {
+	var basepath string
+	if pathI, ok := cfg["basepath"]; ok {
+		basepath, ok = pathI.(string)
+		if !ok {
+			return fmt.Errorf("local.basepath config must be a string, not %s", pathI)
+		}
+	} else {
+		basepath = "/tmp"
+	}
+
+	finfo, err := os.Stat(basepath)
+	if err != nil {
+		return err
+	}
+
+	r.dir.fsnode = newFSNode(finfo, basepath)
+
+	r.EntryBase = plugin.NewEntry("local")
+	r.DisableDefaultCaching()
+	return nil
+}
+
+// Schema returns the root's schema
+func (r *Root) Schema() *plugin.EntrySchema {
+	return plugin.
+		NewEntrySchema(r, "local").
+		SetDescription(rootDescription).
+		IsSingleton()
+}
+
+var _ = plugin.Root(&Root{})
+
+const rootDescription = `
+This plugin exposes part of the local filesystem. The path is mounted based on the path specified
+in the WASH_LOCALFS environment variable.
+`

cmd/internal/server/core.go

@@ -23,6 +23,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+// InternalPlugins lists the plugins enabled by default in Wash.
 var InternalPlugins = map[string]plugin.Root{
 	"aws":        &aws.Root{},
 	"docker":     &docker.Root{},

cmd/server.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/Benchkram/errz"
+	apifs "github.com/puppetlabs/wash/api/fs"
 	"github.com/puppetlabs/wash/cmd/internal/config"
 	"github.com/puppetlabs/wash/cmd/internal/server"
 	cmdutil "github.com/puppetlabs/wash/cmd/util"
@@ -169,6 +170,12 @@ func serverOptsFor(cmd *cobra.Command) (map[string]plugin.Root, server.Opts, err
 		pluginConfig[name] = viper.GetStringMap(name)
 	}
 
+	// Developer flag to enable a local filesystem for testing core functionality.
+	if localfsPath := os.Getenv("WASH_LOCALFS"); localfsPath != "" {
+		plugins["local"] = &apifs.Root{}
+		pluginConfig["local"] = map[string]interface{}{"basepath": localfsPath}
+	}
+
 	// Return the options
 	return plugins, server.Opts{
 		CPUProfilePath: viper.GetString("cpuprofile"),

docs/docs/external-plugins.md

@@ -7,6 +7,7 @@ title: External Plugins
     - [init](#init)
     - [list](#list)
     - [read](#read)
+    - [write](#write)
     - [metadata](#metadata)
     - [stream](#stream)
     - [exec](#exec)
@@ -156,6 +157,23 @@ Som
 
 where `Some content` is the entry's content.
 
+## write
+`<plugin_script> write <path> <state>`
+
+When `write` is invoked, the script must read from `stdin` to get the content to write to the entry.
+
+Wash distinguishes between two different patterns for things you can read and write. It considers a "file-like" entry to be one with a defined size (so the `size` attribute is set when listing the entry). Reading and writing a "file-like" entry edits the contents. The data passed to `stdin` is meant to be the entire content of the file.
+
+Something that can be read and written but doesn't define size has different characteristics. Reading and writing are not symmetrical: if you write to it then read from it, you may not see what you just wrote. So these non-file-like entries error if you try to open them with a ReadWrite handle. If your plugin implements non-file-like write-semantics, remember to document how they work in the plugin schema's description.
+
+### Examples
+
+```
+bash-3.2$ echo 'new content' | /path/to/myplugin.rb write /myplugin/foo ''
+```
+
+results in changing the entry's content to `new content`.
+
 ## metadata
 `<plugin_script> metadata <path> <state>`
 

docs/docs/index.md

@@ -29,6 +29,7 @@ title: Docs
 * [Actions](#actions)
   * [list](#list)
   * [read](#read)
+  * [write](#write)
   * [stream](#stream)
   * [exec](#exec)
   * [delete](#delete)
@@ -245,7 +246,7 @@ gcp/Wash/storage/some-wash-stuff
 ```
 
 ### read
-The `read` action lets you view an entry’s content. Thus, any command that reads a file also works with these entries.
+The `read` action lets you read data from an entry. Thus, any command that reads a file also works with these entries.
 
 #### Examples
 ```
@@ -260,6 +261,34 @@ wash . ❯ grep "Hello" gcp/Wash/storage/some-wash-stuff/an\ example\ folder/sta
 echo "Hello, world!"
 ```
 
+### write
+The `write` action lets you write data to an entry. Thus, any command that writes a file also works with these entries.
+
+Note that Wash distinguishes between file-like and non-file-like entries. An entry is file-like if it's readable and writable and defines its size; you can edit it like a file.
+
+If it doesn't define a size then it's non-file-like, and trying to open it with a ReadWrite handle will error; reads from it may not return data you previously wrote to it. You should check its documentation with the `docs` command for that entry's write semantics. We also recommend not using editors with these entries to avoid weird behavior.
+
+#### Examples
+Modifying a file stored in Google Cloud Storage
+```
+wash . ❯ echo 'exit 1' >> gcp/Wash/storage/some-wash-stuff/an\ example\ folder/static.sh
+wash . ❯ cat gcp/Wash/storage/some-wash-stuff/an\ example\ folder/static.sh
+#!/bin/sh
+
+echo "Hello, world!"
+exit 1
+```
+
+Writing a message to a hypothetical message queue where each write publishes a message and each read consumes a message
+```
+wash > echo 'message 1' >> myqueue
+wash > echo 'message 2' >> myqueue
+wash > cat myqueue
+message 1
+wash > cat myqueue
+message 2
+```
+
 ### stream
 The `stream` action lets you stream an entry’s content for updates.
 

fuse/core.go

@@ -55,18 +55,16 @@ func getIDs() (uint32, uint32) {
 var uid, gid = getIDs()
 
 type fuseNode struct {
-	ftype             string
-	parent            *dir
-	entry             plugin.Entry
-	entryCreationTime time.Time
+	ftype  string
+	parent *dir
+	entry  plugin.Entry
 }
 
-func newFuseNode(ftype string, parent *dir, entry plugin.Entry) *fuseNode {
-	return &fuseNode{
-		ftype:             ftype,
-		parent:            parent,
-		entry:             entry,
-		entryCreationTime: time.Now(),
+func newFuseNode(ftype string, parent *dir, entry plugin.Entry) fuseNode {
+	return fuseNode{
+		ftype:  ftype,
+		parent: parent,
+		entry:  entry,
 	}
 }
 
@@ -75,7 +73,7 @@ func (f *fuseNode) String() string {
 }
 
 // Applies attributes where non-default, and sets defaults otherwise.
-func (f *fuseNode) applyAttr(a *fuse.Attr, attr *plugin.EntryAttributes, defaultMode os.FileMode) {
+func applyAttr(a *fuse.Attr, attr plugin.EntryAttributes, defaultMode os.FileMode) {
 	// Setting a.Valid to 1 second avoids frequent Attr calls.
 	a.Valid = 1 * time.Second
 
@@ -143,44 +141,6 @@ func (f *fuseNode) refind(ctx context.Context) (plugin.Entry, error) {
 	return plugin.FindEntry(ctx, parent, segments)
 }
 
-func (f *fuseNode) Attr(ctx context.Context, a *fuse.Attr) error {
-	// Attr is not a particularly interesting call and happens a lot. Log it to debug like other
-	// activity, but leave it out of activity because it introduces history entries for lots of
-	// miscellaneous shell activity.
-	log.Debugf("FUSE: Attr %v", f)
-
-	// FUSE caches nodes for a long time, meaning there's a chance that
-	// f's attributes are outdated. 'refind' requests the entry from its
-	// parent to ensure it has updated attributes.
-	updatedEntry, err := f.refind(ctx)
-	if err != nil {
-		activity.Warnf(ctx, "FUSE: Attr errored %v, %v", f, err)
-		return err
-	}
-	attr := plugin.Attributes(updatedEntry)
-	// NOTE: We could set f.entry to updatedEntry, but doing so would require
-	// a separate mutex which may hinder performance. Since updating f.entry
-	// is not strictly necessary for the other FUSE operations, we choose to
-	// leave it alone.
-
-	var mode os.FileMode
-	if plugin.ListAction().IsSupportedOn(updatedEntry) {
-		mode = os.ModeDir | 0550
-	} else {
-		if plugin.WriteAction().IsSupportedOn(updatedEntry) {
-			mode |= 0220
-		}
-		if plugin.ReadAction().IsSupportedOn(updatedEntry) ||
-			plugin.StreamAction().IsSupportedOn(updatedEntry) {
-			mode |= 0440
-		}
-	}
-
-	f.applyAttr(a, &attr, mode)
-	log.Debugf("FUSE: Attr finished %v", f)
-	return nil
-}
-
 // ServeFuseFS starts serving a fuse filesystem that lists the registered plugins.
 // It returns three values:
 //   1. A channel to initiate the shutdown (stopCh).

fuse/dir.go

@@ -2,6 +2,7 @@ package fuse
 
 import (
 	"context"
+	"os"
 
 	"bazil.org/fuse"
 	"bazil.org/fuse/fs"
@@ -13,7 +14,7 @@ import (
 // ==== FUSE Directory Interface ====
 
 type dir struct {
-	*fuseNode
+	fuseNode
 }
 
 var _ fs.Node = (*dir)(nil)
@@ -94,3 +95,29 @@ func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 	activity.Record(ctx, "FUSE: Listed in %v: %+v", d, res)
 	return res, nil
 }
+
+func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
+	// FUSE caches nodes for a long time, meaning there's a chance that
+	// f's attributes are outdated. 'refind' requests the entry from its
+	// parent to ensure it has updated attributes.
+	entry, err := d.refind(ctx)
+	if err != nil {
+		activity.Warnf(ctx, "FUSE: Attr errored %v, %v", d, err)
+		return err
+	}
+	// NOTE: We could set f.entry to entry, but doing so would require
+	// a separate mutex which may hinder performance. Since updating f.entry
+	// is not strictly necessary for the other FUSE operations, we choose to
+	// leave it alone.
+
+	applyAttr(a, plugin.Attributes(entry), os.ModeDir|0550)
+	// Attr is not a particularly interesting call and happens a lot. Log it to debug like other
+	// activity, but leave it out of activity because it introduces history entries for lots of
+	// miscellaneous shell activity.
+	if d.parent == nil {
+		log.Tracef("FUSE: Attr %v", d)
+	} else {
+		log.Debugf("FUSE: Attr %v: %+v", d, *a)
+	}
+	return nil
+}