From e8878bde2e6fa73a83d85252b29a18aa9c49414a Mon Sep 17 00:00:00 2001 From: "R.I.Pienaar" Date: Tue, 10 May 2022 18:10:36 +0200 Subject: [PATCH] (#1665) initial basic working app framework This allows applications to be built that wraps RPC and KV components. Application definitions are documented using a schema and are validated on loading, from the definition a CLI app is built that can have nested sub commands and more. Application definitions goes in files called x-app.yaml, a symlink from x to choria would then load x-app.yaml and construct the CLI when x is invoked. An additional config file can be read called applications.yaml that can be accessed via go templates in the definition in a few places. The config and definitions can live in ., ~/.config/choria/builder or /etc/choria/builder. Paths are using XDG, but its a super primitive implementation for the moment, will be refined via issue 1624, thus paths and file names are subject to change Signed-off-by: R.I.Pienaar --- cmd/cmd.go | 18 +- go.mod | 1 + go.sum | 3 + internal/fs/schemas/builder.json | 243 +++++++++++++++++++++ internal/fs/templates.go | 1 + providers/appbuilder/builder.go | 355 +++++++++++++++++++++++++++++++ providers/appbuilder/kv.go | 124 +++++++++++ providers/appbuilder/parent.go | 48 +++++ providers/appbuilder/rpc.go | 354 ++++++++++++++++++++++++++++++ 9 files changed, 1146 insertions(+), 1 deletion(-) create mode 100644 internal/fs/schemas/builder.json create mode 100644 providers/appbuilder/builder.go create mode 100644 providers/appbuilder/kv.go create mode 100644 providers/appbuilder/parent.go create mode 100644 providers/appbuilder/rpc.go diff --git a/cmd/cmd.go b/cmd/cmd.go index 6f7853a25..f96619a51 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -12,11 +12,13 @@ import ( "path/filepath" "runtime" "runtime/pprof" + "strings" "sync" "syscall" "time" "github.com/choria-io/go-choria/protocol" + "github.com/choria-io/go-choria/providers/appbuilder" log "github.com/sirupsen/logrus" "gopkg.in/alecthomas/kingpin.v2" @@ -53,6 +55,20 @@ func ParseCLI() (err error) { go interruptWatcher() + // If we are not invoked as something something choria, then check + // if the app builder has an app configuration matching the name we + // are run as, if it does, we invoke it instead of the standard choria + // cli tools + // + // TODO: too janky, need to do a better job here, looking at the name is not enough + if !strings.Contains(os.Args[0], "choria") { + builder := appbuilder.NewAppBuilder(ctx, filepath.Base(os.Args[0])) + if builder.HasDefinition() { + builder.RunCommand() + os.Exit(0) + } + } + bi = &build.Info{} cli.app = kingpin.New("choria", "Choria Orchestration System") @@ -210,7 +226,7 @@ func forcequit() { } } - <-time.NewTimer(grace).C + <-time.After(grace) dumpGoRoutines() diff --git a/go.mod b/go.mod index bf44ebd8d..8349acb27 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/achanda/go-sysctl v0.0.0-20160222034550-6be7678c45d2 // indirect + github.com/adrg/xdg v0.4.0 // indirect github.com/aelsabbahy/GOnetstat v0.0.0-20160428114218-edf89f784e08 // indirect github.com/aelsabbahy/go-ps v0.0.0-20201009164808-61c449472dcf // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect diff --git a/go.sum b/go.sum index 11488d943..58117ba5d 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/achanda/go-sysctl v0.0.0-20160222034550-6be7678c45d2 h1:NYoPVh1XuUB5VBWLXRKoqzQhl4bajIxh+XuURbJ0uwc= github.com/achanda/go-sysctl v0.0.0-20160222034550-6be7678c45d2/go.mod h1:DCNKSpXhum14Y258jSbRmJvcesbzEdBPincz7yJUx3k= +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/aelsabbahy/GOnetstat v0.0.0-20160428114218-edf89f784e08 h1:oD15ssIOuFLi64zhkPRsaIDvhx4PeZb2QdQoR/wKY2g= github.com/aelsabbahy/GOnetstat v0.0.0-20160428114218-edf89f784e08/go.mod h1:FETZSu2VGNDJbGfeRExaz/SNbX0TTaqJEMo1yvsKoZ8= github.com/aelsabbahy/go-ps v0.0.0-20201009164808-61c449472dcf h1:KyjxaqJO0pHF7Clre644OiJ5s235JVRsz6ioDkoQ96s= @@ -752,6 +754,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/fs/schemas/builder.json b/internal/fs/schemas/builder.json new file mode 100644 index 000000000..fbd02f8f5 --- /dev/null +++ b/internal/fs/schemas/builder.json @@ -0,0 +1,243 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "id": "https://choria.io/schemas/choria/builder/v1/application.json", + "title": "io.choria.builder.v1.application", + "description": "Choria Builder Application Specification", + "type": "object", + "required": ["name","description","version","author","commands"], + "definitions": { + "shortname": { + "type": "string", + "minLength": 1, + "pattern": "^[a-z0-9_-]*$" + }, + "semver": { + "type": "string", + "description": "Semantic Versioning 2.0.0 version string", + "minLength": 5, + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" + }, + "standard_command": { + "type": "object", + "required": ["name","description","type"], + "properties": { + "name": { + "type": "string", + "description": "A unique name for this command", + "$ref": "#/definitions/shortname" + }, + "description": { + "$ref": "#/definitions/description" + } + } + }, + "description": { + "type": "string", + "description": "A human friendly description of what an item does", + "minLength": 1 + }, + "generic_flag": { + "type": "object", + "required": ["name","description"], + "properties": { + "name": { + "description": "A unique name for this flag", + "$ref": "#/definitions/shortname" + }, + "description": { + "$ref": "#/definitions/description" + }, + "required": { + "type": "boolean", + "description": "Indicates this flag must be passed", + "default": false + }, + "placeholder": { + "type": "string", + "description": "String to show as value place holder in help output" + } + } + }, + "generic_argument":{ + "type": "object", + "required": ["name","description"], + "properties": { + "name": { + "description": "A unique name for this argument", + "$ref": "#/definitions/shortname" + }, + "description": { + "$ref": "#/definitions/description" + }, + "required": { + "type": "boolean", + "description": "Indicates that this flag must be passed", + "default": false + } + } + }, + "commands": { + "type": "array", + "items": { + "anyOf": [ + {"$ref": "#/definitions/rpc_command"}, + {"$ref": "#/definitions/parent_command"}, + {"$ref": "#/definitions/kv_command"} + ] + } + }, + "parent_command": { + "type": "object", + "description": "A command that does not do anything but serves as a anchor for sub commands", + "additionalItems": false, + "allOf": [ + {"$ref":"#/definitions/standard_command"}, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "const": "parent" + }, + "commands": { + "description": "Additional CLI commands to add", + "$ref": "#/definitions/commands", + "minLength": 1 + } + } + } + ] + }, + "kv_command": { + "type": "object", + "description": "A command that interact with the Choria Key-Value store", + "additionalItems": false, + "allOf": [ + {"$ref":"#/definitions/standard_command"}, + { + "type": "object", + "required": ["type","action","bucket","key"], + "properties": { + "type": { + "type": "string", + "const": "kv" + }, + "commands": { + "description": "Additional CLI commands to add", + "$ref": "#/definitions/commands" + }, + "bucket": { + "type": "string", + "description": "The name of the Key-Value store bucket", + "pattern": "\\A[a-zA-Z0-9_-]+\\z", + "minLength": 1 + }, + "key": { + "type": "string", + "description": "The key to act on", + "minLength": 1 + }, + "value": { + "type": "string", + "description": "The value to store for the put operation" + }, + "action": { + "type": "string", + "description": "The action to perform against the bucket and key", + "enum": ["get","put","del"] + } + } + } + ] + }, + "rpc_command": { + "type": "object", + "additionalItems": false, + "allOf": [ + {"$ref":"#/definitions/standard_command"}, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "const": "rpc" + }, + "commands": { + "description": "Additional CLI commands to add", + "$ref": "#/definitions/commands" + }, + "std_filters": { + "type": "boolean", + "description": "Enables standard RPC filters like -C, -I etc", + "default": false + }, + "output_formats_flags": { + "type": "boolean", + "description": "Enable flags to adjust the output format like --json, --table etc", + "default": false + }, + "display_flag": { + "type": "boolean", + "description": "Enables the --display flag", + "default": false + }, + "batch_flags": { + "type": "boolean", + "description": "Enables the --batch and --batch-sleep flags", + "default": false + }, + "arguments": { + "type": "array", + "description": "List or arguments to accept after the command name", + "items": { + "$ref": "#/definitions/generic_argument" + } + }, + "flags": { + "type": "array", + "description": "List of flags to add to the command", + "items": { + "allOf": [ + { "$ref":"#/definitions/generic_flag"}, + { + "type": "object", + "properties": { + "reply_filter":{ + "type": "string", + "description": "Choria reply filter" + } + } + } + ] + } + } + } + } + ] + } + }, + "properties": { + "name": { + "description": "A unique name for this application", + "$ref": "#/definitions/shortname" + }, + "description": { + "$ref": "#/definitions/description" + }, + "version": { + "$ref": "#/definitions/semver" + }, + "author": { + "type": "string", + "description": "Contact details for the author", + "minLength": 1 + }, + "commands": { + "description": "A list of commands that make up the application", + "minItems": 1, + "$ref": "#/definitions/commands" + } + } +} diff --git a/internal/fs/templates.go b/internal/fs/templates.go index 76c3a1aaa..65bffde85 100644 --- a/internal/fs/templates.go +++ b/internal/fs/templates.go @@ -22,6 +22,7 @@ import ( //go:embed plugin //go:embed misc //go:embed completion +//go:embed schemas var FS embed.FS type consoleRender interface { diff --git a/providers/appbuilder/builder.go b/providers/appbuilder/builder.go new file mode 100644 index 000000000..ebad3c846 --- /dev/null +++ b/providers/appbuilder/builder.go @@ -0,0 +1,355 @@ +// Copyright (c) 2022, R.I. Pienaar and the Choria Project contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package appbuilder + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "text/template" + + "github.com/adrg/xdg" + "github.com/choria-io/go-choria/choria" + "github.com/choria-io/go-choria/internal/fs" + "github.com/choria-io/go-choria/internal/util" + "github.com/ghodss/yaml" + "github.com/sirupsen/logrus" + "github.com/tidwall/gjson" + "github.com/xeipuuv/gojsonschema" + "gopkg.in/alecthomas/kingpin.v2" +) + +type StandardCommand struct { + Name string `json:"name"` + Description string `json:"description"` + Aliases []string `json:"aliases"` + Type string `json:"type"` + Arguments []GenericArgument `json:"args"` + Flags []GenericFlag `json:"flags"` +} + +type Definition struct { + Name string `json:"name"` + Description string `json:"description"` + Version string `json:"version"` + Author string `json:"author"` + CDefs []json.RawMessage `json:"commands"` + + commands []command +} + +type GenericArgument struct { + Name string `json:"name"` + Description string `json:"description"` + Required bool `json:"required"` +} + +type GenericFlag struct { + Name string `json:"name"` + Description string `json:"description"` + Required bool `json:"required"` + PlaceHolder string `json:"place_holder"` +} + +type templateState struct { + Arguments interface{} + Flags interface{} + Config interface{} +} + +type command interface { + CreateCommand(app kingpinParent) (*kingpin.CmdClause, error) + SubCommands() []json.RawMessage +} + +type kingpinParent interface { + Command(name string, description string) *kingpin.CmdClause +} + +type AppBuilder struct { + ctx context.Context + def *Definition + name string + cfg map[string]interface{} + log *logrus.Entry +} + +var ( + errDefinitionNotfound = errors.New("definition not found") + appDefPattern = "%s-app.yaml" +) + +func NewAppBuilder(ctx context.Context, name string) *AppBuilder { + builder := &AppBuilder{ + ctx: ctx, + name: name, + } + + return builder +} + +func (b *AppBuilder) RunCommand() { + err := b.runCLI() + if err != nil { + fmt.Fprintf(os.Stderr, "Choria application %s: %v\n", b.name, err) + os.Exit(1) + } +} + +func (b *AppBuilder) runCLI() error { + logger := logrus.New() + b.log = logrus.NewEntry(logger) + logger.SetLevel(logrus.WarnLevel) + if os.Getenv("BUILDER_DEBUG") != "" { + logger.SetLevel(logrus.DebugLevel) + } + + var err error + + b.def, err = b.loadDefinition(b.name) + if err != nil { + return err + } + + err = b.loadConfig() + if err != nil { + return err + } + + cmd := kingpin.New(b.name, b.def.Description) + cmd.Version(b.def.Version) + cmd.Author(b.def.Author) + cmd.VersionFlag.Hidden() + + err = b.registerCommands(cmd, b.def.commands...) + if err != nil { + return err + } + + _, err = cmd.Parse(os.Args[1:]) + return err +} + +func (b *AppBuilder) registerCommands(cli kingpinParent, cmds ...command) error { + for _, c := range cmds { + cmd, err := c.CreateCommand(cli) + if err != nil { + return err + } + + subs := c.SubCommands() + if len(subs) > 0 { + for _, sub := range subs { + subCommand, err := b.createCommand(sub) + if err != nil { + return err + } + + err = b.registerCommands(cmd, subCommand) + if err != nil { + return err + } + } + } + } + + return nil +} + +func (b *AppBuilder) HasDefinition() bool { + source, _ := b.findConfigFile(fmt.Sprintf(appDefPattern, b.name), "BUILDER_APP") + if source == "" { + return false + } + + return util.FileExist(source) +} + +func (b *AppBuilder) loadDefinition(name string) (*Definition, error) { + source, err := b.findConfigFile(fmt.Sprintf(appDefPattern, name), "BUILDER_APP") + if err != nil { + return nil, errDefinitionNotfound + } + + if b.log != nil { + b.log.Infof("Loading application definition %v", source) + } + + cfg, err := os.ReadFile(source) + if err != nil { + return nil, err + } + + d := &Definition{} + cfgj, err := yaml.YAMLToJSON(cfg) + if err != nil { + return nil, err + } + + schema, err := fs.FS.ReadFile("schemas/builder.json") + if err != nil { + return nil, fmt.Errorf("could not load schema: %v", err) + } + + sloader := gojsonschema.NewBytesLoader(schema) + dloader := gojsonschema.NewBytesLoader(cfgj) + result, err := gojsonschema.Validate(sloader, dloader) + if err != nil { + return nil, fmt.Errorf("schema validation failed: %s", err) + } + + if !result.Valid() { + fmt.Printf("The Builder Application %s does not pass validation against https://choria.io/schemas/choria/builder/v1/application.json:\n\n", source) + for _, err := range result.Errors() { + fmt.Printf(" - %s\n", err) + } + + return nil, fmt.Errorf("validation failed") + + } + + err = json.Unmarshal(cfgj, d) + if err != nil { + return nil, err + } + + return d, b.createCommands(d, d.CDefs) +} + +func (b *AppBuilder) createCommands(d *Definition, defs []json.RawMessage) error { + for _, c := range defs { + cmd, err := b.createCommand(c) + if err != nil { + return err + } + + d.commands = append(d.commands, cmd) + } + + return nil +} + +func (b *AppBuilder) createCommand(def json.RawMessage) (command, error) { + t := gjson.GetBytes(def, "type") + if !t.Exists() { + return nil, fmt.Errorf("command does not have a type\n%s", string(def)) + } + + switch t.String() { + case "rpc": + return NewRPCCommand(b.ctx, def, b.cfg) + case "parent": + return NewParentCommand(def, b.cfg) + case "kv": + return NewKVCommand(b.ctx, def, b.cfg) + default: + return nil, fmt.Errorf("unknown command type %q", t.String()) + } +} + +func (b *AppBuilder) findConfigFile(name string, env string) (string, error) { + sources := []string{ + filepath.Join(xdg.ConfigHome, "choria", "builder"), + "/etc/choria/builder", + } + + cur, err := filepath.Abs(".") + if err == nil { + sources = append([]string{cur}, sources...) + } + + if b.log != nil { + b.log.Debugf("Searching for app definition %s in %v", name, sources) + } + + source := os.Getenv(env) + + if source == "" { + for _, s := range sources { + path := filepath.Join(s, name) + if choria.FileExist(path) { + source = path + break + } + } + } + + if source == "" { + return "", fmt.Errorf("could not find configuration %s in %s", name, strings.Join(sources, ", ")) + } + + return source, nil +} + +func (b *AppBuilder) loadConfig() error { + source, err := b.findConfigFile("applications.yaml", "BUILDER_CONFIG") + if err != nil { + return nil + } + + b.log.Debugf("Loading configuration file %s", source) + + cfgb, err := os.ReadFile(source) + if err != nil { + return err + } + + cfgj, err := yaml.YAMLToJSON(cfgb) + if err != nil { + return err + } + + b.cfg = map[string]interface{}{} + + return json.Unmarshal(cfgj, &b.cfg) +} + +func parseStateTemplate(body string, args interface{}, flags interface{}, cfg interface{}) (string, error) { + state := templateState{ + Arguments: args, + Flags: flags, + Config: cfg, + } + + funcs := map[string]interface{}{ + "require": func(v interface{}, reason string) (interface{}, error) { + err := errors.New("value required") + if reason != "" { + err = errors.New(reason) + } + + switch val := v.(type) { + case string: + if val == "" { + return "", err + } + default: + if v == nil { + return "", err + } + } + + return v, nil + }, + } + + temp, err := template.New("choria").Funcs(funcs).Parse(body) + if err != nil { + return "", err + } + + var b bytes.Buffer + err = temp.Execute(&b, state) + if err != nil { + return "", err + } + + return b.String(), nil +} diff --git a/providers/appbuilder/kv.go b/providers/appbuilder/kv.go new file mode 100644 index 000000000..c52286271 --- /dev/null +++ b/providers/appbuilder/kv.go @@ -0,0 +1,124 @@ +// Copyright (c) 2022, R.I. Pienaar and the Choria Project contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package appbuilder + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/choria-io/go-choria/choria" + "gopkg.in/alecthomas/kingpin.v2" +) + +type KVCommand struct { + Action string `json:"action"` + Bucket string `json:"bucket"` + Key string `json:"key"` + Value string `json:"value"` + Commands []json.RawMessage `json:"commands"` + + StandardCommand +} + +type KV struct { + Arguments map[string]*string + Flags map[string]*string + cmd *kingpin.CmdClause + def *KVCommand + cfg interface{} + ctx context.Context +} + +func NewKVCommand(ctx context.Context, j json.RawMessage, cfg interface{}) (*KV, error) { + kv := &KV{ + def: &KVCommand{}, + cfg: cfg, + ctx: ctx, + } + + err := json.Unmarshal(j, kv.def) + if err != nil { + return nil, err + } + + return kv, nil +} + +func (r *KV) SubCommands() []json.RawMessage { + return r.def.Commands +} + +func (r *KV) CreateCommand(app kingpinParent) (*kingpin.CmdClause, error) { + r.cmd = app.Command(r.def.Name, r.def.Description).Action(r.runCommand) + for _, a := range r.def.Aliases { + r.cmd.Alias(a) + } + + for _, a := range r.def.Arguments { + arg := r.cmd.Arg(a.Name, a.Description) + if a.Required { + arg.Required() + } + + r.Arguments[a.Name] = arg.String() + } + + for _, f := range r.def.Flags { + flag := r.cmd.Flag(f.Name, f.Description) + if f.Required { + flag.Required() + } + if f.PlaceHolder != "" { + flag.PlaceHolder(f.PlaceHolder) + } + r.Flags[f.Name] = flag.String() + } + + return r.cmd, nil +} + +func (r *KV) runCommand(_ *kingpin.ParseContext) error { + fw, err := choria.New(choria.UserConfig()) + if err != nil { + return err + } + + kv, err := fw.KV(r.ctx, nil, r.def.Bucket, false) + if err != nil { + return err + } + + switch r.def.Action { + case "get": + entry, err := kv.Get(r.def.Key) + if err != nil { + return err + } + fmt.Println(string(entry.Value())) + + case "put": + v, err := parseStateTemplate(r.def.Value, r.Arguments, r.Flags, r.cfg) + if err != nil { + return err + } + + rev, err := kv.PutString(r.def.Key, v) + if err != nil { + return err + } + + fmt.Printf("Wrote revision %d\n", rev) + + case "del": + err = kv.Delete(r.def.Key) + if err != nil { + return err + } + fmt.Printf("Deleted key %s\n", r.def.Key) + } + + return nil +} diff --git a/providers/appbuilder/parent.go b/providers/appbuilder/parent.go new file mode 100644 index 000000000..e653e7025 --- /dev/null +++ b/providers/appbuilder/parent.go @@ -0,0 +1,48 @@ +// Copyright (c) 2022, R.I. Pienaar and the Choria Project contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package appbuilder + +import ( + "encoding/json" + + "gopkg.in/alecthomas/kingpin.v2" +) + +type ParentCommand struct { + Commands []json.RawMessage `json:"commands"` + + StandardCommand +} + +type Parent struct { + cmd *kingpin.CmdClause + def *ParentCommand +} + +func NewParentCommand(j json.RawMessage, _ interface{}) (*Parent, error) { + parent := &Parent{ + def: &ParentCommand{}, + } + + err := json.Unmarshal(j, parent.def) + if err != nil { + return nil, err + } + + return parent, nil +} + +func (p *Parent) SubCommands() []json.RawMessage { + return p.def.Commands +} + +func (p *Parent) CreateCommand(app kingpinParent) (*kingpin.CmdClause, error) { + p.cmd = app.Command(p.def.Name, p.def.Description) + for _, a := range p.def.Aliases { + p.cmd.Alias(a) + } + + return p.cmd, nil +} diff --git a/providers/appbuilder/rpc.go b/providers/appbuilder/rpc.go new file mode 100644 index 000000000..605c2b6db --- /dev/null +++ b/providers/appbuilder/rpc.go @@ -0,0 +1,354 @@ +// Copyright (c) 2022, R.I. Pienaar and the Choria Project contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package appbuilder + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "sync" + + "github.com/choria-io/go-choria/choria" + "github.com/choria-io/go-choria/client/discovery" + "github.com/choria-io/go-choria/protocol" + "github.com/choria-io/go-choria/providers/agent/mcorpc/client" + "github.com/choria-io/go-choria/providers/agent/mcorpc/replyfmt" + "gopkg.in/alecthomas/kingpin.v2" +) + +type RPCFlag struct { + GenericFlag + ReplyFilter string `json:"reply_filter"` +} + +type RPCRequest struct { + Agent string `json:"agent"` + Action string `json:"action"` + Params map[string]string `json:"params"` +} + +type RPCCommand struct { + StandardCommand + + Commands []json.RawMessage `json:"commands"` + StandardFilter bool `json:"std_filters"` + OutputFormatsFlags bool `json:"output_formats_flags"` + DisplayFlag bool `json:"display_flag"` + BatchFlags bool `json:"batch_flags"` + Arguments []GenericArgument `json:"arguments"` + Flags []RPCFlag `json:"flags"` + Request RPCRequest `json:"request"` +} + +type RPC struct { + cmd *kingpin.CmdClause + fo *discovery.StandardOptions + def *RPCCommand + cfg interface{} + Arguments map[string]*string + Flags map[string]*string + senders bool + json bool + table bool + display string + batch int + batchSleep int + ctx context.Context +} + +func NewRPCCommand(ctx context.Context, j json.RawMessage, cfg interface{}) (*RPC, error) { + rpc := &RPC{ + Arguments: map[string]*string{}, + Flags: map[string]*string{}, + def: &RPCCommand{}, + cfg: cfg, + ctx: ctx, + } + + err := json.Unmarshal(j, rpc.def) + if err != nil { + return nil, err + } + + return rpc, nil +} + +func (r *RPC) SubCommands() []json.RawMessage { + return r.def.Commands +} + +func (r *RPC) CreateCommand(app kingpinParent) (*kingpin.CmdClause, error) { + r.cmd = app.Command(r.def.Name, r.def.Description).Action(r.runCommand) + for _, a := range r.def.Aliases { + r.cmd.Alias(a) + } + + for _, a := range r.def.Arguments { + arg := r.cmd.Arg(a.Name, a.Description) + if a.Required { + arg.Required() + } + + r.Arguments[a.Name] = arg.String() + } + + if r.def.OutputFormatsFlags { + r.cmd.Flag("senders", "List only the names of matching nodes").BoolVar(&r.senders) + r.cmd.Flag("json", "Render results as JSON").BoolVar(&r.json) + r.cmd.Flag("table", "Render results as a table").BoolVar(&r.table) + } + + if r.def.StandardFilter { + r.fo = discovery.NewStandardOptions() + r.fo.AddFilterFlags(r.cmd) + r.fo.AddFlatFileFlags(r.cmd) + r.fo.AddSelectionFlags(r.cmd) + } + + if r.def.BatchFlags { + r.cmd.Flag("batch", "Do requests in batches").PlaceHolder("SIZE").IntVar(&r.batch) + r.cmd.Flag("batch-sleep", "Sleep time between batches").PlaceHolder("SECONDS").IntVar(&r.batchSleep) + } + + if r.def.DisplayFlag { + r.cmd.Flag("display", "Display only a subset of results (ok, failed, all, none)").EnumVar(&r.display, "ok", "failed", "all", "none") + } + + for _, f := range r.def.Flags { + flag := r.cmd.Flag(f.Name, f.Description) + if f.Required { + flag.Required() + } + if f.PlaceHolder != "" { + flag.PlaceHolder(f.PlaceHolder) + } + r.Flags[f.Name] = flag.String() + } + + return r.cmd, nil +} + +func (r *RPC) runCommand(_ *kingpin.ParseContext) error { + noisy := !(r.json || r.senders) + + fw, err := choria.New(choria.UserConfig()) + if err != nil { + return err + } + + log := fw.Logger(r.def.Name) + + agent, err := client.New(fw, r.def.Request.Agent) + if err != nil { + return err + } + + err = agent.ResolveDDL(r.ctx) + if err != nil { + return err + } + + ddl := agent.DDL() + action, err := ddl.ActionInterface(r.def.Request.Action) + if err != nil { + return err + } + + cmd, inputs, opts, err := r.choriaCommand() + if err != nil { + return err + } + log.Infof(strings.Join(cmd, " ")) + + if r.fo != nil { + filter, err := r.fo.NewFilter(r.def.Request.Agent) + if err != nil { + return err + } + r.fo.SetDefaultsFromConfig(fw.Configuration()) + + opts = append(opts, client.Filter(filter)) + opts = append(opts, client.Collective(r.fo.Collective)) + } + + rpcInputs, _, err := action.ValidateAndConvertToDDLTypes(inputs) + if err != nil { + return err + } + + results := &replyfmt.RPCResults{ + Agent: r.def.Request.Agent, + Action: r.def.Request.Action, + Replies: []*replyfmt.RPCReply{}, + } + mu := sync.Mutex{} + + opts = append(opts, client.ReplyHandler(func(pr protocol.Reply, reply *client.RPCReply) { + mu.Lock() + if reply != nil { + results.Replies = append(results.Replies, &replyfmt.RPCReply{Sender: pr.SenderID(), RPCReply: reply}) + } + mu.Unlock() + })) + + if noisy { + opts = append(opts, client.DiscoveryStartCB(func() { + fmt.Printf("Discovering nodes...") + })) + opts = append(opts, client.DiscoveryEndCB(func(discovered int, limited int) error { + fmt.Printf("%d\n", limited) + fmt.Println() + return nil + })) + } + + rpcres, err := agent.Do(r.ctx, r.def.Request.Action, rpcInputs, opts...) + if err != nil { + return err + } + results.Stats = rpcres.Stats() + + switch { + case r.senders: + err = results.RenderNames(os.Stdout, r.json, false) + case r.table: + err = results.RenderTable(os.Stdout, action) + case r.json: + err = results.RenderJSON(os.Stdout, action) + default: + mode := replyfmt.DisplayDDL + switch r.display { + case "ok": + mode = replyfmt.DisplayOK + case "failed": + mode = replyfmt.DisplayFailed + case "all": + mode = replyfmt.DisplayAll + case "none": + mode = replyfmt.DisplayNone + } + + err = results.RenderTXT(os.Stdout, action, false, false, mode, fw.Configuration().Color, log) + } + if err != nil { + return err + } + + return nil + +} + +func (r *RPC) choriaCommand() (cmd []string, inputs map[string]string, opts []client.RequestOption, err error) { + var params []string + opts = []client.RequestOption{} + inputs = map[string]string{} + + for k, v := range r.def.Request.Params { + body, err := r.parseStateTemplate(v) + if err != nil { + return nil, nil, nil, err + } + if len(body) > 0 { + params = append(params, fmt.Sprintf("%s=%s", k, body)) + inputs[k] = body + } + } + + if r.senders { + params = append(params, "--senders") + } + if r.json { + params = append(params, "--json") + } + if r.table { + params = append(params, "--table") + } + if r.display != "" { + params = append(params, "--display", r.display) + } + if r.batch > 0 { + opts = append(opts, client.InBatches(r.batch, r.batchSleep)) + params = append(params, "--batch", fmt.Sprintf("%d", r.batch)) + if r.batchSleep > 0 { + params = append(params, "--batch-sleep", fmt.Sprintf("%d", r.batchSleep)) + } + + } + + if r.fo != nil { + opt := r.fo + if opt.DynamicDiscoveryTimeout { + params = append(params, "--discovery-window") + } + if opt.Collective != "" { + params = append(params, "-C", opt.Collective) + } + for _, f := range opt.AgentFilter { + params = append(params, "-A", f) + } + for _, f := range opt.ClassFilter { + params = append(params, "-C", f) + } + for _, f := range opt.FactFilter { + params = append(params, "-F", f) + } + for _, f := range opt.CombinedFilter { + params = append(params, "-W", f) + } + for _, f := range opt.IdentityFilter { + params = append(params, "-I", f) + } + for k, v := range opt.DiscoveryOptions { + params = append(params, "--do", fmt.Sprintf("%s=%s", k, v)) + } + if opt.NodesFile != "" { + params = append(params, "--nodes", opt.NodesFile) + } + if opt.CompoundFilter != "" { + params = append(params, "-S", opt.CompoundFilter) + } + + if opt.DiscoveryMethod != "" { + params = append(params, "--dm", opt.DiscoveryMethod) + } + } + + filter := "" + for _, flag := range r.def.Flags { + if *r.Flags[flag.Name] != "" { + if flag.ReplyFilter == "" { + continue + } + + if filter != "" { + return nil, nil, nil, fmt.Errorf("only one filter flag can match") + } + + body, err := r.parseStateTemplate(flag.ReplyFilter) + if err != nil { + return nil, nil, nil, err + } + + filter = body + break + } + } + + cmd = []string{"choria", "req", r.def.Request.Agent, r.def.Request.Action} + + cmd = append(cmd, params...) + if filter != "" { + opts = append(opts, client.ReplyExprFilter(filter)) + cmd = append(cmd, "--filter-replies", filter) + } + + return cmd, inputs, opts, nil +} + +func (r *RPC) parseStateTemplate(body string) (string, error) { + return parseStateTemplate(body, r.Arguments, r.Flags, r.cfg) +}