Merge pull request #1680 from ripienaar/1665.11

(#1665) support transforming data using JQ

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/gosuri/uiprogress v0.0.1
 	github.com/guptarohit/asciigraph v0.5.5
+	github.com/itchyny/gojq v0.12.7
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/looplab/fsm v0.3.0
 	github.com/miekg/pkcs11 v1.1.1
@@ -80,6 +81,7 @@ require (
 	github.com/gosuri/uilive v0.0.4 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/itchyny/timefmt-go v0.1.3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.15.4 // indirect
 	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect

go.sum

@@ -292,6 +292,10 @@ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
 github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ=
+github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw=
+github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
+github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -741,6 +745,7 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=

internal/fs/schemas/builder.json

@@ -162,6 +162,17 @@
         }
       }
     },
+    "generic_transform": {
+      "description": "Transform the data using the GOJQ implementation of the JQ language",
+      "type": "object",
+      "required": ["query"],
+      "properties": {
+        "query": {
+          "type": "string",
+          "description": "A JQ query to pass the data through"
+        }
+      }
+    },
     "generic_argument": {
       "type": "object",
       "required": [
@@ -393,6 +404,9 @@
               "description": "Additional CLI commands to add",
               "$ref": "#/definitions/commands"
             },
+            "transform": {
+              "$ref": "#/definitions/generic_transform"
+            },
             "request": {
               "type": "object",
               "description": "Details of the RPC request",

providers/appbuilder/builder.go

@@ -68,6 +68,10 @@ type GenericFlag struct {
 	PlaceHolder string `json:"placeholder"`
 }
 
+type GenericTransform struct {
+	Query string `json:"query"`
+}
+
 type templateState struct {
 	Arguments interface{}
 	Flags     interface{}

providers/appbuilder/rpc.go

@@ -5,10 +5,12 @@
 package appbuilder
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"github.com/choria-io/go-choria/config"
+	"io"
 	"os"
 	"sync"
 	"time"
@@ -22,6 +24,7 @@ import (
 	"github.com/choria-io/go-choria/providers/agent/mcorpc/ddl/agent"
 	"github.com/choria-io/go-choria/providers/agent/mcorpc/replyfmt"
 	"github.com/gosuri/uiprogress"
+	"github.com/itchyny/gojq"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/alecthomas/kingpin.v2"
 )
@@ -56,6 +59,7 @@ type RPCCommand struct {
 	Flags                 []RPCFlag                  `json:"flags"`
 	Request               RPCRequest                 `json:"request"`
 	Filter                *discovery.StandardOptions `json:"filter"`
+	Transform             *GenericTransform          `json:"transform"`
 
 	StandardCommand
 	StandardSubCommands
@@ -75,6 +79,7 @@ type RPC struct {
 	display     string
 	batch       int
 	batchSleep  int
+	jqQuery     *gojq.Query
 	progressBar *uiprogress.Bar
 	log         *logrus.Entry
 	ctx         context.Context
@@ -96,6 +101,13 @@ func NewRPCCommand(b *AppBuilder, j json.RawMessage, log *logrus.Entry) (*RPC, e
 		return nil, err
 	}
 
+	if rpc.def.Transform != nil && rpc.def.Transform.Query != "" {
+		rpc.jqQuery, err = gojq.Parse(rpc.def.Transform.Query)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return rpc, nil
 }
 
@@ -240,7 +252,7 @@ func (r *RPC) setupFilter(fw inter.Framework) error {
 
 func (r *RPC) runCommand(_ *kingpin.ParseContext) error {
 	var (
-		noisy   = !(r.json || r.senders || r.def.NoProgress)
+		noisy   = !(r.json || r.senders || r.def.NoProgress || r.def.Transform != nil)
 		mu      = sync.Mutex{}
 		dt      time.Duration
 		targets []string
@@ -344,8 +356,47 @@ func (r *RPC) runCommand(_ *kingpin.ParseContext) error {
 
 }
 
+func (r *RPC) transformResults(w io.Writer, results *replyfmt.RPCResults, action *agent.Action) error {
+	out := bytes.NewBuffer([]byte{})
+	err := results.RenderJSON(out, action)
+	if err != nil {
+		return err
+	}
+
+	data := map[string]interface{}{}
+	err = json.Unmarshal(out.Bytes(), &data)
+	if err != nil {
+		return err
+	}
+
+	iter := r.jqQuery.RunWithContext(r.ctx, data)
+	for {
+		v, ok := iter.Next()
+		if !ok {
+			break
+		}
+
+		switch val := v.(type) {
+		case error:
+			return val
+		case string:
+			fmt.Fprintln(w, val)
+		default:
+			j, err := json.MarshalIndent(val, "", "  ")
+			if err != nil {
+				return err
+			}
+			fmt.Fprintln(w, string(j))
+		}
+	}
+
+	return nil
+}
+
 func (r *RPC) renderResults(fw inter.Framework, log *logrus.Entry, results *replyfmt.RPCResults, action *agent.Action) (err error) {
 	switch {
+	case r.jqQuery != nil:
+		err = r.transformResults(os.Stdout, results, action)
 	case r.senders:
 		err = results.RenderNames(os.Stdout, r.json, false)
 	case r.table: