Merge pull request #7 from web-of-things-open-source/feat/walk

feat: push an entire directory recursively

cmd/push.go

@@ -9,25 +9,43 @@ import (
 
 // pushCmd represents the push command
 var pushCmd = &cobra.Command{
-	Use:   "push [file.tm.json]",
-	Short: "Push TM to remote",
-	Long:  `Push TM to remote`,
-	Args:  cobra.ExactArgs(1),
-	Run:   executePush,
+	Use:   "push file-or-dirname [--remote=remote-name] [--with-path=optional/path] [--copy-tree]",
+	Short: "Push a TM or directory with TMs to remote",
+	Long: `Push a single ThingModel or an directory with ThingModels to remote catalog.
+file-or-dirname
+	The name of the file or directory to push. Pushing a directory will walk the directory tree recursively and 
+	import all found ThingModels.
+
+--remote, -r
+	Name of the target remote repository
+
+--opt-path, -p
+	Appends optional path parts to the target path (and id) of imported files, after the mandatory path structure.
+
+--opt-tree, -t
+	Use original directory tree structure below file-or-dirname as --opt-path for each found ThingModel file.
+	Has no effect when file-or-dirname points to a file.
+	Overrides --opt-path.
+`,
+	Args: cobra.ExactArgs(1),
+	Run:  executePush,
 }
 
 func init() {
 	rootCmd.AddCommand(pushCmd)
 	pushCmd.Flags().StringP("remote", "r", "", "use named remote instead of default")
+	pushCmd.Flags().StringP("opt-path", "p", "", "append optional path to mandatory target directory structure")
+	pushCmd.Flags().BoolP("opt-tree", "t", false, "use original directory tree as optional path for each file. Has no effect with a single file. Overrides -p")
 }
 
 func executePush(cmd *cobra.Command, args []string) {
 	var log = slog.Default()
 
 	log.Debug("executing push", "args", args)
 	remoteName := cmd.Flag("remote").Value.String()
-
-	err := commands.PushToRemote(remoteName, args[0])
+	optPath := cmd.Flag("opt-path").Value.String()
+	optTree, _ := cmd.Flags().GetBool("opt-tree")
+	err := commands.PushToRemote(args[0], remoteName, optPath, optTree)
 	if err != nil {
 		log.Error("push failed", "error", err)
 		os.Exit(1)

internal/commands/push.go

@@ -5,7 +5,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/kennygrant/sanitize"
+	"io/fs"
 	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/buger/jsonparser"
@@ -18,14 +23,67 @@ var now = time.Now
 
 const pseudoVersionTimestampFormat = "20060102150405"
 
-func PushToRemote(remoteName string, filename string) error {
+func PushToRemote(filename string, remoteName string, optPath string, optTree bool) error {
+	optPath = sanitizePath(optPath)
+
 	log := slog.Default()
 	remote, err := remotes.Get(remoteName)
 	if err != nil {
 		log.Error(fmt.Sprintf("could not ìnitialize a remote instance for %s. check config", remoteName), "error", err)
 		return err
 	}
 
+	abs, err := filepath.Abs(filename)
+	if err != nil {
+		log.Error("error expanding file name", "filename", filename, "error", err)
+		return err
+	}
+
+	stat, err := os.Stat(abs)
+	if err != nil {
+		log.Error("cannot read file or directory", "filename", filename, "error", err)
+		return err
+	}
+	if stat.IsDir() {
+		return pushDirectory(abs, remote, optPath, optTree, log)
+	} else {
+		return PushFile(filename, remote, optPath, log)
+	}
+}
+
+func sanitizePath(path string) string {
+	if path == "" {
+		return path
+	}
+	p := sanitize.Path(path)
+	p, _ = strings.CutPrefix(p, "/")
+	p, _ = strings.CutSuffix(p, "/")
+	return p
+}
+
+func pushDirectory(absDirname string, remote remotes.Remote, optPath string, optTree bool, log *slog.Logger) error {
+	err := filepath.WalkDir(absDirname, func(path string, d fs.DirEntry, err error) error {
+		if d.IsDir() || !strings.HasSuffix(d.Name(), ".json") {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+
+		if optTree {
+			optPath = filepath.Dir(strings.TrimPrefix(path, absDirname))
+		}
+
+		err = PushFile(path, remote, optPath, log)
+
+		return err
+	})
+
+	return err
+
+}
+
+func PushFile(filename string, remote remotes.Remote, optPath string, log *slog.Logger) error {
 	abs, raw, err := internal.ReadRequiredFile(filename)
 	if err != nil {
 		log.Error("couldn't read file", "error", err)
@@ -38,7 +96,7 @@ func PushToRemote(remoteName string, filename string) error {
 		return err
 	}
 
-	versioned, id, err := prepareToImport(tm, raw)
+	versioned, id, err := prepareToImport(tm, raw, optPath)
 	if err != nil {
 		return err
 	}
@@ -52,7 +110,7 @@ func PushToRemote(remoteName string, filename string) error {
 	return nil
 }
 
-func prepareToImport(tm *model.ThingModel, raw []byte) ([]byte, model.TMID, error) {
+func prepareToImport(tm *model.ThingModel, raw []byte, optPath string) ([]byte, model.TMID, error) {
 	manuf := tm.Manufacturer.Name
 	auth := tm.Author.Name
 	if tm == nil || len(auth) == 0 || len(manuf) == 0 || len(tm.Mpn) == 0 {
@@ -80,7 +138,7 @@ func prepareToImport(tm *model.ThingModel, raw []byte) ([]byte, model.TMID, erro
 		}
 	}
 
-	generatedId := generateNewId(tm, prepared)
+	generatedId := generateNewId(tm, prepared, optPath)
 	finalId := idFromFile
 	if !generatedId.Equals(idFromFile) {
 		finalId = generatedId
@@ -137,7 +195,7 @@ func moveIdToOriginalLink(raw []byte, id string) []byte {
 	return raw
 }
 
-func generateNewId(tm *model.ThingModel, raw []byte) model.TMID {
+func generateNewId(tm *model.ThingModel, raw []byte, optPath string) model.TMID {
 	fileForHashing := jsonparser.Delete(raw, "id")
 	hasher := sha1.New()
 	hasher.Write(fileForHashing)
@@ -147,7 +205,7 @@ func generateNewId(tm *model.ThingModel, raw []byte) model.TMID {
 	ver.Hash = hashStr
 	ver.Timestamp = now().UTC().Format(pseudoVersionTimestampFormat)
 	return model.TMID{
-		OptionalPath: "", // fixme: pass it down from the command line args
+		OptionalPath: optPath,
 		Author:       tm.Author.Name,
 		Manufacturer: tm.Manufacturer.Name,
 		Mpn:          tm.Mpn,

internal/commands/push_test.go

@@ -2,8 +2,12 @@ package commands
 
 import (
 	"encoding/json"
+	"fmt"
 	"github.com/stretchr/testify/assert"
 	"github.com/web-of-things-open-source/tm-catalog-cli/internal/model"
+	"io/fs"
+	"log"
+	"os"
 	"testing"
 	"time"
 )
@@ -101,7 +105,19 @@ func TestGenerateNewID(t *testing.T) {
 		Mpn:          "senseall",
 		Author:       model.SchemaAuthor{"author"},
 		Version:      model.Version{"v3.2.1"},
-	}, []byte("{}"))
+	}, []byte("{}"), "opt/dir")
 
-	assert.Equal(t, "author/omnicorp/senseall/v3.2.1-20231110123243-bf21a9e8fbc5.tm.json", id.String())
+	assert.Equal(t, "author/omnicorp/senseall/opt/dir/v3.2.1-20231110123243-bf21a9e8fbc5.tm.json", id.String())
+}
+
+func TestName(t *testing.T) {
+	root, _ := os.Getwd()
+	fileSystem := os.DirFS(root)
+	_ = fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Println(path + " " + d.Name())
+		return nil
+	})
 }

internal/util.go

@@ -1,6 +1,7 @@
 package internal
 
 import (
+	"errors"
 	"fmt"
 	"log/slog"
 	"os"
@@ -12,14 +13,22 @@ import (
 func ReadRequiredFile(name string) (string, []byte, error) {
 	var log = slog.Default()
 
-	filename := name
-	abs, err := filepath.Abs(filename)
+	abs, err := filepath.Abs(name)
 	if err != nil {
-		log.Error("error expanding file name", "filename", filename, "error", err)
+		log.Error("error expanding file name", "filename", name, "error", err)
 		return "", nil, err
 	}
-	log.Debug("importing file", "filename", abs)
+	log.Debug("reading file", "filename", abs)
 
+	stat, err := os.Stat(abs)
+	if err != nil {
+		log.Error("error reading file", "filename", abs, "error", err)
+		return "", nil, err
+	}
+	if stat.IsDir() {
+		err = errors.New("not a file")
+		return "", nil, err
+	}
 	raw, err := os.ReadFile(abs)
 	if err != nil {
 		log.Error("error reading file", "filename", abs, "error", err)