From ddd9caf198c0ff55d877f1aa8109fc026e9d776c Mon Sep 17 00:00:00 2001 From: Alex Borodin Date: Mon, 13 Nov 2023 15:35:25 +0100 Subject: [PATCH] feat: push an entire directory recursively; add flags for specifying optional path parts --- cmd/push.go | 32 ++++++++++++---- internal/commands/push.go | 70 +++++++++++++++++++++++++++++++--- internal/commands/push_test.go | 20 +++++++++- internal/util.go | 17 +++++++-- 4 files changed, 120 insertions(+), 19 deletions(-) diff --git a/cmd/push.go b/cmd/push.go index 8ba08f7d..9a663a70 100644 --- a/cmd/push.go +++ b/cmd/push.go @@ -9,16 +9,33 @@ import ( // pushCmd represents the push command var pushCmd = &cobra.Command{ - Use: "push [file.tm.json]", - Short: "Push TM to remote", - Long: `Push TM to remote`, - Args: cobra.ExactArgs(1), - Run: executePush, + Use: "push file-or-dirname [--remote=remote-name] [--with-path=optional/path] [--copy-tree]", + Short: "Push a TM or directory with TMs to remote", + Long: `Push a single ThingModel or an directory with ThingModels to remote catalog. +file-or-dirname + The name of the file or directory to push. Pushing a directory will walk the directory tree recursively and + import all found ThingModels. + +--remote, -r + Name of the target remote repository + +--opt-path, -p + Appends optional path parts to the target path (and id) of imported files, after the mandatory path structure. + +--opt-tree, -t + Use original directory tree structure below file-or-dirname as --opt-path for each found ThingModel file. + Has no effect when file-or-dirname points to a file. + Overrides --opt-path. +`, + Args: cobra.ExactArgs(1), + Run: executePush, } func init() { rootCmd.AddCommand(pushCmd) pushCmd.Flags().StringP("remote", "r", "", "use named remote instead of default") + pushCmd.Flags().StringP("opt-path", "p", "", "append optional path to mandatory target directory structure") + pushCmd.Flags().BoolP("opt-tree", "t", false, "use original directory tree as optional path for each file. Has no effect with a single file. Overrides -p") } func executePush(cmd *cobra.Command, args []string) { @@ -26,8 +43,9 @@ func executePush(cmd *cobra.Command, args []string) { log.Debug("executing push", "args", args) remoteName := cmd.Flag("remote").Value.String() - - err := commands.PushToRemote(remoteName, args[0]) + optPath := cmd.Flag("opt-path").Value.String() + optTree, _ := cmd.Flags().GetBool("opt-tree") + err := commands.PushToRemote(args[0], remoteName, optPath, optTree) if err != nil { log.Error("push failed", "error", err) os.Exit(1) diff --git a/internal/commands/push.go b/internal/commands/push.go index a63fcfbd..f152050e 100644 --- a/internal/commands/push.go +++ b/internal/commands/push.go @@ -5,7 +5,12 @@ import ( "encoding/json" "errors" "fmt" + "github.com/kennygrant/sanitize" + "io/fs" "log/slog" + "os" + "path/filepath" + "strings" "time" "github.com/buger/jsonparser" @@ -18,7 +23,9 @@ var now = time.Now const pseudoVersionTimestampFormat = "20060102150405" -func PushToRemote(remoteName string, filename string) error { +func PushToRemote(filename string, remoteName string, optPath string, optTree bool) error { + optPath = sanitizePath(optPath) + log := slog.Default() remote, err := remotes.Get(remoteName) if err != nil { @@ -26,6 +33,57 @@ func PushToRemote(remoteName string, filename string) error { return err } + abs, err := filepath.Abs(filename) + if err != nil { + log.Error("error expanding file name", "filename", filename, "error", err) + return err + } + + stat, err := os.Stat(abs) + if err != nil { + log.Error("cannot read file or directory", "filename", filename, "error", err) + return err + } + if stat.IsDir() { + return pushDirectory(abs, remote, optPath, optTree, log) + } else { + return PushFile(filename, remote, optPath, log) + } +} + +func sanitizePath(path string) string { + if path == "" { + return path + } + p := sanitize.Path(path) + p, _ = strings.CutPrefix(p, "/") + p, _ = strings.CutSuffix(p, "/") + return p +} + +func pushDirectory(absDirname string, remote remotes.Remote, optPath string, optTree bool, log *slog.Logger) error { + err := filepath.WalkDir(absDirname, func(path string, d fs.DirEntry, err error) error { + if d.IsDir() || !strings.HasSuffix(d.Name(), ".json") { + return nil + } + if err != nil { + return err + } + + if optTree { + optPath = filepath.Dir(strings.TrimPrefix(path, absDirname)) + } + + err = PushFile(path, remote, optPath, log) + + return err + }) + + return err + +} + +func PushFile(filename string, remote remotes.Remote, optPath string, log *slog.Logger) error { abs, raw, err := internal.ReadRequiredFile(filename) if err != nil { log.Error("couldn't read file", "error", err) @@ -38,7 +96,7 @@ func PushToRemote(remoteName string, filename string) error { return err } - versioned, id, err := prepareToImport(tm, raw) + versioned, id, err := prepareToImport(tm, raw, optPath) if err != nil { return err } @@ -52,7 +110,7 @@ func PushToRemote(remoteName string, filename string) error { return nil } -func prepareToImport(tm *model.ThingModel, raw []byte) ([]byte, model.TMID, error) { +func prepareToImport(tm *model.ThingModel, raw []byte, optPath string) ([]byte, model.TMID, error) { manuf := tm.Manufacturer.Name auth := tm.Author.Name if tm == nil || len(auth) == 0 || len(manuf) == 0 || len(tm.Mpn) == 0 { @@ -80,7 +138,7 @@ func prepareToImport(tm *model.ThingModel, raw []byte) ([]byte, model.TMID, erro } } - generatedId := generateNewId(tm, prepared) + generatedId := generateNewId(tm, prepared, optPath) finalId := idFromFile if !generatedId.Equals(idFromFile) { finalId = generatedId @@ -137,7 +195,7 @@ func moveIdToOriginalLink(raw []byte, id string) []byte { return raw } -func generateNewId(tm *model.ThingModel, raw []byte) model.TMID { +func generateNewId(tm *model.ThingModel, raw []byte, optPath string) model.TMID { fileForHashing := jsonparser.Delete(raw, "id") hasher := sha1.New() hasher.Write(fileForHashing) @@ -147,7 +205,7 @@ func generateNewId(tm *model.ThingModel, raw []byte) model.TMID { ver.Hash = hashStr ver.Timestamp = now().UTC().Format(pseudoVersionTimestampFormat) return model.TMID{ - OptionalPath: "", // fixme: pass it down from the command line args + OptionalPath: optPath, Author: tm.Author.Name, Manufacturer: tm.Manufacturer.Name, Mpn: tm.Mpn, diff --git a/internal/commands/push_test.go b/internal/commands/push_test.go index 4350c4cc..1e3867a8 100644 --- a/internal/commands/push_test.go +++ b/internal/commands/push_test.go @@ -2,8 +2,12 @@ package commands import ( "encoding/json" + "fmt" "github.com/stretchr/testify/assert" "github.com/web-of-things-open-source/tm-catalog-cli/internal/model" + "io/fs" + "log" + "os" "testing" "time" ) @@ -101,7 +105,19 @@ func TestGenerateNewID(t *testing.T) { Mpn: "senseall", Author: model.SchemaAuthor{"author"}, Version: model.Version{"v3.2.1"}, - }, []byte("{}")) + }, []byte("{}"), "opt/dir") - assert.Equal(t, "author/omnicorp/senseall/v3.2.1-20231110123243-bf21a9e8fbc5.tm.json", id.String()) + assert.Equal(t, "author/omnicorp/senseall/opt/dir/v3.2.1-20231110123243-bf21a9e8fbc5.tm.json", id.String()) +} + +func TestName(t *testing.T) { + root, _ := os.Getwd() + fileSystem := os.DirFS(root) + _ = fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + log.Fatal(err) + } + fmt.Println(path + " " + d.Name()) + return nil + }) } diff --git a/internal/util.go b/internal/util.go index c3a3d7cf..d6a8e69f 100644 --- a/internal/util.go +++ b/internal/util.go @@ -1,6 +1,7 @@ package internal import ( + "errors" "fmt" "log/slog" "os" @@ -12,14 +13,22 @@ import ( func ReadRequiredFile(name string) (string, []byte, error) { var log = slog.Default() - filename := name - abs, err := filepath.Abs(filename) + abs, err := filepath.Abs(name) if err != nil { - log.Error("error expanding file name", "filename", filename, "error", err) + log.Error("error expanding file name", "filename", name, "error", err) return "", nil, err } - log.Debug("importing file", "filename", abs) + log.Debug("reading file", "filename", abs) + stat, err := os.Stat(abs) + if err != nil { + log.Error("error reading file", "filename", abs, "error", err) + return "", nil, err + } + if stat.IsDir() { + err = errors.New("not a file") + return "", nil, err + } raw, err := os.ReadFile(abs) if err != nil { log.Error("error reading file", "filename", abs, "error", err)