From 0bc6cb31d0e315c21b3d1226ba2d5d89270e0d28 Mon Sep 17 00:00:00 2001 From: Pedram Hadjian Date: Thu, 30 Nov 2023 13:08:51 +0100 Subject: [PATCH] chore(toc): refactored toc - added fetch Name to TMID - added isOfficial() method to ThingModel for TMID parsing - moved Insert() function to model package as a method of TOC - Insert() relies now solely on CatalogThingModel argument - removed any path computation from TOC, relying on TMID as URL - moved reading/walking/writing filesystem to fs remote --- internal/model/id.go | 2 + internal/model/model.go | 11 +++- internal/model/toc.go | 38 +++++++++++ internal/remotes/fs.go | 88 ++++++++++++++++++++++++- internal/toc/toc.go | 141 ---------------------------------------- 5 files changed, 135 insertions(+), 145 deletions(-) delete mode 100644 internal/toc/toc.go diff --git a/internal/model/id.go b/internal/model/id.go index 5e9ae2ad..ceb47c41 100644 --- a/internal/model/id.go +++ b/internal/model/id.go @@ -17,6 +17,7 @@ var ( ) type TMID struct { + Name string OptionalPath string Author string Manufacturer string @@ -119,6 +120,7 @@ func ParseTMID(s string, official bool) (TMID, error) { } return TMID{ + Name: strings.Join(parts, "/"), OptionalPath: optPath, Author: auth, Manufacturer: manuf, diff --git a/internal/model/model.go b/internal/model/model.go index 300d683b..e979ced1 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -1,5 +1,7 @@ package model +import "github.com/web-of-things-open-source/tm-catalog-cli/internal" + // ThingModel is a model for unmarshalling a Thing Model to be // imported. It contains only the fields required to be accepted into // the catalog. @@ -10,6 +12,12 @@ type ThingModel struct { Version Version `json:"version"` } +func (tm *ThingModel) IsOfficial() bool { + compareMan := internal.ToTrimmedLower(tm.Manufacturer.Name) + compareAuthor := internal.ToTrimmedLower(tm.Author.Name) + return compareMan == compareAuthor +} + type SchemaAuthor struct { Name string `json:"schema:name" validate:"required"` } @@ -22,7 +30,8 @@ type Version struct { } type ExtendedFields struct { - Links `json:"links"` + Links `json:"links"` + // TODO: why is ID field not in ThingModel? It is also needed for importing to be moved to "original" ID string `json:"id,omitempty"` Description string `json:"description"` } diff --git a/internal/model/toc.go b/internal/model/toc.go index 34cbaaca..781adbd7 100644 --- a/internal/model/toc.go +++ b/internal/model/toc.go @@ -71,3 +71,41 @@ func (toc *TOC) FindByName(name string) *TOCEntry { } return nil } + +// Insert uses CatalogThingModel to add a version, either to an existing +// entry or as a new entry. +func (toc *TOC) Insert(ctm *CatalogThingModel) error { + tmid, err := ParseTMID(ctm.ID, ctm.IsOfficial()) + if err != nil { + return err + } + // find the right entry, or create if it doesn't exist + tocEntry := toc.FindByName(tmid.Name) + if tocEntry == nil { + tocEntry = &TOCEntry{ + Name: tmid.Name, + Manufacturer: SchemaManufacturer{Name: tmid.Manufacturer}, + Mpn: tmid.Mpn, + Author: SchemaAuthor{Name: tmid.Author}, + } + toc.Data = append(toc.Data, tocEntry) + } + // TODO: check if id already exists? + // Append version information to entry + externalID := "" + original := ctm.Links.FindLink("original") + if original != nil { + externalID = original.HRef + } + tv := TOCVersion{ + Description: ctm.Description, + TimeStamp: tmid.Version.Timestamp, + Version: Version{Model: tmid.Version.Base.String()}, + TMID: ctm.ID, + ExternalID: externalID, + Digest: tmid.Version.Hash, + Links: map[string]string{"content": tmid.String()}, + } + tocEntry.Versions = append(tocEntry.Versions, tv) + return nil +} diff --git a/internal/remotes/fs.go b/internal/remotes/fs.go index daf83c44..18148851 100644 --- a/internal/remotes/fs.go +++ b/internal/remotes/fs.go @@ -9,10 +9,10 @@ import ( "path/filepath" "sort" "strings" + "time" "github.com/web-of-things-open-source/tm-catalog-cli/internal" "github.com/web-of-things-open-source/tm-catalog-cli/internal/model" - "github.com/web-of-things-open-source/tm-catalog-cli/internal/toc" ) const defaultDirPermissions = 0775 @@ -132,7 +132,7 @@ func (f *FileRemote) Fetch(id model.TMID) ([]byte, error) { } func (f *FileRemote) CreateToC() error { - return toc.Create(f.root) + return createTOC(f.root) } func (f *FileRemote) List(filter string) (model.TOC, error) { @@ -143,7 +143,7 @@ func (f *FileRemote) List(filter string) (model.TOC, error) { log.Debug(fmt.Sprintf("Creating list with filter '%s'", filter)) } - data, err := os.ReadFile(filepath.Join(f.root, toc.TOCFilename)) + data, err := os.ReadFile(filepath.Join(f.root, TOCFilename)) if err != nil { return model.TOC{}, errors.New("No toc found. Run `create-toc` for this remote.") } @@ -230,3 +230,85 @@ func makeAbs(dir string) (string, error) { return dir, nil } } + +const TMExt = ".tm.json" +const TOCFilename = "tm-catalog.toc.json" + +func createTOC(rootPath string) error { + // Prepare data collection for logging stats + var log = slog.Default() + fileCount := 0 + start := time.Now() + + newTOC := model.TOC{ + Meta: model.TOCMeta{Created: time.Now()}, + Data: []*model.TOCEntry{}, + } + + err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + if strings.HasSuffix(info.Name(), TMExt) { + thingMeta, err := getThingMetadata(path) + if err != nil { + msg := "Failed to extract metadata from file %s with error:" + msg = fmt.Sprintf(msg, path) + log.Error(msg) + log.Error(err.Error()) + log.Error("The file will be excluded from the table of contents.") + return nil + } + err = newTOC.Insert(&thingMeta) + if err != nil { + log.Error(fmt.Sprintf("Failed to insert %s into toc:", path)) + log.Error(err.Error()) + log.Error("The file will be excluded from the table of contents.") + return nil + } + fileCount++ + } + } + return nil + }) + if err != nil { + return err + } + duration := time.Now().Sub(start) + // Ignore error as we are sure our struct does not contain channel, + // complex or function values that would throw an error. + newTOCJson, _ := json.MarshalIndent(newTOC, "", " ") + err = saveTOC(rootPath, newTOCJson) + msg := "Created table of content with %d entries in %s " + msg = fmt.Sprintf(msg, fileCount, duration.String()) + log.Info(msg) + return nil +} + +func getThingMetadata(path string) (model.CatalogThingModel, error) { + // TODO: should internal.ReadRequiredFiles be used here? + data, err := os.ReadFile(path) + if err != nil { + return model.CatalogThingModel{}, err + } + + var ctm model.CatalogThingModel + err = json.Unmarshal(data, &ctm) + if err != nil { + return model.CatalogThingModel{}, err + } + + return ctm, nil +} + +func saveTOC(rootPath string, tocBytes []byte) error { + file, err := os.Create(filepath.Join(rootPath, TOCFilename)) + if err != nil { + return err + } + defer file.Close() + + _, err = file.Write(tocBytes) + return nil +} diff --git a/internal/toc/toc.go b/internal/toc/toc.go deleted file mode 100644 index c67e72ae..00000000 --- a/internal/toc/toc.go +++ /dev/null @@ -1,141 +0,0 @@ -package toc - -import ( - "encoding/json" - "fmt" - "log/slog" - "os" - "path" - "path/filepath" - "strings" - "time" - - "github.com/web-of-things-open-source/tm-catalog-cli/internal" - "github.com/web-of-things-open-source/tm-catalog-cli/internal/model" -) - -const TMExt = ".tm.json" -const TOCFilename = "tm-catalog.toc.json" - -func Create(rootPath string) error { - // Prepare data collection for logging stats - var log = slog.Default() - fileCount := 0 - start := time.Now() - - newTOC := model.TOC{ - Meta: model.TOCMeta{Created: time.Now()}, - Data: []*model.TOCEntry{}, - } - - err := filepath.Walk(rootPath, func(absPath string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - if strings.HasSuffix(info.Name(), TMExt) { - absPath = filepath.ToSlash(absPath) - rootPath = filepath.ToSlash(rootPath) - thingMeta, err := getThingMetadata(rootPath, absPath) - if err != nil { - msg := "Failed to extract metadata from file %s with error:" - msg = fmt.Sprintf(msg, absPath) - log.Error(msg) - log.Error(err.Error()) - log.Error("The file will be excluded from the table of contents.") - return nil - } - // rootPath/relPath provided by walker, can ignore error - relPath, _ := filepath.Rel(rootPath, absPath) - //TODO: check if slashes are handled correctly (consider windows paths) - relPath = filepath.ToSlash(relPath) - err = insert(relPath, &newTOC, thingMeta) - if err != nil { - log.Error(fmt.Sprintf("Failed to insert %s into toc:", absPath)) - log.Error(err.Error()) - log.Error("The file will be excluded from the table of contents.") - return nil - } - fileCount++ - } - } - return nil - }) - if err != nil { - return err - } - duration := time.Now().Sub(start) - // Ignore error as we are sure our struct does not contain channel, - // complex or function values that would throw an error. - newTOCJson, _ := json.MarshalIndent(newTOC, "", " ") - err = saveToc(rootPath, newTOCJson) - msg := "Created table of content with %d entries in %s " - msg = fmt.Sprintf(msg, fileCount, duration.String()) - log.Info(msg) - return nil -} - -func getThingMetadata(rootPath, absPath string) (model.CatalogThingModel, error) { - // TODO: should internal.ReadRequiredFiles be used here? - data, err := os.ReadFile(absPath) - if err != nil { - return model.CatalogThingModel{}, err - } - - var ctm model.CatalogThingModel - err = json.Unmarshal(data, &ctm) - if err != nil { - return model.CatalogThingModel{}, err - } - - return ctm, nil -} - -func saveToc(rootPath string, tocBytes []byte) error { - file, err := os.Create(filepath.Join(rootPath, TOCFilename)) - if err != nil { - return err - } - defer file.Close() - - _, err = file.Write(tocBytes) - return nil -} - -func insert(relPath string, toc *model.TOC, ctm model.CatalogThingModel) error { - official := internal.ToTrimmedLower(ctm.Manufacturer.Name) == internal.ToTrimmedLower(ctm.Author.Name) - tmid, err := model.ParseTMID(ctm.ID, official) - if err != nil { - return err - } - //TODO: check if slashes are handled correctly (consider windows paths) - name := path.Dir(relPath) - tocEntry := toc.FindByName(name) - // TODO: provide copy method for CatalogThingModel in TocThing - if tocEntry == nil { - tocEntry = &model.TOCEntry{} - toc.Data = append(toc.Data, tocEntry) - tocEntry.Name = name - tocEntry.Manufacturer.Name = tmid.Manufacturer - tocEntry.Mpn = tmid.Mpn - tocEntry.Author.Name = tmid.Author - } - version := model.Version{Model: tmid.Version.Base.String()} - externalID := "" - original := ctm.Links.FindLink("original") - if original != nil { - externalID = original.HRef - } - - tv := model.TOCVersion{ - Description: ctm.Description, - TimeStamp: tmid.Version.Timestamp, - Version: version, - TMID: ctm.ID, - ExternalID: externalID, - Digest: tmid.Version.Hash, - Links: map[string]string{"content": relPath}, - } - tocEntry.Versions = append(tocEntry.Versions, tv) - return nil -}