Merge pull request #27 from web-of-things-open-source/chore/move-toc-pkg

chore(toc): refactored toc

internal/model/id.go

@@ -17,6 +17,7 @@ var (
 )
 
 type TMID struct {
+	Name         string
 	OptionalPath string
 	Author       string
 	Manufacturer string
@@ -119,6 +120,7 @@ func ParseTMID(s string, official bool) (TMID, error) {
 	}
 
 	return TMID{
+		Name:         strings.Join(parts, "/"),
 		OptionalPath: optPath,
 		Author:       auth,
 		Manufacturer: manuf,

internal/model/model.go

@@ -1,5 +1,7 @@
 package model
 
+import "github.com/web-of-things-open-source/tm-catalog-cli/internal/utils"
+
 // ThingModel is a model for unmarshalling a Thing Model to be
 // imported. It contains only the fields required to be accepted into
 // the catalog.
@@ -10,6 +12,12 @@ type ThingModel struct {
 	Version      Version            `json:"version"`
 }
 
+func (tm *ThingModel) IsOfficial() bool {
+	compareMan := utils.ToTrimmedLower(tm.Manufacturer.Name)
+	compareAuthor := utils.ToTrimmedLower(tm.Author.Name)
+	return compareMan == compareAuthor
+}
+
 type SchemaAuthor struct {
 	Name string `json:"schema:name" validate:"required"`
 }
@@ -22,7 +30,8 @@ type Version struct {
 }
 
 type ExtendedFields struct {
-	Links       `json:"links"`
+	Links `json:"links"`
+	// TODO: why is ID field not in ThingModel? It is also needed for importing to be moved to "original"
 	ID          string `json:"id,omitempty"`
 	Description string `json:"description"`
 }

internal/model/toc.go

@@ -71,3 +71,41 @@ func (toc *TOC) FindByName(name string) *TOCEntry {
 	}
 	return nil
 }
+
+// Insert uses CatalogThingModel to add a version, either to an existing
+// entry or as a new entry.
+func (toc *TOC) Insert(ctm *CatalogThingModel) error {
+	tmid, err := ParseTMID(ctm.ID, ctm.IsOfficial())
+	if err != nil {
+		return err
+	}
+	// find the right entry, or create if it doesn't exist
+	tocEntry := toc.FindByName(tmid.Name)
+	if tocEntry == nil {
+		tocEntry = &TOCEntry{
+			Name:         tmid.Name,
+			Manufacturer: SchemaManufacturer{Name: tmid.Manufacturer},
+			Mpn:          tmid.Mpn,
+			Author:       SchemaAuthor{Name: tmid.Author},
+		}
+		toc.Data = append(toc.Data, tocEntry)
+	}
+	// TODO: check if id already exists?
+	// Append version information to entry
+	externalID := ""
+	original := ctm.Links.FindLink("original")
+	if original != nil {
+		externalID = original.HRef
+	}
+	tv := TOCVersion{
+		Description: ctm.Description,
+		TimeStamp:   tmid.Version.Timestamp,
+		Version:     Version{Model: tmid.Version.Base.String()},
+		TMID:        ctm.ID,
+		ExternalID:  externalID,
+		Digest:      tmid.Version.Hash,
+		Links:       map[string]string{"content": tmid.String()},
+	}
+	tocEntry.Versions = append(tocEntry.Versions, tv)
+	return nil
+}

internal/remotes/fs.go

@@ -9,9 +9,9 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/web-of-things-open-source/tm-catalog-cli/internal/model"
-	"github.com/web-of-things-open-source/tm-catalog-cli/internal/toc"
 	"github.com/web-of-things-open-source/tm-catalog-cli/internal/utils"
 )
 
@@ -132,7 +132,7 @@ func (f *FileRemote) Fetch(id model.TMID) ([]byte, error) {
 }
 
 func (f *FileRemote) CreateToC() error {
-	return toc.Create(f.root)
+	return createTOC(f.root)
 }
 
 func (f *FileRemote) List(filter string) (model.TOC, error) {
@@ -143,7 +143,7 @@ func (f *FileRemote) List(filter string) (model.TOC, error) {
 		log.Debug(fmt.Sprintf("Creating list with filter '%s'", filter))
 	}
 
-	data, err := os.ReadFile(filepath.Join(f.root, toc.TOCFilename))
+	data, err := os.ReadFile(filepath.Join(f.root, TOCFilename))
 	if err != nil {
 		return model.TOC{}, errors.New("No toc found. Run `create-toc` for this remote.")
 	}
@@ -230,3 +230,85 @@ func makeAbs(dir string) (string, error) {
 		return dir, nil
 	}
 }
+
+const TMExt = ".tm.json"
+const TOCFilename = "tm-catalog.toc.json"
+
+func createTOC(rootPath string) error {
+	// Prepare data collection for logging stats
+	var log = slog.Default()
+	fileCount := 0
+	start := time.Now()
+
+	newTOC := model.TOC{
+		Meta: model.TOCMeta{Created: time.Now()},
+		Data: []*model.TOCEntry{},
+	}
+
+	err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			if strings.HasSuffix(info.Name(), TMExt) {
+				thingMeta, err := getThingMetadata(path)
+				if err != nil {
+					msg := "Failed to extract metadata from file %s with error:"
+					msg = fmt.Sprintf(msg, path)
+					log.Error(msg)
+					log.Error(err.Error())
+					log.Error("The file will be excluded from the table of contents.")
+					return nil
+				}
+				err = newTOC.Insert(&thingMeta)
+				if err != nil {
+					log.Error(fmt.Sprintf("Failed to insert %s into toc:", path))
+					log.Error(err.Error())
+					log.Error("The file will be excluded from the table of contents.")
+					return nil
+				}
+				fileCount++
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	duration := time.Now().Sub(start)
+	// Ignore error as we are sure our struct does not contain channel,
+	// complex or function values that would throw an error.
+	newTOCJson, _ := json.MarshalIndent(newTOC, "", "  ")
+	err = saveTOC(rootPath, newTOCJson)
+	msg := "Created table of content with %d entries in %s "
+	msg = fmt.Sprintf(msg, fileCount, duration.String())
+	log.Info(msg)
+	return nil
+}
+
+func getThingMetadata(path string) (model.CatalogThingModel, error) {
+	// TODO: should internal.ReadRequiredFiles be used here?
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return model.CatalogThingModel{}, err
+	}
+
+	var ctm model.CatalogThingModel
+	err = json.Unmarshal(data, &ctm)
+	if err != nil {
+		return model.CatalogThingModel{}, err
+	}
+
+	return ctm, nil
+}
+
+func saveTOC(rootPath string, tocBytes []byte) error {
+	file, err := os.Create(filepath.Join(rootPath, TOCFilename))
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, err = file.Write(tocBytes)
+	return nil
+}