Update index files

acoustid · Dec 4, 2024 · ec91054 · ec91054
1 parent 38b431a
commit ec91054
Show file tree

Hide file tree

Showing 2 changed files with 119 additions and 10 deletions.
diff --git a/pkg/publicdata/cli.go b/pkg/publicdata/cli.go
@@ -67,14 +67,15 @@ func (f *storageFlags) Connect(c *cli.Context) (*minio.Client, string, error) {
 	return client, bucket, nil
 }
 
-func RunExport(c *cli.Context, db *sql.DB, storage *minio.Client, bucket string) error {
+func RunExport(c *cli.Context, db *sql.DB, storage *minio.Client, bucketName string) error {
 	log.Info().Msg("Running export")
-	return nil
+	exporter := NewExporter(storage, bucketName, db)
+	return exporter.ExportLastDays(c.Context, 7)
 }
 
 func NewExportCommand() *cli.Command {
-	dbFlags := common.NewDatabaseCliFlags("postgres-", "ACOUSTID_EXPORT_POSTGRES_")
-	storageFlags := NewStorageFlags("storage-", "ACOUSTID_EXPORT_STORAGE_")
+	dbFlags := common.NewDatabaseCliFlags("postgres-", "ASERVER_EXPORT_POSTGRES_")
+	storageFlags := NewStorageFlags("storage-", "ASERVER_EXPORT_STORAGE_")
 	cmd := &cli.Command{
 		Name:  "export",
 		Usage: "Export public data",
@@ -86,12 +87,31 @@ func NewExportCommand() *cli.Command {
 			}
 			defer db.Close()
 
-			storage, bucket, err := storageFlags.Connect(c)
+			storage, bucketName, err := storageFlags.Connect(c)
 			if err != nil {
 				return err
 			}
 
-			return RunExport(c, db, storage, bucket)
+			return RunExport(c, db, storage, bucketName)
+		},
+	}
+	return cmd
+}
+
+func NewUpdateIndexFilesCommand() *cli.Command {
+	storageFlags := NewStorageFlags("storage-", "ASERVER_EXPORT_STORAGE_")
+	cmd := &cli.Command{
+		Name:  "updateindexfiles",
+		Usage: "Update index files",
+		Flags: storageFlags.Flags(),
+		Action: func(c *cli.Context) error {
+			storage, bucketName, err := storageFlags.Connect(c)
+			if err != nil {
+				return err
+			}
+
+			exporter := NewExporter(storage, bucketName, nil)
+			return exporter.UpdateIndexFile(c.Context, "", true)
 		},
 	}
 	return cmd
@@ -103,6 +123,7 @@ func BuildCli() *cli.Command {
 		Usage: "AcoustID public data managemenr",
 		Subcommands: []*cli.Command{
 			NewExportCommand(),
+			NewUpdateIndexFilesCommand(),
 		},
 	}
 }
diff --git a/pkg/publicdata/exporter.go b/pkg/publicdata/exporter.go
@@ -1,12 +1,15 @@
 package publicdata
 
 import (
+	"bytes"
 	"context"
 	"database/sql"
+	"encoding/json"
 	"strings"
 	"time"
 
 	"github.com/minio/minio-go/v7"
+	"github.com/rs/zerolog/log"
 )
 
 type Exporter struct {
@@ -23,8 +26,84 @@ func NewExporter(storage *minio.Client, bucketName string, db *sql.DB) *Exporter
 	}
 }
 
-func (e *Exporter) ExportDay(ctx context.Context, day time.Time) error {
-	prefix := day.Format("2006/01") + "/" + day.Format("2006-01-02") + "-"
+func (e *Exporter) UpdateIndexFile(ctx context.Context, prefix string, recursive bool) error {
+	objects := e.storage.ListObjects(ctx, e.bucketName, minio.ListObjectsOptions{
+		Prefix:    prefix,
+		Recursive: false,
+	})
+
+	var sb strings.Builder
+	sb.WriteString("<!DOCTYPE html>\n")
+	sb.WriteString("<html>\n")
+	sb.WriteString("<head><title>Index of /")
+	sb.WriteString(strings.TrimRight(prefix, "/"))
+	sb.WriteString("</title></head>\n")
+	sb.WriteString("<body>\n")
+	sb.WriteString("<h1>Index of /")
+	sb.WriteString(strings.TrimRight(prefix, "/"))
+	sb.WriteString("</h1>\n")
+	sb.WriteString("<ul>\n")
+
+	files := make([]string, 0)
+
+	for obj := range objects {
+		if obj.Key == prefix+"index.html" || obj.Key == "/" {
+			continue
+		}
+		name := strings.TrimPrefix(obj.Key, prefix)
+		sb.WriteString("<li><a href=\"")
+		sb.WriteString(name)
+		sb.WriteString("\">")
+		sb.WriteString(name)
+		sb.WriteString("</a>")
+		sb.WriteString("</li>\n")
+		if strings.HasSuffix(obj.Key, "/") && recursive {
+			e.UpdateIndexFile(ctx, obj.Key, recursive)
+		}
+		files = append(files, name)
+	}
+
+	sb.WriteString("</ul>\n")
+	sb.WriteString("</body>\n")
+	sb.WriteString("</html>")
+
+	reader := strings.NewReader(sb.String())
+	log.Info().Msgf("Writing index.html to %s/%s", e.bucketName, prefix+"index.html")
+	_, err := e.storage.PutObject(ctx, e.bucketName, prefix+"index.html", reader, int64(sb.Len()), minio.PutObjectOptions{
+		ContentType: "text/html",
+	})
+	if err != nil {
+		return err
+	}
+
+	jsonString, err := json.Marshal(files)
+	if err != nil {
+		return err
+	}
+	jsonReader := bytes.NewReader(jsonString)
+	log.Info().Msgf("Writing index.json to %s/%s", e.bucketName, prefix+"index.json")
+	_, err = e.storage.PutObject(ctx, e.bucketName, prefix+"index.json", jsonReader, int64(len(jsonString)), minio.PutObjectOptions{
+		ContentType: "application/json",
+	})
+	return err
+}
+
+func addFolder(changedFolders map[string]struct{}, folder string) {
+	for {
+		changedFolders[folder+"/"] = struct{}{}
+		i := strings.LastIndexByte(folder, '/')
+		if i == -1 {
+			break
+		}
+		folder = folder[:i]
+	}
+	changedFolders[""] = struct{}{}
+}
+
+func (e *Exporter) ExportDay(ctx context.Context, day time.Time, changedFolders map[string]struct{}) error {
+	folder := day.Format("2006/2006-01")
+	prefix := folder + "/" + day.Format("2006-01-02") + "-"
+	log.Info().Msgf("Exporting data for %s", day.Format("2006-01-02"))
 	objects := e.storage.ListObjects(ctx, e.bucketName, minio.ListObjectsOptions{
 		Prefix: prefix,
 	})
@@ -36,13 +115,22 @@ func (e *Exporter) ExportDay(ctx context.Context, day time.Time) error {
 		key := strings.TrimPrefix(object.Key, prefix)
 		foundKeys[key] = struct{}{}
 	}
+	addFolder(changedFolders, folder)
 	return nil
 }
 
 func (e *Exporter) ExportLastDays(ctx context.Context, numDays int) error {
-	today := time.Now().UTC()
+	changedFolders := make(map[string]struct{})
+	date := time.Now().UTC()
 	for i := 0; i < numDays; i++ {
-		err := e.ExportDay(ctx, today.AddDate(0, 0, -i))
+		err := e.ExportDay(ctx, date, changedFolders)
+		if err != nil {
+			return err
+		}
+		date = date.AddDate(0, 0, -1)
+	}
+	for folder := range changedFolders {
+		err := e.UpdateIndexFile(ctx, folder, false)
 		if err != nil {
 			return err
 		}