From ec910548ff05ceebec45c651d5defbd2321f0925 Mon Sep 17 00:00:00 2001 From: Lukas Lalinsky Date: Wed, 4 Dec 2024 17:10:00 +0100 Subject: [PATCH] Update index files --- pkg/publicdata/cli.go | 33 ++++++++++--- pkg/publicdata/exporter.go | 96 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 119 insertions(+), 10 deletions(-) diff --git a/pkg/publicdata/cli.go b/pkg/publicdata/cli.go index 9c14b81..98a830d 100644 --- a/pkg/publicdata/cli.go +++ b/pkg/publicdata/cli.go @@ -67,14 +67,15 @@ func (f *storageFlags) Connect(c *cli.Context) (*minio.Client, string, error) { return client, bucket, nil } -func RunExport(c *cli.Context, db *sql.DB, storage *minio.Client, bucket string) error { +func RunExport(c *cli.Context, db *sql.DB, storage *minio.Client, bucketName string) error { log.Info().Msg("Running export") - return nil + exporter := NewExporter(storage, bucketName, db) + return exporter.ExportLastDays(c.Context, 7) } func NewExportCommand() *cli.Command { - dbFlags := common.NewDatabaseCliFlags("postgres-", "ACOUSTID_EXPORT_POSTGRES_") - storageFlags := NewStorageFlags("storage-", "ACOUSTID_EXPORT_STORAGE_") + dbFlags := common.NewDatabaseCliFlags("postgres-", "ASERVER_EXPORT_POSTGRES_") + storageFlags := NewStorageFlags("storage-", "ASERVER_EXPORT_STORAGE_") cmd := &cli.Command{ Name: "export", Usage: "Export public data", @@ -86,12 +87,31 @@ func NewExportCommand() *cli.Command { } defer db.Close() - storage, bucket, err := storageFlags.Connect(c) + storage, bucketName, err := storageFlags.Connect(c) if err != nil { return err } - return RunExport(c, db, storage, bucket) + return RunExport(c, db, storage, bucketName) + }, + } + return cmd +} + +func NewUpdateIndexFilesCommand() *cli.Command { + storageFlags := NewStorageFlags("storage-", "ASERVER_EXPORT_STORAGE_") + cmd := &cli.Command{ + Name: "updateindexfiles", + Usage: "Update index files", + Flags: storageFlags.Flags(), + Action: func(c *cli.Context) error { + storage, bucketName, err := storageFlags.Connect(c) + if err != nil { + return err + } + + exporter := NewExporter(storage, bucketName, nil) + return exporter.UpdateIndexFile(c.Context, "", true) }, } return cmd @@ -103,6 +123,7 @@ func BuildCli() *cli.Command { Usage: "AcoustID public data managemenr", Subcommands: []*cli.Command{ NewExportCommand(), + NewUpdateIndexFilesCommand(), }, } } diff --git a/pkg/publicdata/exporter.go b/pkg/publicdata/exporter.go index db56503..b2d9237 100644 --- a/pkg/publicdata/exporter.go +++ b/pkg/publicdata/exporter.go @@ -1,12 +1,15 @@ package publicdata import ( + "bytes" "context" "database/sql" + "encoding/json" "strings" "time" "github.com/minio/minio-go/v7" + "github.com/rs/zerolog/log" ) type Exporter struct { @@ -23,8 +26,84 @@ func NewExporter(storage *minio.Client, bucketName string, db *sql.DB) *Exporter } } -func (e *Exporter) ExportDay(ctx context.Context, day time.Time) error { - prefix := day.Format("2006/01") + "/" + day.Format("2006-01-02") + "-" +func (e *Exporter) UpdateIndexFile(ctx context.Context, prefix string, recursive bool) error { + objects := e.storage.ListObjects(ctx, e.bucketName, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: false, + }) + + var sb strings.Builder + sb.WriteString("\n") + sb.WriteString("\n") + sb.WriteString("Index of /") + sb.WriteString(strings.TrimRight(prefix, "/")) + sb.WriteString("\n") + sb.WriteString("\n") + sb.WriteString("

Index of /") + sb.WriteString(strings.TrimRight(prefix, "/")) + sb.WriteString("

\n") + sb.WriteString("\n") + sb.WriteString("\n") + sb.WriteString("") + + reader := strings.NewReader(sb.String()) + log.Info().Msgf("Writing index.html to %s/%s", e.bucketName, prefix+"index.html") + _, err := e.storage.PutObject(ctx, e.bucketName, prefix+"index.html", reader, int64(sb.Len()), minio.PutObjectOptions{ + ContentType: "text/html", + }) + if err != nil { + return err + } + + jsonString, err := json.Marshal(files) + if err != nil { + return err + } + jsonReader := bytes.NewReader(jsonString) + log.Info().Msgf("Writing index.json to %s/%s", e.bucketName, prefix+"index.json") + _, err = e.storage.PutObject(ctx, e.bucketName, prefix+"index.json", jsonReader, int64(len(jsonString)), minio.PutObjectOptions{ + ContentType: "application/json", + }) + return err +} + +func addFolder(changedFolders map[string]struct{}, folder string) { + for { + changedFolders[folder+"/"] = struct{}{} + i := strings.LastIndexByte(folder, '/') + if i == -1 { + break + } + folder = folder[:i] + } + changedFolders[""] = struct{}{} +} + +func (e *Exporter) ExportDay(ctx context.Context, day time.Time, changedFolders map[string]struct{}) error { + folder := day.Format("2006/2006-01") + prefix := folder + "/" + day.Format("2006-01-02") + "-" + log.Info().Msgf("Exporting data for %s", day.Format("2006-01-02")) objects := e.storage.ListObjects(ctx, e.bucketName, minio.ListObjectsOptions{ Prefix: prefix, }) @@ -36,13 +115,22 @@ func (e *Exporter) ExportDay(ctx context.Context, day time.Time) error { key := strings.TrimPrefix(object.Key, prefix) foundKeys[key] = struct{}{} } + addFolder(changedFolders, folder) return nil } func (e *Exporter) ExportLastDays(ctx context.Context, numDays int) error { - today := time.Now().UTC() + changedFolders := make(map[string]struct{}) + date := time.Now().UTC() for i := 0; i < numDays; i++ { - err := e.ExportDay(ctx, today.AddDate(0, 0, -i)) + err := e.ExportDay(ctx, date, changedFolders) + if err != nil { + return err + } + date = date.AddDate(0, 0, -1) + } + for folder := range changedFolders { + err := e.UpdateIndexFile(ctx, folder, false) if err != nil { return err }