Working version

database/sql/fingerprint/schema.sql

@@ -38,6 +38,7 @@ CREATE TABLE public.fingerprint (
     format_id integer,
     track_id integer NOT NULL,
     submission_count integer NOT NULL,
+    updated timestamp with time zone,
     CONSTRAINT fingerprint_bitrate_check CHECK ((bitrate > 0)),
     CONSTRAINT fingerprint_length_check CHECK ((length > 0))
 );
@@ -107,7 +108,8 @@ CREATE TABLE public.meta (
     album_artist character varying,
     track_no integer,
     disc_no integer,
-    year integer
+    year integer,
+    created timestamp with time zone DEFAULT now() NOT NULL
 );
 
 
@@ -129,7 +131,8 @@ CREATE TABLE public.track (
     id integer NOT NULL,
     created timestamp with time zone DEFAULT now() NOT NULL,
     gid uuid NOT NULL,
-    new_id integer
+    new_id integer,
+    updated timestamp with time zone
 );
 
 
@@ -178,7 +181,8 @@ CREATE TABLE public.track_mbid (
     mbid uuid NOT NULL,
     created timestamp with time zone DEFAULT now() NOT NULL,
     submission_count integer NOT NULL,
-    disabled boolean DEFAULT false NOT NULL
+    disabled boolean DEFAULT false NOT NULL,
+    updated timestamp with time zone
 );
 
 
@@ -202,7 +206,8 @@ CREATE TABLE public.track_meta (
     track_id integer NOT NULL,
     meta_id integer NOT NULL,
     created timestamp with time zone DEFAULT now() NOT NULL,
-    submission_count integer NOT NULL
+    submission_count integer NOT NULL,
+    updated timestamp with time zone
 );
 
 
@@ -226,7 +231,8 @@ CREATE TABLE public.track_puid (
     track_id integer NOT NULL,
     puid uuid NOT NULL,
     created timestamp with time zone DEFAULT now() NOT NULL,
-    submission_count integer NOT NULL
+    submission_count integer NOT NULL,
+    updated timestamp with time zone
 );
 
 

docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
 
   index:
@@ -11,6 +10,16 @@ services:
     ports:
       - "127.0.0.1:16379:6379"
 
+  minio:
+    image: minio/minio:latest
+    command: server --console-address ":9001" /data
+    ports:
+      - "127.0.0.1:19000:9000"
+      - "127.0.0.1:19001:9001"
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: notreallyapassword
+
   postgres:
     image: quay.io/acoustid/postgresql:master
     ports:
@@ -20,7 +29,7 @@ services:
       - ./database/sql:/mnt/acoustid/sql/
     environment:
       POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: password
+      POSTGRES_PASSWORD: notreallyapassword
       POSTGRES_HOST: localhost
       POSTGRES_PORT: 5432
       ACOUSTID_SQL_DIR: /mnt/acoustid/sql

pkg/publicdata/cli.go

@@ -14,6 +14,7 @@ type storageFlags struct {
 	Endpoint        *cli.StringFlag
 	AccessKeyId     *cli.StringFlag
 	SecretAccessKey *cli.StringFlag
+	UseSSL          *cli.BoolFlag
 }
 
 func NewStorageFlags(prefix string, envPrefix string) *storageFlags {
@@ -38,6 +39,11 @@ func NewStorageFlags(prefix string, envPrefix string) *storageFlags {
 			Usage:   "S3-compatible secret access key",
 			EnvVars: []string{envPrefix + "SECRET_ACCESS_KEY"},
 		},
+		UseSSL: &cli.BoolFlag{
+			Name:    prefix + "use-ssl",
+			Usage:   "S3-compatible https",
+			EnvVars: []string{envPrefix + "USE_SSL"},
+		},
 	}
 }
 
@@ -53,7 +59,7 @@ func (f *storageFlags) Flags() []cli.Flag {
 func (f *storageFlags) Connect(c *cli.Context) (*minio.Client, string, error) {
 	client, err := minio.New(f.Endpoint.Get(c), &minio.Options{
 		Creds:  credentials.NewStaticV4(f.AccessKeyId.Get(c), f.SecretAccessKey.Get(c), ""),
-		Secure: true,
+		Secure: f.UseSSL.Get(c),
 	})
 	if err != nil {
 		return nil, "", errors.WithMessage(err, "failed to connect to storage")

pkg/publicdata/export.go

@@ -65,7 +65,7 @@ func (ex *exporter) ExportQuery(ctx context.Context, path string, query string)
 
 	wrappedQuery := fmt.Sprintf("SELECT convert_to(json_strip_nulls(row_to_json(r))::text, 'UTF8') FROM (%s) r", query)
 	rows, err := ex.db.QueryContext(ctx, wrappedQuery)
-	if err != err {
+	if err != nil {
 		return err
 	}
 	for rows.Next() {
@@ -111,7 +111,7 @@ func (ex *exporter) ExportQuery(ctx context.Context, path string, query string)
 }
 
 func (ex *exporter) ExportDeltaFile(ctx context.Context, path string, name string, queryTmpl string, startTime, endTime time.Time) error {
-	logger := log.With().Str("table", name).Time("date", startTime).Logger()
+	logger := log.With().Str("table", name).Str("date", startTime.Format("2006-01-02")).Logger()
 
 	logger.Info().Msgf("Exporting data file")
 
@@ -129,10 +129,23 @@ func (ex *exporter) ExportDeltaFile(ctx context.Context, path string, name strin
 	return nil
 }
 
-func (ex *exporter) ExportDeltaFiles(ctx context.Context, startTime, endTime time.Time) error {
-	log.Info().Msgf("Exporting data files for %s-%s", startTime.Format("2006-01-02"), endTime.Format("2006-01-02"))
+func addFolder(changedFolders map[string]struct{}, folder string) {
+	for {
+		changedFolders[folder+"/"] = struct{}{}
+		i := strings.LastIndexByte(folder, '/')
+		if i == -1 {
+			break
+		}
+		folder = folder[:i]
+	}
+	changedFolders[""] = struct{}{}
+}
+
+func (ex *exporter) ExportDeltaFiles(ctx context.Context, startTime, endTime time.Time, changedFolders map[string]struct{}) error {
+	log.Info().Str("date", startTime.Format("2006-01-02")).Msgf("Exporting data files")
 
-	prefix := startTime.Format("2006/2006-01/2006-01-02-")
+	directory := startTime.Format("2006/2006-01")
+	prefix := directory + startTime.Format("/2006-01-02-")
 	objects := ex.storage.ListObjects(ctx, ex.bucketName, minio.ListObjectsOptions{
 		Prefix: prefix,
 	})
@@ -157,21 +170,47 @@ func (ex *exporter) ExportDeltaFiles(ctx context.Context, startTime, endTime tim
 		if err != nil {
 			return err
 		}
+		addFolder(changedFolders, directory)
 	}
+
 	return nil
 }
 
+func (ex *exporter) CreateBucket(ctx context.Context) error {
+	exists, err := ex.storage.BucketExists(ctx, ex.bucketName)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return nil
+	}
+	return ex.storage.MakeBucket(ctx, ex.bucketName, minio.MakeBucketOptions{})
+}
+
 func (ex *exporter) Run(ctx context.Context) error {
+	ex.CreateBucket(ctx)
+
+	changedFolders := make(map[string]struct{})
+
 	now := time.Now()
 	endTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
 	for i := 0; i < ex.maxDays; i++ {
 		startTime := endTime.AddDate(0, 0, -1)
-		err := ex.ExportDeltaFiles(ctx, startTime, endTime)
+		err := ex.ExportDeltaFiles(ctx, startTime, endTime, changedFolders)
 		if err != nil {
 			return err
 		}
 		endTime = startTime
 	}
+
+	idx := indexer{storage: ex.storage, bucketName: ex.bucketName}
+	for folder := range changedFolders {
+		err := idx.UpdateIndexFile(ctx, folder, false)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

scripts/dev/create-db.sh

@@ -2,15 +2,15 @@
 
 set -e
 
-if [ -n "$POSTGRES_HOST" ]
-then
-    export PGHOST="$POSTGRES_HOST"
-fi
-
-if [ -n "$POSTGRES_PORT" ]
-then
-    export PGPORT="$POSTGRES_PORT"
-fi
+#if [ -n "$POSTGRES_HOST" ]
+#then
+#    export PGHOST="$POSTGRES_HOST"
+#fi
+
+#if [ -n "$POSTGRES_PORT" ]
+#then
+#    export PGPORT="$POSTGRES_PORT"
+#fi
 
 if [ -n "$POSTGRES_USER" ]
 then