Added fetch/list endpoints for remote access to the registry.

This supports read access for applications outside of the shared FS.

README.md

@@ -274,6 +274,26 @@ To check if a Gobbler service is active, a user should touch a file with the `re
 The contents of this file are ignored.
 On success, the asset is deleted and a JSON formatted file will be created in `responses` with the `status` property set to `SUCCESS`.
 
+## Accessing the registry
+
+Most applications on the shared filesystem should be able to directly access the world-readable registry via the usual system calls.
+This is the most efficient access pattern as it avoids any data transfer.
+
+Remote applications can obtain a listing of the registry by performing a GET request to the `/list` endpoint,
+This accepts some optional query parameters:
+
+- `path`, a string specifying a relative path to a subdirectory within the registry.
+  The listing is performed within this subdirectory.
+  If not provided, the entire registry is listed.
+- `recursive`, a boolean indicating whether to list recursively.
+  Defaults to false.
+
+The response is a JSON-encoded array of the relative paths within the registry or one of its requested subdirectories.
+If `recursive=true`, all paths refer to files; otherwise, paths may refer to subdirectories, which are denoted by a `/` suffix.
+
+Any file of interest within the registry can then be obtained via the `/fetch/{path}` endpoint.
+Once downloaded, clients should consider caching the files to reduce future data transfer.
+
 ## Parsing logs
 
 For some actions, the Gobbler creates a log within the `..logs/` subdirectory of the registry.

list.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+    "path/filepath"
+    "fmt"
+    "io/fs"
+)
+
+func listFiles(dir string, recursive bool) ([]string, error) {
+    to_report := []string{}
+
+    err := filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
+        if err != nil {
+            return err
+        }
+
+        is_dir := info.IsDir()
+        if is_dir {
+            if recursive || dir == path {
+                return nil
+            }
+        }
+
+        rel, err := filepath.Rel(dir, path)
+        if err != nil {
+            return err
+        }
+
+        if !recursive && is_dir {
+            to_report = append(to_report, rel + "/")
+            return fs.SkipDir
+        } else {
+            to_report = append(to_report, rel)
+            return nil
+        }
+    })
+
+    if err != nil {
+        return nil, fmt.Errorf("failed to obtain a directory listing; %w", err)
+    }
+
+    return to_report, nil
+}

list_test.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+    "testing"
+    "os"
+    "path/filepath"
+    "sort"
+)
+
+func TestListFiles(t *testing.T) {
+    dir, err := os.MkdirTemp("", "")
+    if (err != nil) {
+        t.Fatalf("failed to create a temporary directory; %v", err)
+    }
+
+    path := filepath.Join(dir, "A")
+    err = os.WriteFile(path, []byte(""), 0644)
+    if err != nil {
+        t.Fatalf("failed to create a mock file; %v", err)
+    }
+
+    subdir := filepath.Join(dir, "sub")
+    err = os.Mkdir(subdir, 0755)
+    if err != nil {
+        t.Fatalf("failed to create a temporary subdirectory; %v", err)
+    }
+
+    subpath := filepath.Join(subdir, "B")
+    err = os.WriteFile(subpath, []byte(""), 0644)
+    if err != nil {
+        t.Fatalf("failed to create a mock file; %v", err)
+    }
+
+    // Checking that we pull out all the files.
+    all, err := listFiles(dir, true)
+    if (err != nil) {
+        t.Fatal(err)
+    }
+
+    sort.Strings(all)
+    if len(all) != 2 || all[0] != "A" || all[1] != "sub/B" {
+        t.Errorf("unexpected results from the listing (%q)", all)
+    }
+
+    // Checking that the directories are properly listed.
+    all, err = listFiles(dir, false)
+    if (err != nil) {
+        t.Fatal(err)
+    }
+
+    sort.Strings(all)
+    if len(all) != 2 || all[0] != "A" || all[1] != "sub/" {
+        t.Errorf("unexpected results from the listing (%q)", all)
+    }
+}

main.go

@@ -62,7 +62,7 @@ func main() {
         }
     }
 
-    // Launching a watcher to pick up changes and launch jobs.
+    // Creating an endpoint to trigger jobs.
     http.HandleFunc("POST /new/{path}", func(w http.ResponseWriter, r *http.Request) {
         path := filepath.Base(r.PathValue("path"))
         log.Println("processing " + path)
@@ -142,6 +142,31 @@ func main() {
         }
     })
 
+    // Creating an endpoint to list and serve files, for remote access to the registry.
+    fs := http.FileServer(http.Dir(globals.Registry))
+    http.Handle("GET /fetch/", http.StripPrefix("/fetch/", fs))
+
+    http.HandleFunc("GET /list", func(w http.ResponseWriter, r *http.Request) {
+        qparams := r.URL.Query()
+        recursive := qparams.Get("recursive") == "true"
+        path := qparams.Get("path")
+        if path == "" {
+            path = globals.Registry
+        } else if filepath.IsLocal(path) {
+            path = filepath.Join(globals.Registry, path)
+        } else {
+            dumpErrorResponse(w, http.StatusBadRequest, "invalid 'path'", "list request")
+            return
+        }
+
+        all, err := listFiles(path, recursive)
+        if err != nil {
+            dumpErrorResponse(w, http.StatusInternalServerError, err.Error(), "list request")
+            return
+        }
+        dumpJsonResponse(w, http.StatusOK, &all, "list request")
+    })
+
     // Adding a per-day job that purges various old files.
 	ticker := time.NewTicker(time.Hour * 24)
 	defer ticker.Stop()