From 37d3a9c458e0f952c09097fed51f52386d48baff Mon Sep 17 00:00:00 2001 From: LTLA Date: Thu, 11 Apr 2024 08:39:38 -0700 Subject: [PATCH] Added fetch/list endpoints for remote access to the registry. This supports read access for applications outside of the shared FS. --- README.md | 20 +++++++++++++++++++ list.go | 43 ++++++++++++++++++++++++++++++++++++++++ list_test.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 27 +++++++++++++++++++++++++- 4 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 list.go create mode 100644 list_test.go diff --git a/README.md b/README.md index 9978baf..7d0713d 100644 --- a/README.md +++ b/README.md @@ -274,6 +274,26 @@ To check if a Gobbler service is active, a user should touch a file with the `re The contents of this file are ignored. On success, the asset is deleted and a JSON formatted file will be created in `responses` with the `status` property set to `SUCCESS`. +## Accessing the registry + +Most applications on the shared filesystem should be able to directly access the world-readable registry via the usual system calls. +This is the most efficient access pattern as it avoids any data transfer. + +Remote applications can obtain a listing of the registry by performing a GET request to the `/list` endpoint, +This accepts some optional query parameters: + +- `path`, a string specifying a relative path to a subdirectory within the registry. + The listing is performed within this subdirectory. + If not provided, the entire registry is listed. +- `recursive`, a boolean indicating whether to list recursively. + Defaults to false. + +The response is a JSON-encoded array of the relative paths within the registry or one of its requested subdirectories. +If `recursive=true`, all paths refer to files; otherwise, paths may refer to subdirectories, which are denoted by a `/` suffix. + +Any file of interest within the registry can then be obtained via the `/fetch/{path}` endpoint. +Once downloaded, clients should consider caching the files to reduce future data transfer. + ## Parsing logs For some actions, the Gobbler creates a log within the `..logs/` subdirectory of the registry. diff --git a/list.go b/list.go new file mode 100644 index 0000000..05959c8 --- /dev/null +++ b/list.go @@ -0,0 +1,43 @@ +package main + +import ( + "path/filepath" + "fmt" + "io/fs" +) + +func listFiles(dir string, recursive bool) ([]string, error) { + to_report := []string{} + + err := filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error { + if err != nil { + return err + } + + is_dir := info.IsDir() + if is_dir { + if recursive || dir == path { + return nil + } + } + + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + + if !recursive && is_dir { + to_report = append(to_report, rel + "/") + return fs.SkipDir + } else { + to_report = append(to_report, rel) + return nil + } + }) + + if err != nil { + return nil, fmt.Errorf("failed to obtain a directory listing; %w", err) + } + + return to_report, nil +} diff --git a/list_test.go b/list_test.go new file mode 100644 index 0000000..89cedaa --- /dev/null +++ b/list_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "testing" + "os" + "path/filepath" + "sort" +) + +func TestListFiles(t *testing.T) { + dir, err := os.MkdirTemp("", "") + if (err != nil) { + t.Fatalf("failed to create a temporary directory; %v", err) + } + + path := filepath.Join(dir, "A") + err = os.WriteFile(path, []byte(""), 0644) + if err != nil { + t.Fatalf("failed to create a mock file; %v", err) + } + + subdir := filepath.Join(dir, "sub") + err = os.Mkdir(subdir, 0755) + if err != nil { + t.Fatalf("failed to create a temporary subdirectory; %v", err) + } + + subpath := filepath.Join(subdir, "B") + err = os.WriteFile(subpath, []byte(""), 0644) + if err != nil { + t.Fatalf("failed to create a mock file; %v", err) + } + + // Checking that we pull out all the files. + all, err := listFiles(dir, true) + if (err != nil) { + t.Fatal(err) + } + + sort.Strings(all) + if len(all) != 2 || all[0] != "A" || all[1] != "sub/B" { + t.Errorf("unexpected results from the listing (%q)", all) + } + + // Checking that the directories are properly listed. + all, err = listFiles(dir, false) + if (err != nil) { + t.Fatal(err) + } + + sort.Strings(all) + if len(all) != 2 || all[0] != "A" || all[1] != "sub/" { + t.Errorf("unexpected results from the listing (%q)", all) + } +} diff --git a/main.go b/main.go index aa1f74c..b8ed784 100644 --- a/main.go +++ b/main.go @@ -62,7 +62,7 @@ func main() { } } - // Launching a watcher to pick up changes and launch jobs. + // Creating an endpoint to trigger jobs. http.HandleFunc("POST /new/{path}", func(w http.ResponseWriter, r *http.Request) { path := filepath.Base(r.PathValue("path")) log.Println("processing " + path) @@ -142,6 +142,31 @@ func main() { } }) + // Creating an endpoint to list and serve files, for remote access to the registry. + fs := http.FileServer(http.Dir(globals.Registry)) + http.Handle("GET /fetch/", http.StripPrefix("/fetch/", fs)) + + http.HandleFunc("GET /list", func(w http.ResponseWriter, r *http.Request) { + qparams := r.URL.Query() + recursive := qparams.Get("recursive") == "true" + path := qparams.Get("path") + if path == "" { + path = globals.Registry + } else if filepath.IsLocal(path) { + path = filepath.Join(globals.Registry, path) + } else { + dumpErrorResponse(w, http.StatusBadRequest, "invalid 'path'", "list request") + return + } + + all, err := listFiles(path, recursive) + if err != nil { + dumpErrorResponse(w, http.StatusInternalServerError, err.Error(), "list request") + return + } + dumpJsonResponse(w, http.StatusOK, &all, "list request") + }) + // Adding a per-day job that purges various old files. ticker := time.NewTicker(time.Hour * 24) defer ticker.Stop()