Generate basic directory metadata

Signed-off-by: Juan Cruz Viotti <[email protected]>

.github/workflows/ci.yml

@@ -38,9 +38,9 @@ jobs:
 
       # Testing
       - run: make test-schemas PRESET=Release
-      - run: docker build --tag registry . --file Dockerfile.${{ matrix.edition.suffix }} --progress plain
-      - run: docker compose --file test/sandbox/compose.yaml build
-      - run: docker compose --file test/sandbox/compose.yaml up --detach --wait
+      - run: docker build --tag registry-${{ matrix.edition.suffix }} . --file Dockerfile.${{ matrix.edition.suffix }} --progress plain
+      - run: docker compose --file test/sandbox/compose-${{ matrix.edition.suffix }}.yaml build
+      - run: docker compose --file test/sandbox/compose-${{ matrix.edition.suffix }}.yaml up --detach --wait
       - run: make test-e2e-common PRESET=Release
       - run: make test-e2e-${{ matrix.edition.suffix }} PRESET=Release
-      - run: docker compose --file test/sandbox/compose.yaml down
+      - run: docker compose --file test/sandbox/compose-${{ matrix.edition.suffix }}.yaml down

Makefile

@@ -73,8 +73,13 @@ sandbox: compile
 
 .PHONY: docker
 docker:
+ifeq ($(ENTERPRISE), ON)
+	$(DOCKER) build --tag registry . --file Dockerfile.ee --progress plain
+	$(DOCKER) compose --file test/sandbox/compose-ee.yaml up --build
+else
 	$(DOCKER) build --tag registry . --file Dockerfile.ce --progress plain
-	$(DOCKER) compose --file test/sandbox/compose.yaml up --build
+	$(DOCKER) compose --file test/sandbox/compose-ce.yaml up --build
+endif
 
 .PHONY: clean
 clean: 

src/enterprise/enterprise_index.h

@@ -3,14 +3,101 @@
 
 #include <sourcemeta/jsontoolkit/json.h>
 
+#include <algorithm>  // std::sort
 #include <cstdlib>    // EXIT_SUCCESS
 #include <filesystem> // std::filesystem
+#include <fstream>    // std::ofstream
+#include <iostream>   // std::cerr
+#include <utility>    // std::move
+
+static auto trim(const std::string &input) -> std::string {
+  auto copy = input;
+  copy.erase(copy.find_last_not_of(' ') + 1);
+  copy.erase(0, copy.find_first_not_of(' '));
+  return copy;
+}
 
 namespace sourcemeta::registry::enterprise {
 
+auto generate_toc(const std::filesystem::path &directory) -> void {
+  auto entries{sourcemeta::jsontoolkit::JSON::make_array()};
+
+  for (const auto &entry : std::filesystem::directory_iterator{directory}) {
+    auto entry_json{sourcemeta::jsontoolkit::JSON::make_object()};
+    entry_json.assign("name",
+                      sourcemeta::jsontoolkit::JSON{entry.path().stem()});
+    if (entry.is_directory()) {
+      entry_json.assign("type", sourcemeta::jsontoolkit::JSON{"directory"});
+      // TODO: Read these from "pages" in the configuration
+      entry_json.assign("title", sourcemeta::jsontoolkit::JSON{nullptr});
+      entry_json.assign("description", sourcemeta::jsontoolkit::JSON{nullptr});
+      entries.push_back(std::move(entry_json));
+    } else if (entry.path().extension() == ".json") {
+      const auto schema{sourcemeta::jsontoolkit::from_file(entry.path())};
+      entry_json.assign("type", sourcemeta::jsontoolkit::JSON{"schema"});
+      if (schema.is_object() && schema.defines("title") &&
+          schema.at("title").is_string()) {
+        entry_json.assign("title", sourcemeta::jsontoolkit::JSON{
+                                       trim(schema.at("title").to_string())});
+      }
+
+      if (schema.is_object() && schema.defines("description") &&
+          schema.at("description").is_string()) {
+        entry_json.assign("description",
+                          sourcemeta::jsontoolkit::JSON{
+                              trim(schema.at("description").to_string())});
+      }
+
+      if (!entry_json.defines("title")) {
+        entry_json.assign("title", sourcemeta::jsontoolkit::JSON{nullptr});
+      }
+
+      if (!entry_json.defines("description")) {
+        entry_json.assign("description",
+                          sourcemeta::jsontoolkit::JSON{nullptr});
+      }
+
+      entries.push_back(std::move(entry_json));
+    }
+  }
+
+  std::sort(entries.as_array().begin(), entries.as_array().end(),
+            [](const auto &left, const auto &right) {
+              if (left.at("type") == right.at("type")) {
+                return left.at("name") < right.at("name");
+              }
+
+              return left.at("type") < right.at("type");
+            });
+
+  auto result{sourcemeta::jsontoolkit::JSON::make_object()};
+  result.assign("entries", std::move(entries));
+
+  const auto meta_path{directory / ".meta.json"};
+  std::cerr << "Saving into: " << meta_path.string() << "\n";
+  std::ofstream stream{meta_path};
+  assert(!stream.fail());
+  sourcemeta::jsontoolkit::prettify(result, stream);
+  stream << "\n";
+  stream.close();
+}
+
 auto attach(const sourcemeta::jsontoolkit::JSON &,
-            const std::filesystem::path &, const std::filesystem::path &)
+            const std::filesystem::path &, const std::filesystem::path &output)
     -> int {
+  std::cerr << "-- Indexing directory: " << output.string() << "\n";
+  generate_toc(output / "schemas");
+
+  for (const auto &entry :
+       std::filesystem::recursive_directory_iterator{output / "schemas"}) {
+    if (!entry.is_directory()) {
+      continue;
+    }
+
+    std::cerr << "-- Processing: " << entry.path().string() << "\n";
+    generate_toc(entry.path());
+  }
+
   return EXIT_SUCCESS;
 }
 

src/index/index.cc

@@ -50,7 +50,8 @@ static auto index(const sourcemeta::jsontoolkit::JSON &configuration,
     std::cerr << "Base URI: " << collection_base_uri.recompose() << "\n";
     for (const auto &entry :
          std::filesystem::recursive_directory_iterator{collection_path}) {
-      if (!entry.is_regular_file() || entry.path().extension() != ".json") {
+      if (!entry.is_regular_file() || entry.path().extension() != ".json" ||
+          entry.path().stem().string().starts_with(".")) {
         continue;
       }
 

src/server/resolver.h

@@ -63,7 +63,10 @@ auto resolver(const sourcemeta::jsontoolkit::URI &server_base_url,
   // TODO: Prevent a malicious client from requesting a JSON file outside
   // the schema directory by using relative paths
   const auto schema_path{path_join(schema_base_directory, uri.path().value())};
-  if (!std::filesystem::exists(schema_path)) {
+  if (!std::filesystem::exists(schema_path) ||
+      // Dot files are not permitted to be schemas, as we use them
+      // for internal purposes
+      schema_path.stem().string().starts_with('.')) {
     return std::nullopt;
   }
 

test/e2e/common/period-schema.hurl

@@ -0,0 +1,10 @@
+GET {{base}}/schemas/doc/.period.json
+HTTP 404
+Content-Type: application/json
+[Captures]
+current_request_id: header "X-Request-id"
+[Asserts]  
+jsonpath "$.code" == 404
+jsonpath "$.error" == "not-found"
+jsonpath "$.message" == "There is no schema at this URL"
+jsonpath "$.request" == "{{current_request_id}}"

test/e2e/ee/no-meta.hurl

@@ -0,0 +1,21 @@
+GET {{base}}/example/schemas/.meta.json
+HTTP 404
+Content-Type: application/json
+[Captures]
+current_request_id: header "X-Request-id"
+[Asserts]  
+jsonpath "$.code" == 404
+jsonpath "$.error" == "not-found"
+jsonpath "$.message" == "There is no schema at this URL"
+jsonpath "$.request" == "{{current_request_id}}"
+
+GET {{base}}/.meta.json
+HTTP 404
+Content-Type: application/json
+[Captures]
+current_request_id: header "X-Request-id"
+[Asserts]  
+jsonpath "$.code" == 404
+jsonpath "$.error" == "not-found"
+jsonpath "$.message" == "There is no schema at this URL"
+jsonpath "$.request" == "{{current_request_id}}"

test/sandbox/Dockerfile → test/sandbox/Dockerfile.ce

@@ -1,7 +1,7 @@
-FROM registry AS builder
+FROM registry-ce AS builder
 COPY configuration.json /app/configuration.json
 COPY schemas /app/schemas
-COPY manifest.txt /app/manifest.txt
+COPY manifest-ce.txt /app/manifest.txt
 COPY manifest-check.sh /app/manifest-check.sh
 RUN sourcemeta-registry-index /app/configuration.json /app/index
 # For basic testing purposes

test/sandbox/Dockerfile.ee

@@ -0,0 +1,13 @@
+FROM registry-ee AS builder
+COPY configuration.json /app/configuration.json
+COPY schemas /app/schemas
+COPY manifest-ee.txt /app/manifest.txt
+COPY manifest-check.sh /app/manifest-check.sh
+RUN sourcemeta-registry-index /app/configuration.json /app/index
+# For basic testing purposes
+RUN /app/manifest-check.sh /app/index /app/manifest.txt
+
+FROM registry
+COPY --from=builder /app/index /app/index
+ENTRYPOINT [ "/usr/bin/sourcemeta-registry-server" ]
+CMD [ "/app/index" ]

test/sandbox/compose.yaml → test/sandbox/compose-ce.yaml

@@ -2,5 +2,6 @@ services:
   sandbox:
     build:
       context: .
+      dockerfile: Dockerfile.ce
     ports:
       - 8000:8000

test/sandbox/compose-ee.yaml

@@ -0,0 +1,7 @@
+services:
+  sandbox:
+    build:
+      context: .
+      dockerfile: Dockerfile.ee
+    ports:
+      - 8000:8000

test/sandbox/manifest-ee.txt

@@ -0,0 +1,17 @@
+./configuration.json
+./schemas
+./schemas/.meta.json
+./schemas/doc
+./schemas/doc/.meta.json
+./schemas/doc/string-1.json
+./schemas/example
+./schemas/example/.meta.json
+./schemas/example/bundling
+./schemas/example/bundling/.meta.json
+./schemas/example/bundling/single.json
+./schemas/example/schemas
+./schemas/example/schemas/.meta.json
+./schemas/example/schemas/string.json
+./schemas/example/v2.0
+./schemas/example/v2.0/.meta.json
+./schemas/example/v2.0/schema.json

test/sandbox/schemas/doc/.period.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://example.com/doc/.period.json",
+  "type": "string"
+}