wasm-pack

.github/workflows/rust.yml

@@ -17,6 +17,9 @@ jobs:
         working-directory: ./parquet-generator
 
     steps:
+      - uses: jetli/[email protected]
+        with:
+          version: 'latest'
       - uses: actions/checkout@v3
       - name: Build
         run: cargo build --verbose

.github/workflows/vitest.yml

@@ -13,4 +13,5 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/[email protected]
       - run: yarn
+      - run: yarn build
       - run: yarn test

compactor/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "compactor",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "deploy": "wrangler publish",
+    "start": "wrangler dev",
+    "build:wasm": "wasm-pack build ../parquet-generator/ --out-dir ../parquet-generator/pkg/",
+    "build": "npm run build:wasm && tsc"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "^4.20230419.0",
+    "@types/uuid": "^9.0.7",
+    "typescript": "^5.0.4",
+    "wrangler": "^3.0.0"
+  },
+  "dependencies": {
+    "parquet-schema-validator": "0.0.0",
+    "parquet-generator": "0.1.0",
+  }
+}

compactor/src/worker.ts

@@ -0,0 +1,56 @@
+import { ParquetSchema } from 'parquet-types';
+import { ValueResult, ErrorsToResponse, SafeJSONParse } from 'rerrors';
+
+export interface Env {
+  LAKESIDE_BUCKET: R2Bucket;
+}
+
+const getSchema = async (env: Env): Promise<ValueResult<ParquetSchema>> => {
+  const schema = await env.LAKESIDE_BUCKET.get(`schema/schema.json`);
+  if (!schema) {
+    return {
+      success: false,
+      errors: ['No schema'],
+    };
+  }
+
+  const schemaText = await schema?.text();
+  const schemaJSON = SafeJSONParse(schemaText);
+  if (!schemaJSON.success) {
+    return {
+      success: false,
+      errors: ['Schema is not valid JSON'],
+    };
+  }
+
+  const parseResult = ParquetSchema.safeParse(schemaJSON.value);
+  if (parseResult.success) {
+    return {
+      success: true,
+      value: parseResult.data,
+    };
+  }
+
+  return {
+    success: false,
+    errors: [JSON.stringify(parseResult.error)],
+  };
+};
+
+export default {
+  async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
+    const schemaResult = await getSchema(env);
+    if (!schemaResult.success) {
+      return ErrorsToResponse(schemaResult.errors);
+    }
+
+    try {
+      if (request.method === 'POST') {
+
+      }
+      return new Response('', { status: 405 });
+    } catch (e) {
+      return new Response('failed to process request', { status: 500 });
+    }
+  },
+};

compactor/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "es2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+    "lib": ["es2021"] /* Specify a set of bundled library declaration files that describe the target runtime environment. */,
+    "jsx": "react" /* Specify what JSX code is generated. */,
+    "module": "es2022" /* Specify what module code is generated. */,
+    "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */,
+    "types": ["@cloudflare/workers-types"] /* Specify type package names to be included without being referenced in a source file. */,
+    "allowJs": true /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */,
+    "checkJs": false /* Enable error reporting in type-checked JavaScript files. */,
+    "noEmit": true /* Disable emitting files from a compilation. */,
+    "isolatedModules": true /* Ensure that each file can be safely transpiled without relying on other imports. */,
+    "allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */,
+    "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
+    "strict": true /* Enable all strict type-checking options. */,
+    "skipLibCheck": true /* Skip type checking all .d.ts files. */
+  }
+}

compactor/wrangler.toml

@@ -0,0 +1,39 @@
+name = "compactor"
+main = "src/worker.ts"
+compatibility_date = "2023-10-02"
+
+# # KV Namespace binding - For more information: https://developers.cloudflare.com/workers/runtime-apis/kv
+# [[kv_namespaces]]
+# binding = "MY_KV_NAMESPACE"
+# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
+# # Durable Object binding - For more information: https://developers.cloudflare.com/workers/runtime-apis/durable-objects
+# [[durable_objects]]
+# binding = "MY_DURABLE_OBJECT"
+# class_name = "MyDurableObject"
+
+# # Bucket binding - For more information: https://developers.cloudflare.com/workers/runtime-apis/kv#bucket
+[[r2_buckets]]
+binding = "LAKESIDE_BUCKET"
+bucket_name = "lakeside"
+
+# # Service binding - For more information: https://developers.cloudflare.com/workers/platform/services
+# [[routes]]
+# binding = "MY_SERVICE"
+# pattern = "/api/*"
+# script = "api.js"
+
+# # Queue binding - For more information: https://developers.cloudflare.com/workers/runtime-apis/queues
+# [[queues]]
+# binding = "MY_QUEUE"
+# name = "my-queue"
+# zone_id = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"
+
+# [env.production]
+# MY_VARIABLE = "production_value"
+
+# [env.staging]
+# MY_VARIABLE = "staging_value"
+
+# [env.shared]
+# SHARED_VARIABLE = "shared_value"

package.json

@@ -16,6 +16,8 @@
   },
   "scripts": {
     "deploy": "deno run --allow-read --allow-net --allow-run scripts/deploy.ts",
-    "test": "vitest"
+    "test": "vitest",
+    "build:wasm": "wasm-pack build parquet-generator/ --out-dir parquet-generator/pkg/",
+    "build": "npm run build:wasm"
   }
 }