From 241b4aca37774310b88a761c115676affde45ac4 Mon Sep 17 00:00:00 2001 From: cnrdh Date: Thu, 3 Nov 2022 16:02:23 +0100 Subject: [PATCH] code drop --- config.ts | 62 +++++++++++++++++++++++ create_proxy.ts | 64 ++++++++++++++++++++++++ deno.jsonc | 6 +++ dev.ts | 4 ++ documentation.ts | 30 ++++++++++++ errors.ts | 17 +++++++ imports.json | 7 +++ mod.ts | 5 ++ readme.md | 117 ++++++++++++++++++++++++++++++++++++++++++++ storage_handlers.ts | 60 +++++++++++++++++++++++ text.ts | 44 +++++++++++++++++ types.ts | 27 ++++++++++ 12 files changed, 443 insertions(+) create mode 100644 config.ts create mode 100644 create_proxy.ts create mode 100644 deno.jsonc create mode 100644 dev.ts create mode 100644 documentation.ts create mode 100644 errors.ts create mode 100644 imports.json create mode 100644 mod.ts create mode 100644 readme.md create mode 100644 storage_handlers.ts create mode 100644 text.ts create mode 100644 types.ts diff --git a/config.ts b/config.ts new file mode 100644 index 0000000..f9d6363 --- /dev/null +++ b/config.ts @@ -0,0 +1,62 @@ +import { get, options } from "./storage_handlers.ts"; +import { documentation } from "./documentation.ts"; +import { CreateProxyOptions, StorageHandler } from "./types.ts"; + +export const safeHandlers = new Map< + string, + StorageHandler +>([ + ["GET", get], + ["HEAD", get], + ["OPTIONS", options], +]); + +export const safeMethods = new Set(["GET", "HEAD", "OPTIONS"]); +export const xmlheaders = { "content-type": "application/xml; charset=utf-8" }; + +export const corsheaders = new Headers([ + ["access-control-allow-origin", "*"], + [ + "access-control-allow-methods", + "DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT", + ], +]); +export const defaultSuffix = "core.windows.net"; + +export const buildConnectionString = ( + { account = "", key = "", suffix = defaultSuffix } = {}, +): string => + `AccountName=${account};AccountKey=${key};EndpointSuffix=${suffix};DefaultEndpointsProtocol=https;`; + +export const containerPattern = new URLPattern({ + pathname: "/:container([a-z][\\w-]+)/:path*", +}); + +const defaults = (env: Deno.Env): CreateProxyOptions => { + const account = env.get("azure_account") ?? ""; + const key = env.get("azure_key") ?? ""; + + const _cont = env.get("azure_containers"); + const containers = _cont && _cont.length > 4 + ? new Set(JSON.parse(_cont)) + : new Set(); + + const suffix = env.get("azure_suffix") ?? defaultSuffix; + + const handlers = safeHandlers; + + const fallback = documentation({ account, containers, suffix, handlers }); + + return { + account, + key, + containers, + suffix, + handlers, + fallback, + }; +}; +export function config(options = {}): CreateProxyOptions { + const defaultOptions = defaults(Deno.env); + return { ...defaultOptions, ...options }; +} diff --git a/create_proxy.ts b/create_proxy.ts new file mode 100644 index 0000000..fb18994 --- /dev/null +++ b/create_proxy.ts @@ -0,0 +1,64 @@ +import { buildConnectionString, config, containerPattern } from "./config.ts"; +import { + notAllowedFactory, + notConfigured, + unauthorizedContainer, +} from "./errors.ts"; + +import { AzureStorage } from "azure_storage_client/storage.ts"; +import { CreateProxyOptions } from "./types.ts"; + +export const createProxy = + (options?: CreateProxyOptions) => + async (request: Request): Promise => { + const { + key, + account, + containers, + auth, + handlers, + fallback, + suffix, + } = config( + options, + ); + + if (!account) { + return notConfigured(); + } + + const method = request.method.toUpperCase(); + const methods = new Set(handlers.keys()); + if (!methods.has(method)) { + return notAllowedFactory(methods)(request); + } + + const unauthorized = auth ? await auth(request) : undefined; + if (unauthorized && unauthorized?.status >= 400) { + return unauthorized; + } + + const match = containerPattern.exec(request.url); + + if (match?.pathname) { + const { container, path } = match.pathname.groups; + + if (containers && !containers.has(container)) { + return unauthorizedContainer(); + } + + const handler = handlers.get(method); + if (!handler) { + return notAllowedFactory(methods)(request); + } + const storage = new AzureStorage( + buildConnectionString({ account, key, suffix }), + ); + return handler({ request, storage, container, path }); + } + // No matching container, provide fallback or 405 + if (fallback) { + return fallback(request); + } + return notAllowedFactory(methods)(request); + }; diff --git a/deno.jsonc b/deno.jsonc new file mode 100644 index 0000000..849c713 --- /dev/null +++ b/deno.jsonc @@ -0,0 +1,6 @@ +{ + "importMap": "imports.json", + "tasks": { + "dev": "deno run --watch --unstable --allow-env --allow-net dev.ts" + } +} diff --git a/dev.ts b/dev.ts new file mode 100644 index 0000000..b7dadcf --- /dev/null +++ b/dev.ts @@ -0,0 +1,4 @@ +import { createAzureBlobProxy } from "./mod.ts"; +if (import.meta.main) { + Deno.serve(createAzureBlobProxy()); +} diff --git a/documentation.ts b/documentation.ts new file mode 100644 index 0000000..76d0469 --- /dev/null +++ b/documentation.ts @@ -0,0 +1,30 @@ +import { CreateProxyOptions } from "./types.ts"; +export const documentation = + ({ account, containers, suffix, handlers }: CreateProxyOptions) => + (_request: Request): Response => { + const methods = new Set(handlers.keys()); + + const methodsList = [...methods].map((m) => `\n
  • ${m}
    • `).join(""); + + const containerList = [...containers].map((c) => + `\n
  • ${c}
    • ` + ).join(""); + + return new Response( + ` + +

    azure_blob_proxy

    +
    +
    Account
    +
    ${account}
    +
    Suffix
    +
    ${suffix}
    +
    Methods
    +
      ${methodsList}
    +
    Containers
    +
      ${containerList}
    +
    +`, + { headers: { "content-type": "text/html; charset=utf-8" } }, + ); + }; diff --git a/errors.ts b/errors.ts new file mode 100644 index 0000000..36cfaca --- /dev/null +++ b/errors.ts @@ -0,0 +1,17 @@ +export const notAllowedFactory = (methods: Set) => (request: Request) => + new Response( + `405 ${request.method} not allowed\nOnly: [${[...methods].join(",")}]`, + { status: 405 }, + ); + +export const notConfigured = () => + new Response( + "503 Service Unavailable\nAzure account or key is missing in config\n", + { status: 503 }, + ); + +export const notImplemented = (request: Request) => + new Response(`501 ${request.method} not implemented\n`, { status: 501 }); + +export const unauthorizedContainer = () => + new Response("Unauthorized container\n", { status: 403 }); diff --git a/imports.json b/imports.json new file mode 100644 index 0000000..c1b9ba1 --- /dev/null +++ b/imports.json @@ -0,0 +1,7 @@ +{ + "imports": { + "std/": "https://deno.land/std@0.161.0/", + "azure_storage_client/": "https://deno.land/x/azure_storage_client@0.4.0/", + "azure_blob_proxy/": "./" + } +} diff --git a/mod.ts b/mod.ts new file mode 100644 index 0000000..e73353b --- /dev/null +++ b/mod.ts @@ -0,0 +1,5 @@ +export * from "./config.ts"; +export * from "./create_proxy.ts"; +export * from "./errors.ts"; +export * from "./storage_handlers.ts"; +export * from "./types.ts"; diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..fd5ebca --- /dev/null +++ b/readme.md @@ -0,0 +1,117 @@ +# azure_blob_proxy + +Azure blob storage proxy for [Deno](https://deno.land) (Deploy) + +## Use + +Start a basic, safe proxy: + +```js +import { createAzureBlobProxy } from "https://deno.land/x/azure_blob_proxy/mod.ts"; +Deno.serve(createAzureBlobProxy()); +``` + +Above code will require setting env variables `azure_account` and `azure_key`. + +## Dev + +```sh +deno task dev +``` + +Open http://localhost:9000/ and you should see: a `azure_blob_proxy` header, +followed by account name, suffix, methods, and containers configured. + +## DELETE and PUT + +By default, only safe HTTP methods (`GET`, `HEAD`, and `OPTIONS`) are +operational. + +Add support for `DELETE` and `PUT`: + +```js +import { config, createProxy, del, put } from "azure_blob_proxy/mod.ts"; + +const { handlers } = config(); // Get default handlers map +handlers.set("PUT", put); // Add (generic) PUT handler +handlers.set("DELETE", del); // Add (generic) DELETE handler +Deno.serve(createProxy(config({ handlers }))); // Inject updated handlers +``` + +## Auth + +Bring your own authorization handler (that must return `401`/`403` or +`undefined`): + +```js +import { users, authHandler } from "…"; + +Deno.serve(createAzureBlobProxy({ + auth: async (request) => await authHandler(request, users) +}); +``` + +## Adding a POST handler + +Using `POST` to create new blobs requires writing a POST storage handler. + +The POST handler must build a path from the request Essentially could be as easy +as providing the default [PUT](./storage_handlers.ts#put) handler with a `path`: + +and should return a `location` response header with the fresh new resource. + +```ts +import { put } from "azure_blob_proxy/storage_handlers.ts"; + +const { handlers } = config(); +handlers.set( + "POST", + ({ request, storage, container, path }) => + put({ request, storage, container, path: `text/${crypto.randomUUID()}` }), +); // Add POST handler +Deno.serve(createAzureBlobProxy(config({ handlers }))); +``` + +See [text.ts](./text.ts) for a fully working production ready example. + +## Config + +Use `env` variables for basic configuration, extends with a +`AzureBlobProxyOptions` config object. + +### env variables + +```sh +azure_account="" # Account name +azure_key="" # Account key (or shared access signature) +azure_containers="" # Optional. Comma-separated list of allowed containers +``` + +### config object + +```js +const options = {}; +createAzureBlobProxy(config(options)); +``` + +Available options: + +```ts +export type AzureBlobProxyOptions = { + account: string; // Azure storage account name + key?: string; // Azure account key (or SAS) + containers: Set; // Allowed containers + suffix: string; // Azure URL suffix + handlers: Map; // HTTP method handlers + fallback?: Handler; // Fallback request handler + auth?: Handler; // Brin your own authorization request handler +}; +``` + +## Dependencies + +See [imports.json][imports], but shootout to +[itte1/azure_storage_client][azure_storage_client] + +[azure_storage_client]: https://github.com/itte1/azure_storage_client +[imports]: ./imports.json diff --git a/storage_handlers.ts b/storage_handlers.ts new file mode 100644 index 0000000..f634250 --- /dev/null +++ b/storage_handlers.ts @@ -0,0 +1,60 @@ +import { StorageHandler, StorageHandlerParams } from "./types.ts"; +import { corsheaders, xmlheaders } from "./config.ts"; +import { + // formatMediaType, + // getCharset, + parseMediaType, +} from "std/media_types/mod.ts"; + +export const del: StorageHandler = ( + { storage, container, path }: StorageHandlerParams, +): Promise => storage.container(container).delete(path); + +// $ curl http://127.0.0.1:8000/container/dir/file.json --netrc-file .netrc +// $ curl http://127.0.0.1:8000/container/dir/file.json -u "$user:$secret" +export const get: StorageHandler = async ( + { request, storage, container, path }: StorageHandlerParams, +): Promise => { + const { searchParams } = new URL(request.url); + const list = searchParams.has("list") || searchParams.get("comp") === "list"; + + const response = list + ? await storage.container(container).list(path) + : await storage.container(container).get(path); + + const notFound = response.headers.get("x-ms-error-code") === "BlobNotFound"; // beware: unknown "directories" are not marked + if (notFound) { + return new Response(response.body, { status: 404, headers: xmlheaders }); + } + if (list) { + return new Response(response.body, { headers: xmlheaders }); + } + return response; + // @todo For text, json, xml: Append utf-8 as charset + // Below code works, but sometimes crashes on formayMediaType + // const [_mt, _charset] = parseMediaType(response.headers.get("content-type")); + // const charset = _charset?.length > 0 ? _charset : getCharset(_mt); + // const mediaType = formatMediaType(_mt, { charset }); + // const headers = new Headers(response.headers); + // headers.set("content-type", mediaType); + // return new Response(response.body, { headers }); +}; +export const options: StorageHandler = ( + //_params: StorageHandlerParams, +): Response => + new Response(undefined, { + status: 204, + headers: corsheaders, + }); + +// $ curl -vXPUT http://127.0.0.1:8000/container/dir/file.json --netrc-file .netrc -d@f.json -H "content-type: application/json" +export const put: StorageHandler = async ( + { request, storage, container, path }: StorageHandlerParams, +): Promise => { + const blob = await request.blob(); + if (blob.size === 0) { + return new Response("Cannot PUT 0 bytes\n", { status: 400 }); + } + const [mediaType] = parseMediaType(blob.type); + return await storage.container(container).put(path, blob, mediaType); +}; diff --git a/text.ts b/text.ts new file mode 100644 index 0000000..153e597 --- /dev/null +++ b/text.ts @@ -0,0 +1,44 @@ +// A RESTful text file API example +// $ deno run --allow-net --allow-env text.ts +import { + config, + createProxy, + del, + put, + StorageHandler, + StorageHandlerParams, +} from "azure_blob_proxy/mod.ts"; + +import { serve } from "std/http/server.ts"; + +//$ curl -vXPOST --netrc-file .netrc --data-binary @text.txt -H "content-type: text/plain" http://localhost:8000/container +export const postText: StorageHandler = async ( + { request, storage, container }: StorageHandlerParams, +): Promise => { + if (/^text\//i.test(request.headers.get("content-type") ?? "")) { + const path = `text/${crypto.randomUUID()}.txt`; + const r: Response = await put({ request, storage, container, path }); + if (!r.ok) { + return r; + } + const { headers, body, ...rest } = r; + const locationHeaders = new Headers(headers); + const location = new URL( + `/${container}${path ? `/${path}` : ""}`, + request.url, + ); + locationHeaders.set("location", location.href); + return new Response(body, { headers: locationHeaders, ...rest }); + } + return new Response("400 Bad Request\nPOST body must be text", { + status: 400, + }); +}; + +if (import.meta.main) { + const { handlers } = config(); // Get default config + handlers.set("POST", postText); // Add POST handler + handlers.set("PUT", put); // Add (generic) PUT handler + handlers.set("DELETE", del); // Add (generic) DELETE handler + serve(createProxy(config({ handlers }))); +} diff --git a/types.ts b/types.ts new file mode 100644 index 0000000..d5d17fb --- /dev/null +++ b/types.ts @@ -0,0 +1,27 @@ +import { AzureStorage } from "azure_storage_client/storage.ts"; +export type CreateProxyOptions = { + account: string; // Azure storage account name + key?: string; // Azure account key (or SAS) + containers: Set; // Allowed containers + suffix: string; // Azure URL suffix + handlers: Map; // HTTP method handlers + fallback?: Handler; // Fallback request handler + auth?: Handler; // BYO authorization request handler +}; +export type Handler = (request: Request) => Response | Promise; + +export type StorageHandlerParams = { + account?: string; + key?: string; + suffix?: string; + request: Request; + storage: AzureStorage; + container: string; + path: string; +}; + +export type ResponseType = Promise | Response; + +export type StorageHandler = ( + params: StorageHandlerParams, +) => Response | Promise;