diff --git a/.gitignore b/.gitignore index 27a8546..3db35f2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /test/dist/ /statical /dist/ +/staticalize diff --git a/README.md b/README.md index d88f949..24366c1 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,28 @@ The framework agnostic static site generator. -Every language has its own static site generator, and every static site generator is made obsolete by the next static site generator that comes along to replace it every few years. Staticalize lets you hop off that hamster wheel. +Every language has its own static site generator, and every static site +generator is made obsolete by the next static site generator that comes along to +replace it every few years. Staticalize lets you hop off that hamster wheel. -It does this by providing a _general_ mechanism to convert any dynamically generated website into a static one. It doesn't care _what_ framework you use to generate your content so long as it is served over HTTP and has a [sitemap][sitemap]. It will analyze your sitemap and generate a static website for it in the output directory of your choice. All you need to provide is url of the server you want to staticalize and the base url of your production server. - -For example, if you have the sourcecode of the frontside.com website running on port `8000`, you can build a static version of the website fit to serve on `frontside.com` into the `dist/` directory with the following command: +It does this by providing a _general_ mechanism to convert any dynamically +generated website into a static one. It doesn't care _what_ framework you use to +generate your content so long as it is served over HTTP and has a +[sitemap][sitemap]. It will analyze your sitemap and generate a static website +for it in the output directory of your choice. All you need to provide is url of +the server you want to staticalize and the base url of your production server. +For example, if you have the sourcecode of the frontside.com website running on +port `8000`, you can build a static version of the website fit to serve on +`frontside.com` into the `dist/` directory with the following command: ```ts $ staticalize --site https://localhost:8000 --base-url http://frontside.com --outdir dist ``` -This will read `https://localhost:800/sitemap.xml` and download the entire website to the `dist/` directory in a format that can be served from a simple file server running at `frontside.com`. - +This will read `https://localhost:800/sitemap.xml` and download the entire +website to the `dist/` directory in a format that can be served from a simple +file server running at `frontside.com`. ### CLI diff --git a/deno.json b/deno.json index e0a8490..17e08d2 100644 --- a/deno.json +++ b/deno.json @@ -1,7 +1,7 @@ { "tasks": { "dev": "deno run --watch main.ts", - "compile": "deno compile --allow-read --allow-write --allow-env --allow-sys --allow-run --allow-net -o statical main.ts" + "compile": "deno compile --allow-read --allow-write --allow-env --allow-sys --allow-run --allow-net -o staticalize main.ts" }, "imports": { "@std/assert": "jsr:@std/assert@1", @@ -10,7 +10,7 @@ "@std/fs": "jsr:@std/fs", "@libs/xml": "jsr:@libs/xml", "deno-dom": "jsr:@b-fuze/deno-dom", - "effection": "https://deno.land/x/effection@3.0.3/mod.ts" + "effection": "npm:effection@4.0.0-alpha.4" }, "lint": { "rules": { diff --git a/deno.lock b/deno.lock index da11ff5..d576695 100644 --- a/deno.lock +++ b/deno.lock @@ -1,97 +1,96 @@ { - "version": "3", - "packages": { - "specifiers": { - "jsr:@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@0.1.47", - "jsr:@b-fuze/deno-dom@0.1.47": "jsr:@b-fuze/deno-dom@0.1.47", - "jsr:@hono/hono": "jsr:@hono/hono@4.5.8", - "jsr:@libs/typing@2": "jsr:@libs/typing@2.8.1", - "jsr:@libs/xml": "jsr:@libs/xml@5.4.13", - "jsr:@std/fs": "jsr:@std/fs@1.0.1", - "jsr:@std/path": "jsr:@std/path@1.0.2", - "jsr:@std/path@^1.0.2": "jsr:@std/path@1.0.2", - "jsr:@std/testing@^1.0.0": "jsr:@std/testing@1.0.0", - "npm:zod": "npm:zod@3.23.8", - "npm:zod-opts": "npm:zod-opts@0.1.8", - "npm:zx": "npm:zx@8.1.4" + "version": "4", + "specifiers": { + "jsr:@b-fuze/deno-dom@*": "0.1.47", + "jsr:@b-fuze/deno-dom@0.1.47": "0.1.47", + "jsr:@hono/hono@*": "4.5.8", + "jsr:@libs/typing@2": "2.8.1", + "jsr:@libs/xml@*": "5.4.13", + "jsr:@std/fs@*": "1.0.1", + "jsr:@std/path@*": "1.0.2", + "jsr:@std/path@^1.0.2": "1.0.2", + "jsr:@std/testing@1": "1.0.0", + "npm:effection@4.0.0-alpha.4": "4.0.0-alpha.4", + "npm:zod-opts@*": "0.1.8", + "npm:zod@*": "3.23.8", + "npm:zx@*": "8.1.4" + }, + "jsr": { + "@b-fuze/deno-dom@0.1.47": { + "integrity": "270a888de91329f8ce3849211ece0ad97ce1e8b9a8a774f2bed2f43c8b0ffe8e" + }, + "@hono/hono@4.5.8": { + "integrity": "60f5b4c61edae2016022d6087b4fc381f378d337f046f56e00a3a3512c7e9c16" + }, + "@libs/typing@2.8.1": { + "integrity": "08437a01ec51f74a20a5ab5d683475025f93a2ad641d2394a97e87f7b5194d78" + }, + "@libs/xml@5.4.13": { + "integrity": "995320d1ce4a29ced82233e5e46d47a880e338197bbd257a686bf9afcc3ac0e4", + "dependencies": [ + "jsr:@libs/typing" + ] + }, + "@std/fs@0.229.1": { + "integrity": "38d3fb31f0ca0a8c1118e039939188f32e291a3f7f17dc0868fec22024bdfadd" + }, + "@std/fs@1.0.1": { + "integrity": "d6914ca2c21abe591f733b31dbe6331e446815e513e2451b3b9e472daddfefcb", + "dependencies": [ + "jsr:@std/path@^1.0.2" + ] + }, + "@std/path@1.0.2": { + "integrity": "a452174603f8c620bd278a380c596437a9eef50c891c64b85812f735245d9ec7" + }, + "@std/testing@1.0.0": { + "integrity": "27cfc06392c69c2acffe54e6d0bcb5f961cf193f519255372bd4fff1481bfef8" + } + }, + "npm": { + "@types/fs-extra@11.0.4": { + "integrity": "sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==", + "dependencies": [ + "@types/jsonfile", + "@types/node@18.16.19" + ] + }, + "@types/jsonfile@6.1.4": { + "integrity": "sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==", + "dependencies": [ + "@types/node@18.16.19" + ] + }, + "@types/node@18.16.19": { + "integrity": "sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==" + }, + "@types/node@20.12.7": { + "integrity": "sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==", + "dependencies": [ + "undici-types" + ] + }, + "effection@4.0.0-alpha.4": { + "integrity": "sha512-xor8XVkRGy6pvBoZBlBoZ0NIXBLzNHSgWQn29itt+6Fy42bqMPRhbUtyxwDI4LRlkmlVhPWk60mZPflNlDTSIQ==" + }, + "undici-types@5.26.5": { + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, + "zod-opts@0.1.8": { + "integrity": "sha512-YZhdEcIL3D2W9fXCCf/UBgrBS90c8w25RTteh5GihGIZzadYr/qIFxyM2L98zHUkZ2S8MMxwn3ny8fzPNnvPlg==", + "dependencies": [ + "zod" + ] }, - "jsr": { - "@b-fuze/deno-dom@0.1.47": { - "integrity": "270a888de91329f8ce3849211ece0ad97ce1e8b9a8a774f2bed2f43c8b0ffe8e" - }, - "@hono/hono@4.5.8": { - "integrity": "60f5b4c61edae2016022d6087b4fc381f378d337f046f56e00a3a3512c7e9c16" - }, - "@libs/typing@2.8.1": { - "integrity": "08437a01ec51f74a20a5ab5d683475025f93a2ad641d2394a97e87f7b5194d78" - }, - "@libs/xml@5.4.13": { - "integrity": "995320d1ce4a29ced82233e5e46d47a880e338197bbd257a686bf9afcc3ac0e4", - "dependencies": [ - "jsr:@libs/typing@2" - ] - }, - "@std/fs@0.229.1": { - "integrity": "38d3fb31f0ca0a8c1118e039939188f32e291a3f7f17dc0868fec22024bdfadd" - }, - "@std/fs@1.0.1": { - "integrity": "d6914ca2c21abe591f733b31dbe6331e446815e513e2451b3b9e472daddfefcb", - "dependencies": [ - "jsr:@std/path@^1.0.2" - ] - }, - "@std/path@1.0.2": { - "integrity": "a452174603f8c620bd278a380c596437a9eef50c891c64b85812f735245d9ec7" - }, - "@std/testing@1.0.0": { - "integrity": "27cfc06392c69c2acffe54e6d0bcb5f961cf193f519255372bd4fff1481bfef8" - } + "zod@3.23.8": { + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==" }, - "npm": { - "@types/fs-extra@11.0.4": { - "integrity": "sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==", - "dependencies": { - "@types/jsonfile": "@types/jsonfile@6.1.4", - "@types/node": "@types/node@18.16.19" - } - }, - "@types/jsonfile@6.1.4": { - "integrity": "sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==", - "dependencies": { - "@types/node": "@types/node@18.16.19" - } - }, - "@types/node@18.16.19": { - "integrity": "sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==", - "dependencies": {} - }, - "@types/node@20.12.7": { - "integrity": "sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==", - "dependencies": { - "undici-types": "undici-types@5.26.5" - } - }, - "undici-types@5.26.5": { - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "dependencies": {} - }, - "zod-opts@0.1.8": { - "integrity": "sha512-YZhdEcIL3D2W9fXCCf/UBgrBS90c8w25RTteh5GihGIZzadYr/qIFxyM2L98zHUkZ2S8MMxwn3ny8fzPNnvPlg==", - "dependencies": { - "zod": "zod@3.23.8" - } - }, - "zod@3.23.8": { - "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", - "dependencies": {} - }, - "zx@8.1.4": { - "integrity": "sha512-QFDYYpnzdpRiJ3dL2102Cw26FpXpWshW4QLTGxiYfIcwdAqg084jRCkK/kuP/NOSkxOjydRwNFG81qzA5r1a6w==", - "dependencies": { - "@types/fs-extra": "@types/fs-extra@11.0.4", - "@types/node": "@types/node@20.12.7" - } - } + "zx@8.1.4": { + "integrity": "sha512-QFDYYpnzdpRiJ3dL2102Cw26FpXpWshW4QLTGxiYfIcwdAqg084jRCkK/kuP/NOSkxOjydRwNFG81qzA5r1a6w==", + "dependencies": [ + "@types/fs-extra", + "@types/node@20.12.7" + ] } }, "remote": { @@ -141,12 +140,13 @@ }, "workspace": { "dependencies": [ - "jsr:@b-fuze/deno-dom", - "jsr:@libs/xml", + "jsr:@b-fuze/deno-dom@*", + "jsr:@libs/xml@*", "jsr:@std/assert@1", "jsr:@std/cli@1", - "jsr:@std/fs", - "jsr:@std/testing@^1.0.0" + "jsr:@std/fs@*", + "jsr:@std/testing@1", + "npm:effection@4.0.0-alpha.4" ] } } diff --git a/staticalize.ts b/staticalize.ts index f718636..7422ce9 100644 --- a/staticalize.ts +++ b/staticalize.ts @@ -105,13 +105,7 @@ function useDownloader(opts: DownloaderOptions): Operation { if (source.host !== host.host) { return; } - let path = normalize(join(outdir, source.pathname)); - - if (path.endsWith("/") || !path.match(/\.\w+/)) { - path = join(path, "index.html"); - } - - let destpath = path.endsWith("/") || !path.match(/\.\w+/) ? join(path, "index.html") : path; + let path = normalize(join(outdir, source.pathname)); yield* buffer.spawn(function* () { let response = yield* call(() => @@ -119,6 +113,7 @@ function useDownloader(opts: DownloaderOptions): Operation { ); if (response.ok) { if (response.headers.get("Content-Type")?.includes("html")) { + let destpath = join(path, "index.html"); let content = yield* call(() => response.text()); let document = new DOMParser().parseFromString( content, @@ -146,6 +141,7 @@ function useDownloader(opts: DownloaderOptions): Operation { }); } else { yield* call(async () => { + let destpath = path; let destdir = dirname(destpath); await ensureDir(destdir); await Deno.writeFile(destpath, response.body!); diff --git a/task-buffer.ts b/task-buffer.ts index 681d221..61bf590 100644 --- a/task-buffer.ts +++ b/task-buffer.ts @@ -1,5 +1,4 @@ import { - action, createChannel, Err, Ok, @@ -7,19 +6,20 @@ import { Resolve, resource, Result, - sleep, spawn, + Stream, Task, useScope, + withResolvers, } from "effection"; export interface TaskBuffer extends Operation { - spawn(op: () => Operation): Operation>; + spawn(op: () => Operation): Operation>>; } export function useTaskBuffer(max: number): Operation { return resource(function* (provide) { - let input = createChannel, never>(); + let input = createChannel(); let output = createChannel, never>(); @@ -27,48 +27,49 @@ export function useTaskBuffer(max: number): Operation { let scope = yield* useScope(); - let requests = yield* input; + let requests: SpawnRequest[] = []; yield* spawn(function* () { while (true) { - if (buffer.size < max) { - let { value: request } = yield* requests.next(); - // TODO: this is a bug in Effection. - // when an error occurs, this is still running. - yield* sleep(0); - let task = scope.run(request.operation); + if (requests.length === 0) { + yield* next(input); + } else if (buffer.size < max) { + let request = requests.pop()!; + let task = yield* scope.spawn(request.operation); buffer.add(task); yield* spawn(function* () { try { - yield* output.send(Ok(yield* task)); + let result = Ok(yield* task); + buffer.delete(task); + yield* output.send(result); } catch (error) { - yield* output.send(Err(error)); - } finally { buffer.delete(task); + yield* output.send(Err(error as Error)); } }); request.resolve(task); } else { - yield* (yield* output).next(); + yield* next(output); } } }); yield* provide({ *[Symbol.iterator]() { - while (buffer.size > 0) { - for (let task of buffer.values()) { - yield* task; - } + let outputs = yield* output; + while (buffer.size > 0 || requests.length > 0) { + yield* outputs.next(); } }, - spawn: (operation) => - action(function* (resolve) { - yield* input.send({ - operation, - resolve: resolve as Resolve>, - }); - }), + *spawn(fn: () => Operation) { + let { operation, resolve } = withResolvers>(); + requests.unshift({ + operation: fn, + resolve: resolve as Resolve, + }); + yield* input.send(); + return operation; + }, }); }); } @@ -77,3 +78,10 @@ interface SpawnRequest { operation(): Operation; resolve: Resolve>; } + +function* next( + stream: Stream, +): Operation> { + let subscription = yield* stream; + return yield* subscription.next(); +} diff --git a/test/staticalize.test.ts b/test/staticalize.test.ts index 331afb1..3e8ab21 100644 --- a/test/staticalize.test.ts +++ b/test/staticalize.test.ts @@ -60,7 +60,9 @@ describe("staticalize", () => { await expect(content("test/dist/index.html")).resolves.toEqual( "

Index

", ); - await expect(content("test/dist/about/index.html")).resolves.toEqual("

About

"); + await expect(content("test/dist/about/index.html")).resolves.toEqual( + "

About

", + ); await expect(content("test/dist/contact/index.html")).resolves.toEqual( "

Contact

", ); @@ -188,7 +190,9 @@ describe("staticalize", () => { }); async function content(path: string): Promise { - await expect(exists(path)).resolves.toEqual(true); + if (!await exists(path)) { + expect("did not exist").toEqual(path); + } let bytes = await Deno.readFile(path); return new TextDecoder().decode(bytes); }