Merge branch 'main' into fil/duckdb-wasm-1.29

observablehq · Oct 18, 2024 · 8bb2866 · 8bb2866
2 parents 13f892c + e2a3de6
commit 8bb2866
Show file tree

Hide file tree

Showing 16 changed files with 108 additions and 122 deletions.
diff --git a/docs/lib/sqlite.md b/docs/lib/sqlite.md
@@ -2,13 +2,13 @@
 
 [SQLite](https://sqlite.org/) is “a small, fast, self-contained, high-reliability, full-featured, SQL database engine” and “the most used database engine in the world.” Observable provides a ESM-compatible distribution of [sql.js](https://sql.js.org), a WASM-based distribution of SQLite. It is available by default as `SQLite` in Markdown, but you can import it like so:
 
-```js echo
+```js run=false
 import SQLite from "npm:@observablehq/sqlite";
 ```
 
 We also provide `SQLiteDatabaseClient`, a [`DatabaseClient`](https://observablehq.com/@observablehq/database-client-specification) implementation.
 
-```js echo
+```js run=false
 import {SQLiteDatabaseClient} from "npm:@observablehq/sqlite";
 ```
 

diff --git a/docs/telemetry.md b/docs/telemetry.md
diff --git a/docs/telemetry.md.js b/docs/telemetry.md.js
@@ -0,0 +1,32 @@
+import {readFile} from "node:fs/promises";
+
+process.stdout.write(`# Telemetry
+
+Observable Framework collects anonymous usage data to help us improve the product. This data is sent to Observable and is not shared with third parties. Telemetry data is covered by [Observable’s privacy policy](https://observablehq.com/privacy-policy).
+
+You can [opt-out of telemetry](#disabling-telemetry) by setting the \`OBSERVABLE_TELEMETRY_DISABLE\` environment variable to \`true\`.
+
+## What is collected?
+
+The following data is collected:
+
+~~~ts run=false
+${(await readFile("./src/telemetryData.d.ts", "utf-8")).trim()}
+~~~
+
+To inspect telemetry data, set the \`OBSERVABLE_TELEMETRY_DEBUG\` environment variable to \`true\`. This will print the telemetry data to stderr instead of sending it to Observable. See [\`telemetry.ts\`](https://github.com/observablehq/framework/blob/main/src/telemetry.ts) for source code.
+
+## What is not collected?
+
+We never collect identifying or sensitive information, such as environment variables, file names or paths, or file contents.
+
+## Disabling telemetry
+
+Setting the \`OBSERVABLE_TELEMETRY_DISABLE\` environment variable to \`true\` disables telemetry collection entirely. For example:
+
+~~~sh
+OBSERVABLE_TELEMETRY_DISABLE=true npm run build
+~~~
+
+Setting the \`OBSERVABLE_TELEMETRY_DEBUG\` environment variable to \`true\` also disables telemetry collection, instead printing telemetry data to stderr. Use this to inspect what telemetry data would be collected.
+`);
diff --git a/src/build.ts b/src/build.ts
@@ -255,7 +255,7 @@ export async function build(
     if (!path.endsWith(".js")) continue;
     const sourcePath = join(cacheRoot, path);
     effects.output.write(`${faint("build")} ${path} ${faint("→")} `);
-    const resolveImport = (i: string) => relativePath(path, aliases.get((i = resolvePath(path, i))) ?? i);
+    const resolveImport = (i: string) => isPathImport(i) ? relativePath(path, aliases.get((i = resolvePath(path, i))) ?? i) : i; // prettier-ignore
     await effects.writeFile(aliases.get(path)!, rewriteNpmImports(await readFile(sourcePath, "utf-8"), resolveImport));
   }
 

diff --git a/src/telemetry.ts b/src/telemetry.ts
@@ -1,49 +1,15 @@
 import {exec} from "node:child_process";
+import type {UUID} from "node:crypto";
 import {createHash, randomUUID} from "node:crypto";
 import {readFile, writeFile} from "node:fs/promises";
 import os from "node:os";
 import {join} from "node:path/posix";
 import {CliError} from "./error.js";
 import type {Logger} from "./logger.js";
 import {getObservableUiOrigin} from "./observableApiClient.js";
+import type {TelemetryData, TelemetryEnvironment, TelemetryIds, TelemetryTime} from "./telemetryData.js";
 import {link, magenta} from "./tty.js";
 
-type uuid = ReturnType<typeof randomUUID>;
-
-type TelemetryIds = {
-  session: uuid | null; // random, held in memory for the duration of the process
-  device: uuid | null; // persists to ~/.observablehq
-  project: string | null; // one-way hash of private salt + repository URL or cwd
-};
-
-type TelemetryEnvironment = {
-  version: string; // version from package.json
-  userAgent: string; // npm_config_user_agent
-  node: string; // node.js version
-  systemPlatform: string; // linux, darwin, win32, ...
-  systemRelease: string; // 20.04, 11.2.3, ...
-  systemArchitecture: string; // x64, arm64, ...
-  cpuCount: number; // number of cpu cores
-  cpuModel: string | null; // cpu model name
-  cpuSpeed: number | null; // cpu speed in MHz
-  memoryInMb: number; // truncated to mb
-  isCI: string | boolean; // inside CI heuristic, name or false
-  isDocker: boolean; // inside Docker heuristic
-  isWSL: boolean; // inside WSL heuristic
-};
-
-type TelemetryTime = {
-  now: number; // performance.now
-  timeOrigin: number; // performance.timeOrigin
-  timeZoneOffset: number; // minutes from UTC
-};
-
-type TelemetryData = {
-  event: "build" | "deploy" | "preview" | "signal" | "login";
-  step?: "start" | "finish" | "error";
-  [key: string]: unknown;
-};
-
 type TelemetryEffects = {
   logger: Logger;
   process: NodeJS.Process;
@@ -79,7 +45,7 @@ export class Telemetry {
   private endpoint: URL;
   private timeZoneOffset = new Date().getTimezoneOffset();
   private readonly _pending = new Set<Promise<unknown>>();
-  private _config: Promise<Record<string, uuid>> | undefined;
+  private _config: Promise<Record<string, UUID>> | undefined;
   private _ids: Promise<TelemetryIds> | undefined;
   private _environment: Promise<TelemetryEnvironment> | undefined;
 
@@ -142,7 +108,7 @@ export class Telemetry {
     process.on(name, signaled);
   }
 
-  private async getPersistentId(name: string, generator = randomUUID): Promise<uuid | null> {
+  private async getPersistentId(name: string, generator = randomUUID): Promise<UUID | null> {
     const {readFile, writeFile} = this.effects;
     const file = join(os.homedir(), ".observablehq");
     if (!this._config) {
@@ -213,7 +179,7 @@ export class Telemetry {
   }
 
   private async showBannerIfNeeded() {
-    let called: uuid | undefined;
+    let called: UUID | undefined;
     await this.getPersistentId("cli_telemetry_banner", () => (called = randomUUID()));
     if (called) {
       this.effects.logger.error(

diff --git a/src/telemetryData.d.ts b/src/telemetryData.d.ts
@@ -0,0 +1,44 @@
+import type {UUID} from "node:crypto";
+
+export type TelemetryIds = {
+  session: UUID | null; // random, held in memory for the duration of the process
+  device: UUID | null; // persists to ~/.observablehq
+  project: string | null; // one-way hash of private salt + repository URL or cwd
+};
+
+export type TelemetryEnvironment = {
+  version: string; // version from package.json
+  userAgent: string; // npm_config_user_agent
+  node: string; // node.js version
+  systemPlatform: string; // linux, darwin, win32, ...
+  systemRelease: string; // 20.04, 11.2.3, ...
+  systemArchitecture: string; // x64, arm64, ...
+  cpuCount: number; // number of cpu cores
+  cpuModel: string | null; // cpu model name
+  cpuSpeed: number | null; // cpu speed in MHz
+  memoryInMb: number; // truncated to mb
+  isCI: string | boolean; // inside CI heuristic, name or false
+  isDocker: boolean; // inside Docker heuristic
+  isWSL: boolean; // inside WSL heuristic
+};
+
+export type TelemetryTime = {
+  now: number; // performance.now
+  timeOrigin: number; // performance.timeOrigin
+  timeZoneOffset: number; // minutes from UTC
+};
+
+export type TelemetryData =
+  | {event: "build"; step: "start"}
+  | {event: "build"; step: "finish"; pageCount: number}
+  | {event: "deploy"; step: "start"; force: boolean | null | "build" | "deploy"}
+  | {event: "deploy"; step: "finish"}
+  | {event: "deploy"; step: "error"}
+  | {event: "deploy"; buildManifest: "found" | "missing" | "error"}
+  | {event: "preview"; step: "start"}
+  | {event: "preview"; step: "finish"}
+  | {event: "preview"; step: "error"}
+  | {event: "signal"; signal: NodeJS.Signals}
+  | {event: "login"; step: "start"}
+  | {event: "login"; step: "finish"}
+  | {event: "login"; step: "error"; code: "expired" | "consumed" | "no-key" | `unknown-${string}`};
diff --git a/test/input/build/imports/foo/foo.js b/test/input/build/imports/foo/foo.js
@@ -1,4 +1,5 @@
 import "npm:d3";
+import "npm:@example/url-import";
 import {bar} from "../bar/bar.js";
 export {top} from "/top.js";
 

diff --git a/test/javascript/module-test.ts b/test/javascript/module-test.ts
@@ -8,8 +8,8 @@ const emptyHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b
 // through the code and verifying that they consider all the relevant files.
 describe("getModuleHash(root, path)", () => {
   it("returns the transitive content hash for the specified module", () => {
-    assert.strictEqual(getModuleHash("test/input/build/imports", "foo/foo.js"), "32f934a52fa34ba1b06aa6089fe5922dc442c9bf2dcddef864bc649a39d9eace"); // prettier-ignore
-    assert.strictEqual(getModuleHash("test/input/build/imports", "bar/bar.js"), "7fe009c8bb0049d9b84d53a00b29fb172bbf07d8232d2ace5f7c6f220b23eb16"); // prettier-ignore
+    assert.strictEqual(getModuleHash("test/input/build/imports", "foo/foo.js"), "e743cc5455594df5a3bd78622594dfb7a8ddb9277957be9b9732f33a88955d82"); // prettier-ignore
+    assert.strictEqual(getModuleHash("test/input/build/imports", "bar/bar.js"), "34442bce5f38762986a81229c551723cdc3d4c1509ac14dde193555e65013d76"); // prettier-ignore
     assert.strictEqual(getModuleHash("test/input/build/imports", "top.js"), "160847a6b4890d59f8e8862911bfbe3b8066955d31f2708cafbe51945c3c57b6"); // prettier-ignore
     assert.strictEqual(getModuleHash("test/input/build/fetches", "foo/foo.js"), "3bb4a170d2f3539934168741572d4aa3cd11da649d4ca88b408edefb5c287360"); // prettier-ignore
     assert.strictEqual(getModuleHash("test/input/build/fetches", "top.js"), "6c858de52de6ff26b19508e95448288da02fac62251b7ca2710a308a0ebfd7ba"); // prettier-ignore
@@ -27,8 +27,8 @@ describe("getModuleInfo(root, path)", () => {
     assert.deepStrictEqual(redactModuleInfo("test/input/build/imports", "foo/foo.js"), {
       fileMethods: new Set(),
       files: new Set(),
-      hash: "c77c2490ea7b9a89dce7bad39973995e5158921bf8576955ae4a596c47a5a2a4",
-      globalStaticImports: new Set(["npm:d3"]),
+      hash: "17e03fbc08c28530c84ab1163901890915302d3f1d5af2c9256e3e8cab1324a9",
+      globalStaticImports: new Set(["npm:@example/url-import", "npm:d3"]),
       globalDynamicImports: new Set(),
       localDynamicImports: new Set(),
       localStaticImports: new Set(["../bar/bar.js", "../top.js"])

diff --git a/test/mocks/jsdelivr.ts b/test/mocks/jsdelivr.ts
@@ -1,7 +1,8 @@
 import {getCurrentAgent, mockAgent} from "./undici.js";
 
-const packages: [name: string, {version: string; dependencies?: Record<string, string>}][] = [
+const packages: [name: string, {version: string; contents?: string; dependencies?: Record<string, string>}][] = [
   ["@duckdb/duckdb-wasm", {version: "1.29.0"}],
+  ["@example/url-import", {version: "1.0.0", contents: "import('https://example.com');"}],
   ["@observablehq/inputs", {version: "0.10.6"}],
   ["@observablehq/plot", {version: "0.6.11"}],
   ["@observablehq/sample-datasets", {version: "1.0.1"}],
@@ -50,7 +51,7 @@ export function mockJsDelivr() {
         .persist(); // prettier-ignore
       cdnClient
         .intercept({path: new RegExp(`^/npm/${name}@${pkg.version}/`), method: "GET"})
-        .reply(200, "", {headers: {"cache-control": "public, immutable", "content-type": "text/javascript; charset=utf-8"}})
+        .reply(200, pkg.contents ?? "", {headers: {"cache-control": "public, immutable", "content-type": "text/javascript; charset=utf-8"}})
         .persist(); // prettier-ignore
     }
   });

diff --git a/test/output/build/imports/_import/bar/bar.13bb8056.js b/test/output/build/imports/_import/bar/bar.13bb8056.js
diff --git a/test/output/build/imports/_import/bar/bar.4460ccc2.js b/test/output/build/imports/_import/bar/bar.4460ccc2.js
@@ -0,0 +1 @@
+export {bar} from "./baz.2add1dd0.js";
diff --git a/...build/imports/_import/bar/baz.cdbfb28b.js → ...build/imports/_import/bar/baz.2add1dd0.js b/...build/imports/_import/bar/baz.cdbfb28b.js → ...build/imports/_import/bar/baz.2add1dd0.js
@@ -1,4 +1,4 @@
-import {foo} from "../foo/foo.5963bf78.js";
+import {foo} from "../foo/foo.bcd720b2.js";
 
 export const bar = "bar";
 export const foobar = foo + "bar";
diff --git a/...build/imports/_import/foo/foo.5963bf78.js → ...build/imports/_import/foo/foo.bcd720b2.js b/...build/imports/_import/foo/foo.5963bf78.js → ...build/imports/_import/foo/foo.bcd720b2.js
@@ -1,5 +1,6 @@
 import "../../_npm/[email protected]/cd372fb8.js";
-import {bar} from "../bar/bar.13bb8056.js";
+import "../../_npm/@example/[email protected]/1dd108c5.js";
+import {bar} from "../bar/bar.4460ccc2.js";
 export {top} from "../top.160847a6.js";
 
 export const foo = "foo";

diff --git a/test/output/build/imports/_npm/@example/[email protected]/1dd108c5.js b/test/output/build/imports/_npm/@example/[email protected]/1dd108c5.js
@@ -0,0 +1 @@
+import('https://example.com');
diff --git a/test/output/build/imports/foo/foo.html b/test/output/build/imports/foo/foo.html
@@ -14,11 +14,12 @@
 <link rel="modulepreload" href="../_observablehq/runtime.00000002.js">
 <link rel="modulepreload" href="../_observablehq/stdlib.00000003.js">
 <link rel="modulepreload" href="../_npm/[email protected]/cd372fb8.js">
-<link rel="modulepreload" href="../_import/bar/bar.13bb8056.js">
+<link rel="modulepreload" href="../_import/bar/bar.4460ccc2.js">
 <link rel="modulepreload" href="../_import/top.160847a6.js">
 <link rel="modulepreload" href="../_import/foo/foo bar.b173d3de.js">
-<link rel="modulepreload" href="../_import/bar/baz.cdbfb28b.js">
-<link rel="modulepreload" href="../_import/foo/foo.5963bf78.js">
+<link rel="modulepreload" href="../_import/bar/baz.2add1dd0.js">
+<link rel="modulepreload" href="../_import/foo/foo.bcd720b2.js">
+<link rel="modulepreload" href="../_npm/@example/[email protected]/1dd108c5.js">
 <script type="module">
 
 import {define} from "../_observablehq/client.00000001.js";
@@ -28,7 +29,7 @@
 registerFile("./hello.txt", {"name":"./hello.txt","mimeType":"text/plain","path":"../_file/foo/hello.5891b5b5.txt","lastModified":/* ts */1706742000000,"size":6});
 
 define({id: "261e010e", inputs: ["display","FileAttachment"], outputs: ["d3","bar","top"], body: async (display,FileAttachment) => {
-const [d3, {bar}, {top}] = await Promise.all([import("../_npm/[email protected]/cd372fb8.js"), import("../_import/bar/bar.13bb8056.js"), import("../_import/top.160847a6.js")]);
+const [d3, {bar}, {top}] = await Promise.all([import("../_npm/[email protected]/cd372fb8.js"), import("../_import/bar/bar.4460ccc2.js"), import("../_import/top.160847a6.js")]);
 
 display(bar);
 display(top);

diff --git a/test/telemetry-test.ts b/test/telemetry-test.ts
@@ -29,22 +29,22 @@ describe("telemetry", () => {
 
   it("sends data", async () => {
     Telemetry._instance = new Telemetry(noopEffects);
-    Telemetry.record({event: "build", step: "start", test: true});
+    Telemetry.record({event: "build", step: "start"});
     await Telemetry.instance.pending;
     agent.assertNoPendingInterceptors();
   });
 
   it("shows a banner", async () => {
     const logger = new MockLogger();
     const telemetry = new Telemetry({...noopEffects, logger, readFile: () => Promise.reject()});
-    telemetry.record({event: "build", step: "start", test: true});
+    telemetry.record({event: "build", step: "start"});
     await telemetry.pending;
     logger.assertExactErrors([/Attention.*observablehq.com.*OBSERVABLE_TELEMETRY_DISABLE=true/s]);
   });
 
   it("can be disabled", async () => {
     const telemetry = new Telemetry({...noopEffects, process: processMock({env: {OBSERVABLE_TELEMETRY_DISABLE: "1"}})});
-    telemetry.record({event: "build", step: "start", test: true});
+    telemetry.record({event: "build", step: "start"});
     await telemetry.pending;
     assert.equal(agent.pendingInterceptors().length, 1);
   });
@@ -56,7 +56,7 @@ describe("telemetry", () => {
       logger,
       process: processMock({env: {OBSERVABLE_TELEMETRY_DEBUG: "1"}})
     });
-    telemetry.record({event: "build", step: "start", test: true});
+    telemetry.record({event: "build", step: "start"});
     await telemetry.pending;
     assert.equal(logger.errorLines.length, 1);
     assert.equal(logger.errorLines[0][0], "[telemetry]");
@@ -71,7 +71,7 @@ describe("telemetry", () => {
       process: processMock({env: {OBSERVABLE_TELEMETRY_DEBUG: "1"}}),
       writeFile: () => Promise.reject()
     });
-    telemetry.record({event: "build", step: "start", test: true});
+    telemetry.record({event: "build", step: "start"});
     await telemetry.pending;
     assert.notEqual(logger.errorLines[0][1].ids.session, null);
     assert.equal(logger.errorLines[0][1].ids.device, null);
@@ -86,7 +86,7 @@ describe("telemetry", () => {
       logger,
       process: processMock({env: {OBSERVABLE_TELEMETRY_ORIGIN: "https://invalid."}})
     });
-    telemetry.record({event: "build", step: "start", test: true});
+    telemetry.record({event: "build", step: "start"});
     await telemetry.pending;
     assert.equal(logger.errorLines.length, 0);
     assert.equal(agent.pendingInterceptors().length, 1);