feat(experiment): Setup Amplitude experiment & Add first dev collab experiment

packages/manager/package.json

@@ -97,6 +97,7 @@
 		"unified": "^10.1.2"
 	},
 	"devDependencies": {
+		"@amplitude/experiment-js-client": "1.9.8",
 		"@prismicio/mock": "0.2.0",
 		"@size-limit/preset-small-lib": "8.2.4",
 		"@types/cookie": "0.5.1",

packages/manager/src/constants/API_TOKENS.ts

@@ -2,17 +2,24 @@ import { APPLICATION_MODE } from "./APPLICATION_MODE";
 
 type APITokens = {
 	SegmentKey: string;
+	AmplitudeKey: string;
 };
 
 export const API_TOKENS: APITokens = (() => {
 	switch (process.env.SM_ENV) {
 		case APPLICATION_MODE.Development:
 		case APPLICATION_MODE.Staging:
-			return { SegmentKey: "Ng5oKJHCGpSWplZ9ymB7Pu7rm0sTDeiG" };
+			return {
+				SegmentKey: "Ng5oKJHCGpSWplZ9ymB7Pu7rm0sTDeiG",
+				AmplitudeKey: "client-rqVU4xTNaz7F51nBfKRUa0K3qnODiqzh",
+			};
 		case undefined:
 		case "":
 		case APPLICATION_MODE.Production:
-			return { SegmentKey: "cGjidifKefYb6EPaGaqpt8rQXkv5TD6P" };
+			return {
+				SegmentKey: "cGjidifKefYb6EPaGaqpt8rQXkv5TD6P",
+				AmplitudeKey: "client-JuQQWUPimfKWId3WWU6p8xSkTiFqd1qV",
+			};
 		default:
 			throw new Error(`Unknown application mode "${process.env.SM_ENV}".`);
 	}

packages/manager/src/index.ts

@@ -47,3 +47,5 @@ export type { SliceMachineConfig, PackageManager } from "./types";
 export type { APIEndpoints } from "./constants/API_ENDPOINTS";
 
 export { REPOSITORY_NAME_VALIDATION } from "./constants/REPOSITORY_NAME_VALIDATION";
+
+export type { Variant } from "./managers/telemetry/types";

packages/manager/src/managers/telemetry/TelemetryManager.ts

@@ -1,5 +1,9 @@
+import {
+	Experiment,
+	ExperimentClient,
+	Variant,
+} from "@amplitude/experiment-js-client";
 import { randomUUID } from "node:crypto";
-
 import { Analytics, GroupParams, TrackParams } from "@segment/analytics-node";
 
 import { readPrismicrc } from "../../lib/prismicrc";
@@ -58,6 +62,7 @@ export class TelemetryManager extends BaseManager {
 	private _anonymousID: string | undefined = undefined;
 	private _userID: string | undefined = undefined;
 	private _context: TelemetryManagerContext | undefined = undefined;
+	private _experiment: ExperimentClient | undefined = undefined;
 
 	async initTelemetry(args: TelemetryManagerInitTelemetryArgs): Promise<void> {
 		const isTelemetryEnabled = await this.checkIsTelemetryEnabled();
@@ -82,6 +87,10 @@ export class TelemetryManager extends BaseManager {
 			return analytics;
 		};
 
+		if (isTelemetryEnabled) {
+			await this.initExperiment();
+		}
+
 		this._anonymousID = randomUUID();
 		this._context = { app: { name: args.appName, version: args.appVersion } };
 	}
@@ -268,4 +277,44 @@ export class TelemetryManager extends BaseManager {
 
 		return readPrismicrc(root).telemetry !== false;
 	}
+
+	private async initExperiment(): Promise<void> {
+		try {
+			const repositoryName = await this.project.getRepositoryName();
+
+			this._experiment = Experiment.initialize(API_TOKENS.AmplitudeKey, {
+				pollOnStart: false,
+				fetchOnStart: false,
+			});
+
+			await this._experiment
+				.start({
+					user_properties: {
+						Repository: repositoryName,
+					},
+				})
+				.catch((error) => {
+					console.error("Error starting experiment", error);
+				});
+		} catch (error) {
+			if (import.meta.env.DEV) {
+				console.error("Error initializing experiment", error);
+			}
+		}
+	}
+
+	async experimentVariant(variantKey: string): Promise<Variant | undefined> {
+		if (this._experiment) {
+			await this._experiment.fetch(
+				this._userID ? { user_id: this._userID } : undefined,
+				{
+					flagKeys: [variantKey],
+				},
+			);
+
+			return this._experiment.variant(variantKey);
+		}
+
+		return undefined;
+	}
 }

packages/manager/src/managers/telemetry/types.ts

@@ -1,6 +1,8 @@
 import { CustomTypeFormat } from "../customTypes/types";
 import type { LimitType } from "../prismicRepository/types";
 
+export type { Variant } from "@amplitude/experiment-js-client";
+
 export const SegmentEventType = {
 	command_init_start: "command:init:start",
 	command_init_identify: "command:init:identify",
@@ -26,6 +28,9 @@ export const SegmentEventType = {
 	open_page_snippet: "page-type:open-snippet",
 	copy_page_snippet: "page-type:copy-snippet",
 	switch_environment: "environment:switch",
+	devCollab_joinBetaClicked: "dev-collab:join-beta-clicked",
+	devCollab_setUpWorkflowOpened: "dev-collab:set-up-workflow-opened",
+	devCollab_workflowStubDisplayed: "dev-collab:workflow-stub-displayed",
 } as const;
 type SegmentEventTypes =
 	(typeof SegmentEventType)[keyof typeof SegmentEventType];
@@ -63,6 +68,12 @@ export const HumanSegmentEventType = {
 	[SegmentEventType.copy_page_snippet]:
 		"Slice Machine page code snippet copied",
 	[SegmentEventType.switch_environment]: "SliceMachine environment switch",
+	[SegmentEventType.devCollab_joinBetaClicked]:
+		"SliceMachine Dev Collab Join Beta Clicked",
+	[SegmentEventType.devCollab_setUpWorkflowOpened]:
+		"SliceMachine Dev Collab Set Up Workflow Opened",
+	[SegmentEventType.devCollab_workflowStubDisplayed]:
+		"SliceMachine Dev Collab Workflow Stub Displayed",
 } as const;
 export type HumanSegmentEventTypes =
 	(typeof HumanSegmentEventType)[keyof typeof HumanSegmentEventType];
@@ -256,6 +267,18 @@ type EditorWidgetUsedSegmentEvent = SegmentEvent<
 	{ sliceId: string }
 >;
 
+type DevCollabJoinBetaClicked = SegmentEvent<
+	typeof SegmentEventType.devCollab_joinBetaClicked
+>;
+
+type DevCollabSetUpWorkflowOpened = SegmentEvent<
+	typeof SegmentEventType.devCollab_setUpWorkflowOpened
+>;
+
+type DevCollabWorkflowStubDisplayed = SegmentEvent<
+	typeof SegmentEventType.devCollab_workflowStubDisplayed
+>;
+
 export type SegmentEvents =
 	| CommandInitStartSegmentEvent
 	| CommandInitIdentifySegmentEvent
@@ -280,4 +303,7 @@ export type SegmentEvents =
 	| OpenPageSnippetSegmentEvent
 	| CopyPageSnippetSegmentEvent
 	| UsersInviteButtonClickedSegmentEvent
-	| SwitchEnvironmentSegmentEvent;
+	| SwitchEnvironmentSegmentEvent
+	| DevCollabJoinBetaClicked
+	| DevCollabSetUpWorkflowOpened
+	| DevCollabWorkflowStubDisplayed;

packages/manager/test/SliceMachineManager-telemetry-experimentVariant.test.ts

@@ -0,0 +1,71 @@
+import { expect, it, vi } from "vitest";
+
+import { createTestPlugin } from "./__testutils__/createTestPlugin";
+import { createTestProject } from "./__testutils__/createTestProject";
+
+import { createSliceMachineManager } from "../src";
+
+vi.mock("@amplitude/experiment-js-client", () => {
+	const MockAmplitudeClient = {
+		start: vi.fn(() => Promise.resolve()),
+		fetch: vi.fn(),
+		variant: vi.fn((variant: string) => {
+			if (variant === "test-variant-on") {
+				return {
+					value: "on",
+				};
+			} else {
+				return {
+					value: "off",
+				};
+			}
+		}),
+	};
+
+	const MockExperiment = {
+		initialize: vi.fn(() => MockAmplitudeClient),
+	};
+
+	return {
+		Experiment: MockExperiment,
+		ExperimentClient: MockAmplitudeClient,
+	};
+});
+
+it("get the experiment 'on' value for a specific variant", async () => {
+	const adapter = createTestPlugin();
+	const cwd = await createTestProject({ adapter });
+	const manager = createSliceMachineManager({
+		nativePlugins: { [adapter.meta.name]: adapter },
+		cwd,
+	});
+
+	await manager.telemetry.initTelemetry({
+		appName: "slice-machine-ui",
+		appVersion: "0.0.1-test",
+	});
+
+	const experimentVariant =
+		await manager.telemetry.experimentVariant("test-variant-on");
+
+	expect(experimentVariant).toEqual({ value: "on" });
+});
+
+it("get the experiment 'off' value for a specific variant", async () => {
+	const adapter = createTestPlugin();
+	const cwd = await createTestProject({ adapter });
+	const manager = createSliceMachineManager({
+		nativePlugins: { [adapter.meta.name]: adapter },
+		cwd,
+	});
+
+	await manager.telemetry.initTelemetry({
+		appName: "slice-machine-ui",
+		appVersion: "0.0.1-test",
+	});
+
+	const experimentVariant =
+		await manager.telemetry.experimentVariant("test-variant-off");
+
+	expect(experimentVariant).toEqual({ value: "off" });
+});

packages/manager/test/SliceMachineManager-telemetry-initTelemetry.test.ts

@@ -1,4 +1,5 @@
 import { beforeAll, expect, it } from "vitest";
+import { ExperimentClient } from "@amplitude/experiment-js-client";
 import { Analytics } from "@segment/analytics-node";
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
@@ -43,7 +44,7 @@ it("creates a reusable Segment client", async () => {
 	expect(manager.telemetry._segmentClient()._publisher._disable).toBe(false);
 });
 
-it("disables the Segment client if .prismicrc is configured to disable telemery", async () => {
+it("disables the Segment client if .prismicrc is configured to disable telemetry", async () => {
 	const adapter = createTestPlugin();
 	const cwd = await createTestProject({ adapter });
 	const manager = createSliceMachineManager({
@@ -61,3 +62,42 @@ it("disables the Segment client if .prismicrc is configured to disable telemery"
 	// @ts-expect-error - Accessing an internal private property
 	expect(manager.telemetry._segmentClient()._publisher._disable).toBe(true);
 });
+
+it("creates a reusable Experiment client", async () => {
+	const adapter = createTestPlugin();
+	const cwd = await createTestProject({ adapter });
+	const manager = createSliceMachineManager({
+		nativePlugins: { [adapter.meta.name]: adapter },
+		cwd,
+	});
+
+	// @ts-expect-error - Accessing an internal private property
+	expect(manager.telemetry._experiment).toBeUndefined();
+
+	await manager.telemetry.initTelemetry({
+		appName: "slice-machine-ui",
+		appVersion: "0.0.1-test",
+	});
+
+	// @ts-expect-error - Accessing an internal private property
+	expect(manager.telemetry._experiment).toBeInstanceOf(ExperimentClient);
+});
+
+it("disables the Experiment client if .prismicrc is configured to disable telemetry", async () => {
+	const adapter = createTestPlugin();
+	const cwd = await createTestProject({ adapter });
+	const manager = createSliceMachineManager({
+		nativePlugins: { [adapter.meta.name]: adapter },
+		cwd,
+	});
+
+	await fs.writeFile(path.join(os.homedir(), ".prismicrc"), "telemetry=false");
+
+	await manager.telemetry.initTelemetry({
+		appName: "slice-machine-ui",
+		appVersion: "0.0.1-test",
+	});
+
+	// @ts-expect-error - Accessing an internal private property
+	expect(manager.telemetry._experiment).toBeUndefined();
+});