feat: add ffmpeg exporter to monorepo

packages/ffmpeg/CHANGELOG.md

@@ -0,0 +1,24 @@
+# Change Log
+
+All notable changes to this project will be documented in this file. See
+[Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+
+## [1.1.1](https://github.com/motion-canvas/exporters/compare/v1.1.0...v1.1.1) (2023-05-13)
+
+### Bug Fixes
+
+- account for resolution scale
+  ([#3](https://github.com/motion-canvas/exporters/issues/3))
+  ([ba03bf1](https://github.com/motion-canvas/exporters/commit/ba03bf1db62c7aae45a4a98d9519f21085afff91))
+
+# Change Log
+
+All notable changes to this project will be documented in this file. See
+[Conventional Commits](https://conventionalcommits.org) for commit guidelines.
+
+# 1.1.0 (2023-05-10)
+
+### Features
+
+- initial commit
+  ([7a01fd5](https://github.com/motion-canvas/exporters/commit/7a01fd5614f2d62b4bd6e24c1096706f5dbf218b))

packages/ffmpeg/client/FFmpegExporterClient.ts

@@ -0,0 +1,139 @@
+import type {
+  MetaField,
+  Project,
+  RendererResult,
+  RendererSettings,
+} from '@motion-canvas/core';
+import {
+  BoolMetaField,
+  EventDispatcher,
+  Exporter,
+  ObjectMetaField,
+  ValueOf,
+} from '@motion-canvas/core';
+
+type ServerResponse =
+  | {
+      status: 'success';
+      method: string;
+      data: unknown;
+    }
+  | {
+      status: 'error';
+      method: string;
+      message?: string;
+    };
+
+type FFmpegExporterOptions = ValueOf<
+  ReturnType<typeof FFmpegExporterClient.meta>
+>;
+
+/**
+ * FFmpeg video exporter.
+ *
+ * @remarks
+ * Most of the export logic is handled on the server. This class communicates
+ * with the FFmpegBridge through a WebSocket connection which lets it invoke
+ * methods on the FFmpegExporterServer class.
+ *
+ * For example, calling the following method:
+ * ```ts
+ * async this.invoke('process', 7);
+ * ```
+ * Will invoke the `process` method on the FFmpegExporterServer class with `7`
+ * as the argument. The result of the method will be returned as a Promise.
+ *
+ * Before any methods can be invoked, the FFmpegExporterServer class must be
+ * initialized by invoking `start`.
+ */
+export class FFmpegExporterClient implements Exporter {
+  public static readonly id = '@motion-canvas/ffmpeg';
+  public static readonly displayName = 'Video (FFmpeg)';
+
+  public static meta(project: Project): MetaField<any> {
+    return new ObjectMetaField(this.displayName, {
+      fastStart: new BoolMetaField('fast start', true),
+      includeAudio: new BoolMetaField('include audio', true).disable(
+        !project.audio,
+      ),
+    });
+  }
+
+  public static async create(project: Project, settings: RendererSettings) {
+    return new FFmpegExporterClient(project, settings);
+  }
+
+  private static readonly response = new EventDispatcher<ServerResponse>();
+
+  static {
+    if (import.meta.hot) {
+      import.meta.hot.on(
+        `motion-canvas/ffmpeg-ack`,
+        (response: ServerResponse) => this.response.dispatch(response),
+      );
+    }
+  }
+
+  public constructor(
+    private readonly project: Project,
+    private readonly settings: RendererSettings,
+  ) {}
+
+  public async start(): Promise<void> {
+    const options = this.settings.exporter.options as FFmpegExporterOptions;
+    await this.invoke('start', {
+      ...this.settings,
+      ...options,
+      audio: this.project.audio,
+      audioOffset:
+        this.project.meta.shared.audioOffset.get() - this.settings.range[0],
+    });
+  }
+
+  public async handleFrame(canvas: HTMLCanvasElement): Promise<void> {
+    await this.invoke('handleFrame', {
+      data: canvas.toDataURL('image/png'),
+    });
+  }
+
+  public async stop(result: RendererResult): Promise<void> {
+    await this.invoke('end', result);
+  }
+
+  /**
+   * Remotely invoke a method on the server and wait for a response.
+   *
+   * @param method - The method name to execute on the server.
+   * @param data - The data that will be passed as an argument to the method.
+   *               Should be serializable.
+   */
+  private invoke<TResponse = unknown, TData = unknown>(
+    method: string,
+    data: TData,
+  ): Promise<TResponse> {
+    if (import.meta.hot) {
+      return new Promise((resolve, reject) => {
+        const handle = (response: ServerResponse) => {
+          if (response.method !== method) {
+            return;
+          }
+
+          FFmpegExporterClient.response.unsubscribe(handle);
+          if (response.status === 'success') {
+            resolve(response.data as TResponse);
+          } else {
+            reject({
+              message: 'An error occurred while exporting the video.',
+              remarks: `Method: ${method}<br>Server error: ${response.message}`,
+              object: data,
+            });
+          }
+        };
+        FFmpegExporterClient.response.subscribe(handle);
+        import.meta.hot!.send('motion-canvas/ffmpeg', {method, data});
+      });
+    } else {
+      throw new Error('FFmpegExporter can only be used locally.');
+    }
+  }
+}

packages/ffmpeg/client/global.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />

packages/ffmpeg/client/index.ts

@@ -0,0 +1,10 @@
+import type {ExporterClass} from '@motion-canvas/core';
+import {makePlugin} from '@motion-canvas/core';
+import {FFmpegExporterClient} from './FFmpegExporterClient';
+
+export default makePlugin({
+  name: 'ffmpeg-plugin',
+  exporters(): ExporterClass[] {
+    return [FFmpegExporterClient];
+  },
+});

packages/ffmpeg/client/motion-canvas.d.ts

@@ -0,0 +1 @@
+/// <reference types="@motion-canvas/core/project" />

packages/ffmpeg/client/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "baseUrl": ".",
+    "outDir": "../lib/client",
+    "inlineSourceMap": true,
+    "noImplicitAny": true,
+    "allowSyntheticDefaultImports": true,
+    "moduleResolution": "node",
+    "strict": true,
+    "module": "esnext",
+    "target": "es2020",
+    "declaration": true,
+    "declarationMap": true
+  },
+  "include": ["."]
+}

packages/ffmpeg/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@motion-canvas/ffmpeg",
+  "version": "1.1.1",
+  "description": "An FFmpeg video exporter for Motion Canvas",
+  "main": "lib/server/index.js",
+  "author": "motion-canvas",
+  "homepage": "https://motioncanvas.io/",
+  "bugs": "https://github.com/motion-canvas/exporters/issues",
+  "license": "MIT",
+  "scripts": {
+    "build": "npm run client:build && npm run server:build",
+    "client:build": "tsc --project client/tsconfig.json",
+    "client:dev": "tsc -w --project client/tsconfig.json",
+    "server:build": "tsc --project server/tsconfig.json",
+    "server:dev": "tsc -w --project server/tsconfig.json"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/motion-canvas/exporters.git"
+  },
+  "files": [
+    "lib"
+  ],
+  "devDependencies": {
+    "@types/fluent-ffmpeg": "^2.1.21"
+  },
+  "dependencies": {
+    "@ffmpeg-installer/ffmpeg": "^1.1.0",
+    "@ffprobe-installer/ffprobe": "^2.0.0",
+    "fluent-ffmpeg": "^2.1.2",
+    "@motion-canvas/core": "^3.14.1",
+    "@motion-canvas/vite-plugin": "^3.14.1",
+    "vite": "4.x"
+  }
+}

packages/ffmpeg/server/FFmpegBridge.ts

@@ -0,0 +1,80 @@
+import {PluginConfig} from '@motion-canvas/vite-plugin/lib/plugins';
+import type {WebSocketServer} from 'vite';
+import {
+  FFmpegExporterServer,
+  FFmpegExporterSettings,
+} from './FFmpegExporterServer';
+
+interface BrowserRequest {
+  method: string;
+  data: unknown;
+}
+
+/**
+ * A simple bridge between the FFmpegExporterServer and FFmpegExporterClient.
+ *
+ * @remarks
+ * This class lets the client exporter invoke methods on the server and receive
+ * responses using a simple Promise-based API.
+ */
+export class FFmpegBridge {
+  private process: FFmpegExporterServer | null = null;
+
+  public constructor(
+    private readonly ws: WebSocketServer,
+    private readonly config: PluginConfig,
+  ) {
+    ws.on('motion-canvas/ffmpeg', this.handleMessage);
+  }
+
+  private handleMessage = async ({method, data}: BrowserRequest) => {
+    if (method === 'start') {
+      try {
+        this.process = new FFmpegExporterServer(
+          data as FFmpegExporterSettings,
+          this.config,
+        );
+        this.respondSuccess(method, await this.process.start());
+      } catch (e: any) {
+        this.respondError(method, e?.message);
+      }
+      return;
+    }
+
+    if (!this.process) {
+      this.respondError(method, 'The exporting process has not been started.');
+      return;
+    }
+
+    if (!(method in this.process)) {
+      this.respondError(method, `Unknown method: "${method}".`);
+      return;
+    }
+
+    try {
+      this.respondSuccess(method, await (this.process as any)[method](data));
+    } catch (e: any) {
+      this.respondError(method, e?.message);
+    }
+
+    if (method === 'end') {
+      this.process = null;
+    }
+  };
+
+  private respondSuccess(method: string, data: any = {}) {
+    this.ws.send('motion-canvas/ffmpeg-ack', {
+      status: 'success',
+      method,
+      data,
+    });
+  }
+
+  private respondError(method: string, message = 'Unknown error.') {
+    this.ws.send('motion-canvas/ffmpeg-ack', {
+      status: 'error',
+      method,
+      message,
+    });
+  }
+}

packages/ffmpeg/server/FFmpegExporterServer.ts

@@ -0,0 +1,96 @@
+import {path as ffmpegPath} from '@ffmpeg-installer/ffmpeg';
+import {path as ffprobePath} from '@ffprobe-installer/ffprobe';
+import type {
+  RendererResult,
+  RendererSettings,
+} from '@motion-canvas/core/lib/app';
+import type {PluginConfig} from '@motion-canvas/vite-plugin/lib/plugins';
+import * as ffmpeg from 'fluent-ffmpeg';
+import * as fs from 'fs';
+import * as path from 'path';
+import {ImageStream} from './ImageStream';
+
+ffmpeg.setFfmpegPath(ffmpegPath);
+ffmpeg.setFfprobePath(ffprobePath);
+
+export interface FFmpegExporterSettings extends RendererSettings {
+  audio?: string;
+  audioOffset?: number;
+  fastStart: boolean;
+  includeAudio: boolean;
+}
+
+/**
+ * The server-side implementation of the FFmpeg video exporter.
+ */
+export class FFmpegExporterServer {
+  private readonly stream: ImageStream;
+  private readonly command: ffmpeg.FfmpegCommand;
+  private readonly promise: Promise<void>;
+
+  public constructor(
+    settings: FFmpegExporterSettings,
+    private readonly config: PluginConfig,
+  ) {
+    this.stream = new ImageStream();
+    this.command = ffmpeg();
+
+    // Input image sequence
+    this.command
+      .input(this.stream)
+      .inputFormat('image2pipe')
+      .inputFps(settings.fps);
+
+    // Input audio file
+    if (settings.includeAudio && settings.audio) {
+      this.command
+        .input((settings.audio as string).slice(1))
+        // FIXME Offset only works for negative values.
+        .inputOptions([`-itsoffset ${settings.audioOffset ?? 0}`]);
+    }
+
+    // Output settings
+    const size = {
+      x: Math.round(settings.size.x * settings.resolutionScale),
+      y: Math.round(settings.size.y * settings.resolutionScale),
+    };
+    this.command
+      .output(path.join(this.config.output, `${settings.name}.mp4`))
+      .outputOptions(['-pix_fmt yuv420p', '-shortest'])
+      .outputFps(settings.fps)
+      .size(`${size.x}x${size.y}`);
+    if (settings.fastStart) {
+      this.command.outputOptions(['-movflags +faststart']);
+    }
+
+    this.promise = new Promise<void>((resolve, reject) => {
+      this.command.on('end', resolve).on('error', reject);
+    });
+  }
+
+  public async start() {
+    if (!fs.existsSync(this.config.output)) {
+      await fs.promises.mkdir(this.config.output, {recursive: true});
+    }
+    this.command.run();
+  }
+
+  public async handleFrame({data}: {data: string}) {
+    const base64Data = data.slice(data.indexOf(',') + 1);
+    this.stream.pushImage(Buffer.from(base64Data, 'base64'));
+  }
+
+  public async end(result: RendererResult) {
+    this.stream.pushImage(null);
+    if (result === 1) {
+      try {
+        this.command.kill('SIGKILL');
+        await this.promise;
+      } catch (_) {
+        // do nothing
+      }
+    } else {
+      await this.promise;
+    }
+  }
+}

packages/ffmpeg/server/ImageStream.ts

@@ -0,0 +1,20 @@
+import {Readable} from 'stream';
+
+export class ImageStream extends Readable {
+  private image: Buffer | null = null;
+  private hasData = false;
+
+  public pushImage(image: Buffer | null) {
+    this.image = image;
+    this.hasData = true;
+    this._read();
+  }
+
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  public override _read() {
+    if (this.hasData) {
+      this.hasData = false;
+      this.push(this.image);
+    }
+  }
+}

packages/ffmpeg/server/global.d.ts

@@ -0,0 +1,9 @@
+declare module '@ffmpeg-installer/ffmpeg' {
+  let object: {path: string};
+  export = object;
+}
+
+declare module '@ffprobe-installer/ffprobe' {
+  let object: {path: string};
+  export = object;
+}

packages/ffmpeg/server/index.ts

@@ -0,0 +1,22 @@
+import {
+  Plugin,
+  PLUGIN_OPTIONS,
+  PluginConfig,
+} from '@motion-canvas/vite-plugin/lib/plugins';
+import {FFmpegBridge} from './FFmpegBridge';
+
+export default (): Plugin => {
+  let config: PluginConfig;
+  return {
+    name: 'motion-canvas/ffmpeg',
+    [PLUGIN_OPTIONS]: {
+      entryPoint: '@motion-canvas/ffmpeg/lib/client',
+      async config(value) {
+        config = value;
+      },
+    },
+    configureServer(server) {
+      new FFmpegBridge(server.ws, config);
+    },
+  };
+};

packages/ffmpeg/server/motion-canvas.d.ts

@@ -0,0 +1 @@
+/// <reference types="@motion-canvas/core/project" />

packages/ffmpeg/server/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "baseUrl": ".",
+    "outDir": "../lib/server",
+    "inlineSourceMap": true,
+    "allowSyntheticDefaultImports": true,
+    "moduleResolution": "node",
+    "strict": true,
+    "module": "CommonJS",
+    "target": "es2020",
+    "declaration": true,
+    "declarationMap": true
+  },
+  "include": ["."]
+}

packages/ffmpeg/tsdoc.json

@@ -0,0 +1,4 @@
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json",
+  "extends": ["../../tsdoc.json"]
+}

packages/template/package.json

@@ -12,7 +12,8 @@
   },
   "dependencies": {
     "@motion-canvas/core": "*",
-    "@motion-canvas/2d": "*"
+    "@motion-canvas/2d": "*",
+    "@motion-canvas/ffmpeg": "*"
   },
   "devDependencies": {
     "@motion-canvas/ui": "*",

packages/template/src/project.meta

@@ -21,11 +21,10 @@
     "resolutionScale": 1,
     "colorSpace": "srgb",
     "exporter": {
-      "name": "@motion-canvas/core/image-sequence",
+      "name": "@motion-canvas/ffmpeg",
       "options": {
-        "fileType": "image/png",
-        "quality": 100,
-        "groupByScene": false
+        "fastStart": true,
+        "includeAudio": true
       }
     }
   }

packages/template/vite.config.ts

@@ -1,3 +1,4 @@
+import ffmpeg from '@motion-canvas/ffmpeg';
 import markdown from '@motion-canvas/internal/vite/markdown-literals';
 import preact from '@preact/preset-vite';
 import {defineConfig} from 'vite';
@@ -23,6 +24,7 @@ export default defineConfig({
   },
   plugins: [
     markdown(),
+    ffmpeg(),
     preact({
       include: [
         /packages\/ui\/src\/(.*)\.tsx?$/,