Integrate ffmpeg package, export videos with audio, add Audio tag (#3)

This PR does the following:

- extends ffmpeg exporter to include not just visuals, but also audio from videos
- adds <Audio /> tag
- few bug fixes w.r.t. display of media in the player

package.json

@@ -38,6 +38,7 @@
   "devDependencies": {
     "@commitlint/cli": "^18.4.2",
     "@commitlint/config-conventional": "^18.4.2",
+    "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.11.0",
     "@typescript-eslint/parser": "^6.11.0",
     "eslint": "^8.54.0",
@@ -53,5 +54,8 @@
   "lint-staged": {
     "*.{ts,tsx}": "eslint --fix",
     "*.{js,jsx,ts,tsx,md,scss}": "prettier --write"
+  },
+  "dependencies": {
+    "uuid": "^9.0.1"
   }
 }

packages/2d/src/lib/components/Audio.ts

@@ -0,0 +1,130 @@
+import {DependencyContext, PlaybackState, viaProxy} from '@motion-canvas/core';
+import {computed, nodeName} from '../decorators';
+import {Media, MediaProps} from './Media';
+
+@nodeName('Audio')
+export class Audio extends Media {
+  private static readonly pool: Record<string, HTMLAudioElement> = {};
+
+  public constructor(props: MediaProps) {
+    super(props);
+  }
+
+  protected mediaElement(): HTMLAudioElement {
+    return this.audio();
+  }
+
+  protected seekedMedia(): HTMLAudioElement {
+    return this.seekedAudio();
+  }
+
+  protected fastSeekedMedia(): HTMLAudioElement {
+    return this.fastSeekedAudio();
+  }
+
+  @computed()
+  protected audio(): HTMLAudioElement {
+    const src = viaProxy(this.src());
+    const key = `${this.key}/${src}`;
+    let audio = Audio.pool[key];
+    if (!audio) {
+      audio = document.createElement('audio');
+      audio.crossOrigin = 'anonymous';
+      audio.src = src;
+      Audio.pool[key] = audio;
+    }
+    if (audio.readyState < 2) {
+      DependencyContext.collectPromise(
+        new Promise<void>(resolve => {
+          const listener = () => {
+            resolve();
+            audio.removeEventListener('canplay', listener);
+          };
+          audio.addEventListener('canplay', listener);
+        }),
+      );
+    }
+
+    return audio;
+  }
+
+  @computed()
+  protected seekedAudio(): HTMLAudioElement {
+    const audio = this.audio();
+
+    audio.addEventListener('ended', () => {
+      this.pause();
+    });
+
+    if (!(this.time() < audio.duration)) {
+      this.pause();
+      return audio;
+    }
+
+    const time = this.clampTime(this.time());
+    audio.playbackRate = this.playbackRate();
+
+    if (!audio.paused) {
+      audio.pause();
+    }
+
+    if (this.lastTime === time) {
+      return audio;
+    }
+
+    this.setCurrentTime(time);
+
+    return audio;
+  }
+
+  @computed()
+  protected fastSeekedAudio(): HTMLAudioElement {
+    const audio = this.audio();
+
+    if (!(this.time() < audio.duration)) {
+      this.pause();
+      return audio;
+    }
+
+    const time = this.clampTime(this.time());
+
+    audio.playbackRate = this.playbackRate();
+
+    if (this.lastTime === time) {
+      return audio;
+    }
+
+    const playing =
+      this.playing() && time < audio.duration && audio.playbackRate > 0;
+    if (playing) {
+      if (audio.paused) {
+        DependencyContext.collectPromise(audio.play());
+      }
+    } else {
+      if (!audio.paused) {
+        audio.pause();
+      }
+    }
+    if (Math.abs(audio.currentTime - time) > 0.3) {
+      this.setCurrentTime(time);
+    } else if (!playing) {
+      audio.currentTime = time;
+    }
+
+    return audio;
+  }
+
+  protected override draw(context: CanvasRenderingContext2D) {
+    const playbackState = this.view().playbackState();
+
+    playbackState === PlaybackState.Playing ||
+    playbackState === PlaybackState.Presenting
+      ? this.fastSeekedAudio()
+      : this.seekedAudio();
+
+    context.save();
+    context.restore();
+
+    this.drawChildren(context);
+  }
+}

packages/2d/src/lib/components/Media.ts

@@ -0,0 +1,165 @@
+import {
+  DependencyContext,
+  SignalValue,
+  SimpleSignal,
+  clamp,
+  isReactive,
+  useLogger,
+  useThread,
+} from '@motion-canvas/core';
+import {computed, initial, signal} from '../decorators';
+import {Rect, RectProps} from './Rect';
+import reactivePlaybackRate from './__logs__/reactive-playback-rate.md';
+
+export interface MediaProps extends RectProps {
+  src?: SignalValue<string>;
+  loop?: SignalValue<boolean>;
+  playbackRate?: number;
+  time?: SignalValue<number>;
+  play?: boolean;
+}
+
+export abstract class Media extends Rect {
+  @signal()
+  public declare readonly src: SimpleSignal<string, this>;
+
+  @initial(false)
+  @signal()
+  public declare readonly loop: SimpleSignal<boolean, this>;
+
+  @initial(1)
+  @signal()
+  public declare readonly playbackRate: SimpleSignal<number, this>;
+
+  @initial(0)
+  @signal()
+  protected declare readonly time: SimpleSignal<number, this>;
+
+  @initial(false)
+  @signal()
+  protected declare readonly playing: SimpleSignal<boolean, this>;
+
+  protected lastTime = -1;
+
+  public constructor(props: MediaProps) {
+    super(props);
+    if (props.play) {
+      this.play();
+    }
+  }
+
+  public isPlaying(): boolean {
+    return this.playing();
+  }
+
+  public getCurrentTime(): number {
+    return this.clampTime(this.time());
+  }
+
+  public getDuration(): number {
+    return this.mediaElement().duration;
+  }
+
+  public override dispose() {
+    this.pause();
+    this.remove();
+    super.dispose();
+  }
+
+  @computed()
+  public override completion(): number {
+    return this.clampTime(this.time()) / this.getDuration();
+  }
+
+  protected abstract mediaElement(): HTMLMediaElement;
+
+  protected abstract seekedMedia(): HTMLMediaElement;
+
+  protected abstract fastSeekedMedia(): HTMLMediaElement;
+
+  protected abstract override draw(context: CanvasRenderingContext2D): void;
+
+  protected setCurrentTime(value: number) {
+    const media = this.mediaElement();
+    if (media.readyState < 2) return;
+
+    media.currentTime = value;
+    this.lastTime = value;
+    if (media.seeking) {
+      DependencyContext.collectPromise(
+        new Promise<void>(resolve => {
+          const listener = () => {
+            resolve();
+            media.removeEventListener('seeked', listener);
+          };
+          media.addEventListener('seeked', listener);
+        }),
+      );
+    }
+  }
+
+  protected setPlaybackRate(playbackRate: number) {
+    let value: number;
+    if (isReactive(playbackRate)) {
+      value = playbackRate();
+      useLogger().warn({
+        message: 'Invalid value set as the playback rate',
+        remarks: reactivePlaybackRate,
+        inspect: this.key,
+        stack: new Error().stack,
+      });
+    } else {
+      value = playbackRate;
+    }
+    this.playbackRate.context.setter(value);
+
+    if (this.playing()) {
+      if (value === 0) {
+        this.pause();
+      } else {
+        const time = useThread().time;
+        const start = time();
+        const offset = this.time();
+        this.time(() => this.clampTime(offset + (time() - start) * value));
+      }
+    }
+  }
+
+  public play() {
+    const time = useThread().time;
+    const start = time();
+    const offset = this.time();
+    const playbackRate = this.playbackRate();
+    this.playing(true);
+    this.time(() => this.clampTime(offset + (time() - start) * playbackRate));
+  }
+
+  public pause() {
+    this.playing(false);
+    this.time.save();
+    this.mediaElement().pause();
+  }
+
+  public seek(time: number) {
+    const playing = this.playing();
+    this.time(this.clampTime(time));
+    if (playing) {
+      this.play();
+    } else {
+      this.pause();
+    }
+  }
+
+  public clampTime(time: number): number {
+    const duration = this.getDuration();
+    if (this.loop()) {
+      time %= duration;
+    }
+    return clamp(0, duration, time);
+  }
+
+  protected override collectAsyncResources() {
+    super.collectAsyncResources();
+    this.seekedMedia();
+  }
+}