Add demo for tensorflow bodypix (#1027)

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- [DEMO] Add a image segmentation video filter based on Tensorflow BodyPix
 
 
 ### Changed

demos/browser/app/meetingV2/meetingV2.ts

@@ -52,6 +52,11 @@ import {
 
 import CircularCut from './videofilter/CircularCut';
 import EmojifyVideoFrameProcessor from './videofilter/EmojifyVideoFrameProcessor';
+import SegmentationProcessor from './videofilter/SegmentationProcessor';
+import {
+  loadBodyPixDependency,
+  platformCanSupportBodyPixWithoutDegradation,
+} from './videofilter/SegmentationUtil';
 import WebRTCStatsCollector from './webrtcstatscollector/WebRTCStatsCollector';
 
 const SHOULD_DIE_ON_FATALS = (() => {
@@ -119,7 +124,7 @@ const VOICE_FOCUS_SPEC = {
   paths: VOICE_FOCUS_PATHS,
 };
 
-type VideoFilterName = 'Emojify' | 'CircularCut' | 'NoOp' | 'None';
+type VideoFilterName = 'Emojify' | 'CircularCut' | 'NoOp' | 'None' | 'Segmentation';
 
 const VIDEO_FILTERS: VideoFilterName[] = ['Emojify', 'CircularCut', 'NoOp'];
 
@@ -213,6 +218,9 @@ export class DemoMeetingApp
   static readonly DATA_MESSAGE_TOPIC: string = 'chat';
   static readonly DATA_MESSAGE_LIFETIME_MS: number = 300000;
 
+  // ideally we don't need to change this. Keep this configurable in case users have super slow network.
+  loadingBodyPixDependencyTimeoutMs: number = 10000;
+
   showActiveSpeakerScores = false;
   activeSpeakerLayout = true;
   meeting: string | null = null;
@@ -1588,6 +1596,16 @@ export class DemoMeetingApp
       this.enableUnifiedPlanForChromiumBasedBrowsers
     ) {
       filters = filters.concat(VIDEO_FILTERS);
+
+      if (platformCanSupportBodyPixWithoutDegradation()) {
+        try {
+          await loadBodyPixDependency(this.loadingBodyPixDependencyTimeoutMs);
+          filters.push('Segmentation');
+        } catch (error) {
+          fatal(error);
+          this.log('failed to load segmentation dependencies', error);
+        }
+      }
     }
 
     this.populateInMeetingDeviceList(
@@ -2030,6 +2048,10 @@ export class DemoMeetingApp
       return new NoOpVideoFrameProcessor();
     }
 
+    if (videoFilter === 'Segmentation') {
+      return new SegmentationProcessor();
+    }
+
     return null;
   }
 

demos/browser/app/meetingV2/videofilter/CircularCut.ts

@@ -1,11 +1,7 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import {
-  CanvasVideoFrameBuffer,
-  VideoFrameBuffer,
-  VideoFrameProcessor,
-} from 'amazon-chime-sdk-js';
+import { CanvasVideoFrameBuffer, VideoFrameBuffer, VideoFrameProcessor } from 'amazon-chime-sdk-js';
 
 /**
  * [[CircularCut]] is an implementation of {@link VideoFrameProcessor} for demonstration purpose.

demos/browser/app/meetingV2/videofilter/ResizeProcessor.ts

@@ -1,11 +1,7 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import {
-  CanvasVideoFrameBuffer,
-  VideoFrameBuffer,
-  VideoFrameProcessor,
-} from 'amazon-chime-sdk-js';
+import { CanvasVideoFrameBuffer, VideoFrameBuffer, VideoFrameProcessor } from 'amazon-chime-sdk-js';
 
 /**
  * [[ResizeProcessor]] updates the input {@link VideoFrameBuffer} and resize given the display aspect ratio.

demos/browser/app/meetingV2/videofilter/SegmentationProcessor.ts

@@ -0,0 +1,70 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// eslint-disable-next-line
+declare var bodyPix: any;
+
+import { CanvasVideoFrameBuffer, VideoFrameBuffer, VideoFrameProcessor } from 'amazon-chime-sdk-js';
+
+export default class SegmentationProcessor implements VideoFrameProcessor {
+  private targetCanvas: HTMLCanvasElement = document.createElement('canvas') as HTMLCanvasElement;
+  private canvasVideoFrameBuffer = new CanvasVideoFrameBuffer(this.targetCanvas);
+  private sourceWidth: number = 0;
+  private sourceHeight: number = 0;
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private mask: any | undefined = undefined;
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private model: any | undefined = undefined;
+
+  constructor() {}
+
+  async process(buffers: VideoFrameBuffer[]): Promise<VideoFrameBuffer[]> {
+    if (!this.model) {
+      this.model = await bodyPix.load();
+    }
+
+    const inputCanvas = buffers[0].asCanvasElement();
+    if (!inputCanvas) {
+      throw new Error('buffer is already destroyed');
+    }
+
+    const frameWidth = inputCanvas.width;
+    const frameHeight = inputCanvas.height;
+    if (frameWidth === 0 || frameHeight === 0) {
+      return buffers;
+    }
+
+    if (this.sourceWidth !== frameWidth || this.sourceHeight !== frameHeight) {
+      this.sourceWidth = frameWidth;
+      this.sourceHeight = frameHeight;
+
+      // update target canvas size to match the frame size
+      this.targetCanvas.width = this.sourceWidth;
+      this.targetCanvas.height = this.sourceHeight;
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    let predictions = await this.model.segmentPerson(inputCanvas);
+    if (predictions) {
+      const foregroundColor = { r: 255, g: 255, b: 255, a: 255 };
+      const backgroundColor = { r: 0, g: 0, b: 0, a: 255 };
+      this.mask = bodyPix.toMask(predictions, foregroundColor, backgroundColor, true);
+    }
+
+    if (this.mask) {
+      bodyPix.drawMask(this.targetCanvas, inputCanvas as HTMLCanvasElement, this.mask);
+      buffers[0] = this.canvasVideoFrameBuffer;
+    }
+    predictions = undefined;
+    return buffers;
+  }
+
+  async destroy(): Promise<void> {
+    if (this.model) {
+      this.model.dispose();
+    }
+    this.model = undefined;
+  }
+}

demos/browser/app/meetingV2/videofilter/SegmentationUtil.ts

@@ -0,0 +1,60 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { detect } from 'detect-browser';
+
+const SEGMENTATION_DEPENDENCIES = [
+  {
+    src: 'https://cdn.jsdelivr.net/npm/@tensorflow/[email protected]/dist/tf.min.js',
+    integrity: 'sha384-uI1PW0SEa/QzAUuRQ6Bz5teBONsa9D0ZbVxMcM8mu4IjJ5msHyM7RRtZtL8LnSf3',
+    crossOrigin: 'anonymous',
+  },
+
+  {
+    src: 'https://cdn.jsdelivr.net/npm/@tensorflow/[email protected]/dist/tf-core.min.js',
+    integrity: 'sha384-DlI/SVdTGUBY5hi4h0p+nmC6V8i0FW5Nya/gYElz0L68HrSiXsBh+rqWcoZx3SXY',
+    crossOrigin: 'anonymous',
+  },
+
+  {
+    src:
+      'https://cdn.jsdelivr.net/npm/@tensorflow/[email protected]/dist/tf-backend-webgl.min.js',
+    integrity: 'sha384-21TV9Kpzn8SF68G1U6nYN3qPZnb97F06JuW4v0FDDBzW+CUwv8GcKMR+BjnE7Vmm',
+    crossOrigin: 'anonymous',
+  },
+
+  {
+    src: 'https://cdn.jsdelivr.net/npm/@tensorflow-models/[email protected]/dist/body-pix.min.js',
+    integrity: 'sha384-dPJ/sICXCqdh39bsuGLVFcOiRyeL/XcFrwiFrJq9oh7k1TCtsUKhX6UV2X4UuKU4',
+    crossOrigin: 'anonymous',
+  },
+];
+
+
+export async function loadBodyPixDependency(timeoutMs: number): Promise<void> {
+  // the tf library loading order must be followed
+  for (let i = 0; i < SEGMENTATION_DEPENDENCIES.length; i++) {
+    const dep = SEGMENTATION_DEPENDENCIES[i];
+    await new Promise<void>((resolve, reject) => {
+      let script = document.createElement('script');
+      const timer = setTimeout(() => {
+        reject(new Error(`Loading script ${dep.src} takes longer than ${timeoutMs}`));
+      }, timeoutMs);
+      script.onload = function(_ev) {
+        clearTimeout(timer);
+        resolve();
+      };
+      script.src = dep.src;
+      script.integrity = dep.integrity;
+      script.crossOrigin = dep.crossOrigin;
+      document.body.appendChild(script);
+    });
+  }
+}
+
+export function platformCanSupportBodyPixWithoutDegradation(): boolean {
+  // https://blog.tensorflow.org/2019/11/updated-bodypix-2.html for more detail on performance
+  // https://github.com/tensorflow/tfjs/issues/3319 which results in firefox memory leak
+  const browser = detect();
+  return browser.name === 'chrome' && /(android)/i.test(navigator.userAgent) === false;
+}