From 93cd182a1a43a4c1c7020292ea38864e1c555d36 Mon Sep 17 00:00:00 2001 From: Xuan He <46945302+XHatan@users.noreply.github.com> Date: Wed, 27 Jan 2021 15:33:23 -0800 Subject: [PATCH] Revert "Add demo for tensorflow bodypix (#1027)" (#1032) This reverts commit 9f3eaa1c1615e0d12115d4f56b4d722beedbbf99. --- CHANGELOG.md | 1 - demos/browser/app/meetingV2/meetingV2.ts | 24 +------ .../app/meetingV2/videofilter/CircularCut.ts | 6 +- .../meetingV2/videofilter/ResizeProcessor.ts | 6 +- .../videofilter/SegmentationProcessor.ts | 70 ------------------- .../meetingV2/videofilter/SegmentationUtil.ts | 60 ---------------- 6 files changed, 11 insertions(+), 156 deletions(-) delete mode 100644 demos/browser/app/meetingV2/videofilter/SegmentationProcessor.ts delete mode 100644 demos/browser/app/meetingV2/videofilter/SegmentationUtil.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ee5fe8a4b..0571b30fe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- [DEMO] Add a image segmentation video filter based on Tensorflow BodyPix ### Changed diff --git a/demos/browser/app/meetingV2/meetingV2.ts b/demos/browser/app/meetingV2/meetingV2.ts index c385da35f3..ac942be6e9 100644 --- a/demos/browser/app/meetingV2/meetingV2.ts +++ b/demos/browser/app/meetingV2/meetingV2.ts @@ -52,11 +52,6 @@ import { import CircularCut from './videofilter/CircularCut'; import EmojifyVideoFrameProcessor from './videofilter/EmojifyVideoFrameProcessor'; -import SegmentationProcessor from './videofilter/SegmentationProcessor'; -import { - loadBodyPixDependency, - platformCanSupportBodyPixWithoutDegradation, -} from './videofilter/SegmentationUtil'; import WebRTCStatsCollector from './webrtcstatscollector/WebRTCStatsCollector'; const SHOULD_DIE_ON_FATALS = (() => { @@ -124,7 +119,7 @@ const VOICE_FOCUS_SPEC = { paths: VOICE_FOCUS_PATHS, }; -type VideoFilterName = 'Emojify' | 'CircularCut' | 'NoOp' | 'None' | 'Segmentation'; +type VideoFilterName = 'Emojify' | 'CircularCut' | 'NoOp' | 'None'; const VIDEO_FILTERS: VideoFilterName[] = ['Emojify', 'CircularCut', 'NoOp']; @@ -218,9 +213,6 @@ export class DemoMeetingApp static readonly DATA_MESSAGE_TOPIC: string = 'chat'; static readonly DATA_MESSAGE_LIFETIME_MS: number = 300000; - // ideally we don't need to change this. Keep this configurable in case users have super slow network. - loadingBodyPixDependencyTimeoutMs: number = 10000; - showActiveSpeakerScores = false; activeSpeakerLayout = true; meeting: string | null = null; @@ -1596,16 +1588,6 @@ export class DemoMeetingApp this.enableUnifiedPlanForChromiumBasedBrowsers ) { filters = filters.concat(VIDEO_FILTERS); - - if (platformCanSupportBodyPixWithoutDegradation()) { - try { - await loadBodyPixDependency(this.loadingBodyPixDependencyTimeoutMs); - filters.push('Segmentation'); - } catch (error) { - fatal(error); - this.log('failed to load segmentation dependencies', error); - } - } } this.populateInMeetingDeviceList( @@ -2048,10 +2030,6 @@ export class DemoMeetingApp return new NoOpVideoFrameProcessor(); } - if (videoFilter === 'Segmentation') { - return new SegmentationProcessor(); - } - return null; } diff --git a/demos/browser/app/meetingV2/videofilter/CircularCut.ts b/demos/browser/app/meetingV2/videofilter/CircularCut.ts index 7752fd180f..d219feb4e6 100644 --- a/demos/browser/app/meetingV2/videofilter/CircularCut.ts +++ b/demos/browser/app/meetingV2/videofilter/CircularCut.ts @@ -1,7 +1,11 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { CanvasVideoFrameBuffer, VideoFrameBuffer, VideoFrameProcessor } from 'amazon-chime-sdk-js'; +import { + CanvasVideoFrameBuffer, + VideoFrameBuffer, + VideoFrameProcessor, +} from 'amazon-chime-sdk-js'; /** * [[CircularCut]] is an implementation of {@link VideoFrameProcessor} for demonstration purpose. diff --git a/demos/browser/app/meetingV2/videofilter/ResizeProcessor.ts b/demos/browser/app/meetingV2/videofilter/ResizeProcessor.ts index 9049a78a71..0776a77a13 100644 --- a/demos/browser/app/meetingV2/videofilter/ResizeProcessor.ts +++ b/demos/browser/app/meetingV2/videofilter/ResizeProcessor.ts @@ -1,7 +1,11 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { CanvasVideoFrameBuffer, VideoFrameBuffer, VideoFrameProcessor } from 'amazon-chime-sdk-js'; +import { + CanvasVideoFrameBuffer, + VideoFrameBuffer, + VideoFrameProcessor, +} from 'amazon-chime-sdk-js'; /** * [[ResizeProcessor]] updates the input {@link VideoFrameBuffer} and resize given the display aspect ratio. diff --git a/demos/browser/app/meetingV2/videofilter/SegmentationProcessor.ts b/demos/browser/app/meetingV2/videofilter/SegmentationProcessor.ts deleted file mode 100644 index 133795000b..0000000000 --- a/demos/browser/app/meetingV2/videofilter/SegmentationProcessor.ts +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -// eslint-disable-next-line -declare var bodyPix: any; - -import { CanvasVideoFrameBuffer, VideoFrameBuffer, VideoFrameProcessor } from 'amazon-chime-sdk-js'; - -export default class SegmentationProcessor implements VideoFrameProcessor { - private targetCanvas: HTMLCanvasElement = document.createElement('canvas') as HTMLCanvasElement; - private canvasVideoFrameBuffer = new CanvasVideoFrameBuffer(this.targetCanvas); - private sourceWidth: number = 0; - private sourceHeight: number = 0; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private mask: any | undefined = undefined; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private model: any | undefined = undefined; - - constructor() {} - - async process(buffers: VideoFrameBuffer[]): Promise { - if (!this.model) { - this.model = await bodyPix.load(); - } - - const inputCanvas = buffers[0].asCanvasElement(); - if (!inputCanvas) { - throw new Error('buffer is already destroyed'); - } - - const frameWidth = inputCanvas.width; - const frameHeight = inputCanvas.height; - if (frameWidth === 0 || frameHeight === 0) { - return buffers; - } - - if (this.sourceWidth !== frameWidth || this.sourceHeight !== frameHeight) { - this.sourceWidth = frameWidth; - this.sourceHeight = frameHeight; - - // update target canvas size to match the frame size - this.targetCanvas.width = this.sourceWidth; - this.targetCanvas.height = this.sourceHeight; - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let predictions = await this.model.segmentPerson(inputCanvas); - if (predictions) { - const foregroundColor = { r: 255, g: 255, b: 255, a: 255 }; - const backgroundColor = { r: 0, g: 0, b: 0, a: 255 }; - this.mask = bodyPix.toMask(predictions, foregroundColor, backgroundColor, true); - } - - if (this.mask) { - bodyPix.drawMask(this.targetCanvas, inputCanvas as HTMLCanvasElement, this.mask); - buffers[0] = this.canvasVideoFrameBuffer; - } - predictions = undefined; - return buffers; - } - - async destroy(): Promise { - if (this.model) { - this.model.dispose(); - } - this.model = undefined; - } -} diff --git a/demos/browser/app/meetingV2/videofilter/SegmentationUtil.ts b/demos/browser/app/meetingV2/videofilter/SegmentationUtil.ts deleted file mode 100644 index 7c83398c1f..0000000000 --- a/demos/browser/app/meetingV2/videofilter/SegmentationUtil.ts +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { detect } from 'detect-browser'; - -const SEGMENTATION_DEPENDENCIES = [ - { - src: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@2.7.0/dist/tf.min.js', - integrity: 'sha384-uI1PW0SEa/QzAUuRQ6Bz5teBONsa9D0ZbVxMcM8mu4IjJ5msHyM7RRtZtL8LnSf3', - crossOrigin: 'anonymous', - }, - - { - src: 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-core@2.7.0/dist/tf-core.min.js', - integrity: 'sha384-DlI/SVdTGUBY5hi4h0p+nmC6V8i0FW5Nya/gYElz0L68HrSiXsBh+rqWcoZx3SXY', - crossOrigin: 'anonymous', - }, - - { - src: - 'https://cdn.jsdelivr.net/npm/@tensorflow/tfjs-backend-webgl@2.7.0/dist/tf-backend-webgl.min.js', - integrity: 'sha384-21TV9Kpzn8SF68G1U6nYN3qPZnb97F06JuW4v0FDDBzW+CUwv8GcKMR+BjnE7Vmm', - crossOrigin: 'anonymous', - }, - - { - src: 'https://cdn.jsdelivr.net/npm/@tensorflow-models/body-pix@2.0.5/dist/body-pix.min.js', - integrity: 'sha384-dPJ/sICXCqdh39bsuGLVFcOiRyeL/XcFrwiFrJq9oh7k1TCtsUKhX6UV2X4UuKU4', - crossOrigin: 'anonymous', - }, -]; - - -export async function loadBodyPixDependency(timeoutMs: number): Promise { - // the tf library loading order must be followed - for (let i = 0; i < SEGMENTATION_DEPENDENCIES.length; i++) { - const dep = SEGMENTATION_DEPENDENCIES[i]; - await new Promise((resolve, reject) => { - let script = document.createElement('script'); - const timer = setTimeout(() => { - reject(new Error(`Loading script ${dep.src} takes longer than ${timeoutMs}`)); - }, timeoutMs); - script.onload = function(_ev) { - clearTimeout(timer); - resolve(); - }; - script.src = dep.src; - script.integrity = dep.integrity; - script.crossOrigin = dep.crossOrigin; - document.body.appendChild(script); - }); - } -} - -export function platformCanSupportBodyPixWithoutDegradation(): boolean { - // https://blog.tensorflow.org/2019/11/updated-bodypix-2.html for more detail on performance - // https://github.com/tensorflow/tfjs/issues/3319 which results in firefox memory leak - const browser = detect(); - return browser.name === 'chrome' && /(android)/i.test(navigator.userAgent) === false; -}