Added robot-to-operator audio streaming (#62)

src/pages/operator/css/AudioControl.css

@@ -0,0 +1,15 @@
+.audioControlContainer {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  text-align: center;
+}
+
+.audioControl {
+  display: none;
+}
+
+.audioControlButton {
+  border-radius: 50%;
+  aspect-ratio: 1;
+}

src/pages/operator/tsx/Operator.tsx

@@ -1,4 +1,5 @@
 import React from "react";
+import { AudioControl } from "./static_components/AudioControl";
 import { SpeedControl } from "./static_components/SpeedControl";
 import { LayoutArea } from "./static_components/LayoutArea";
 import { CustomizeButton } from "./static_components/CustomizeButton";
@@ -289,6 +290,7 @@ export const Operator = (props: {
           showActive
           placement="bottom"
         />
+        <AudioControl remoteStreams={remoteStreams} />
         <SpeedControl
           scale={velocityScale}
           onChange={(newScale: number) => {

src/pages/operator/tsx/index.tsx

@@ -115,17 +115,19 @@ root = createRoot(container!);
 
 /** Handle when the WebRTC connection adds a new track on a camera video stream. */
 function handleRemoteTrackAdded(event: RTCTrackEvent) {
-  console.log("Remote track added.");
   const track = event.track;
   const stream = event.streams[0];
-  console.log(stream.getVideoTracks()[0].getConstraints());
+  let streamName = connection.cameraInfo[stream.id];
+  console.log("Adding remote track", streamName);
+  if (streamName != "audio") {
+    console.log(stream.getVideoTracks()[0].getConstraints());
+  }
   console.log("got track id=" + track.id, track);
   if (stream) {
     console.log("stream id=" + stream.id, stream);
   }
   console.log("OPERATOR: adding remote tracks");
 
-  let streamName = connection.cameraInfo[stream.id];
   allRemoteStreams.set(streamName, { track: track, stream: stream });
 }
 

src/pages/operator/tsx/static_components/AudioControl.tsx

@@ -0,0 +1,60 @@
+import "operator/css/AudioControl.css";
+import { className, RemoteStream } from "shared/util";
+import React from "react";
+
+/**Props for {@link AudioControl} */
+type AudioControlProps = {
+  /** Remote robot video streams */
+  remoteStreams: Map<string, RemoteStream>;
+};
+
+export const AudioControl = (props: AudioControlProps) => {
+  // Create the audio element for listening to audio from the robot
+  const audioRef = React.useRef<HTMLAudioElement>(null);
+
+  // Keep track of whether the audio stream is muted or not.
+  // NOTE: Audio must start muted for AutoPlay to work (see below)
+  // https://developer.mozilla.org/en-US/docs/Web/Media/Autoplay_guide#autoplay_availability
+  const [muted, setMuted] = React.useState(true);
+
+  // Assign the source of the audio element to the audio stream from the robot
+  const stream = React.useMemo(() => {
+    if (props.remoteStreams.has("audio")) {
+      return props.remoteStreams.get("audio")?.stream;
+    } else {
+      return null;
+    }
+  }, [props.remoteStreams]);
+  React.useEffect(() => {
+    if (!audioRef?.current) return;
+    if (stream) {
+      audioRef.current.srcObject = stream;
+    }
+  }, [audioRef.current, stream]);
+
+  // Callback for when the user presses the button to toggle mute
+  const toggleMute = React.useCallback(() => {
+    setMuted((prev) => !prev);
+  }, [setMuted]);
+
+  return (
+    <div className="audioControlContainer">
+      <audio
+        id="audio"
+        ref={audioRef}
+        className="audioControl"
+        autoPlay
+        muted={muted}
+      >
+        Robot audio playback is not supported on this browser
+      </audio>
+      <button className="button audioControlButton" onClick={toggleMute}>
+        {muted ? (
+          <span className="material-icons">volume_off</span>
+        ) : (
+          <span className="material-icons">volume_up</span>
+        )}
+      </button>
+    </div>
+  );
+};

src/pages/robot/tsx/audiostreams.tsx

@@ -0,0 +1,19 @@
+import React from "react";
+
+type AudioStreamProps = {};
+
+export class AudioStream extends React.Component<AudioStreamProps> {
+  outputAudioStream?: MediaStream;
+
+  constructor(props: AudioStreamProps) {
+    super(props);
+    this.outputAudioStream = new MediaStream();
+  }
+
+  async start() {
+    this.outputAudioStream = await navigator.mediaDevices.getUserMedia({
+      audio: true,
+      video: false,
+    });
+  }
+}

src/pages/robot/tsx/index.tsx

@@ -7,6 +7,7 @@ import {
   navigationProps,
   realsenseProps,
   gripperProps,
+  audioProps,
   WebRTCMessage,
   ValidJointStateDict,
   ValidJointStateMessage,
@@ -22,6 +23,7 @@ import {
   BatteryVoltageMessage,
 } from "shared/util";
 import { AllVideoStreamComponent, VideoStream } from "./videostreams";
+import { AudioStream } from "./audiostreams";
 import ROSLIB from "roslib";
 import {
   HasBetaTeleopKitMessage,
@@ -46,6 +48,7 @@ export let connection: WebRTCConnection;
 export let navigationStream = new VideoStream(navigationProps);
 export let realsenseStream = new VideoStream(realsenseProps);
 export let gripperStream = new VideoStream(gripperProps);
+export let audioStream = new AudioStream(audioProps);
 // let occupancyGrid: ROSOccupancyGrid | undefined;
 
 connection = new WebRTCConnection({
@@ -75,6 +78,8 @@ robot.connect().then(() => {
   });
   gripperStream.start();
 
+  audioStream.start();
+
   robot.getOccupancyGrid();
   robot.getJointLimits();
 
@@ -101,6 +106,11 @@ function handleSessionStart() {
     .getTracks()
     .forEach((track) => connection.addTrack(track, stream, "gripper"));
 
+  stream = audioStream.outputAudioStream!;
+  stream
+    .getTracks()
+    .forEach((track) => connection.addTrack(track, stream, "audio"));
+
   connection.openDataChannels();
 }
 

src/shared/util.tsx

@@ -349,6 +349,10 @@ export const gripperProps = {
   streamName: "gripper",
 };
 
+// audioProps are empty for now, but are included in case we want to customize
+// the audio stream in the future.
+export const audioProps = {};
+
 export interface VideoProps {
   topicName: string;
   callback: (message: ROSCompressedImage) => void;

start_robot_browser.js

@@ -27,11 +27,15 @@ if (process.argv.length > 2) {
   const browser = await firefox.launch({
     headless: true, // default is true
     defaultViewport: null,
+    // NOTE: I (Amal) believe the below args are unnecessary now that we've switched from Chromium to Firefox.
     args: [
       "--use-fake-ui-for-media-stream", //gives permission to access the robot's cameras and microphones (cleaner and simpler than changing the user directory)
       "--disable-features=WebRtcHideLocalIpsWithMdns", // Disables mDNS hostname use in local network P2P discovery. Necessary for enterprise networks that don't forward mDNS traffic
       "--ignore-certificate-errors",
     ],
+    firefoxUserPrefs: {
+      "permissions.default.microphone": 1, // Give permission to access the robot's microphone
+    },
   });
 
   const context = await browser.newContext({ ignoreHTTPSErrors: true }); // avoid ERR_CERT_COMMON_NAME_INVALID