configure output resolution from frontend + decoder black screen

apps/desktop/src/routes/editor/ConfigSidebar.tsx

@@ -703,7 +703,7 @@ export function ConfigSidebar() {
                     )
                   }
                   minValue={1}
-                  maxValue={2.5}
+                  maxValue={4.5}
                   step={0.001}
                 />
               </Field>

apps/desktop/src/routes/editor/Editor.tsx

@@ -21,7 +21,14 @@ import { createEventListenerMap } from "@solid-primitives/event-listener";
 import { convertFileSrc } from "@tauri-apps/api/core";
 
 import { events } from "~/utils/tauri";
-import { EditorContextProvider, FPS, useEditorContext } from "./context";
+import {
+  EditorContextProvider,
+  EditorInstanceContextProvider,
+  FPS,
+  OUTPUT_SIZE,
+  useEditorContext,
+  useEditorInstanceContext,
+} from "./context";
 import {
   Dialog,
   DialogContent,
@@ -30,10 +37,6 @@ import {
   Subfield,
   Toggle,
 } from "./ui";
-import {
-  EditorInstanceContextProvider,
-  useEditorInstanceContext,
-} from "./editorInstanceContext";
 import { Header } from "./Header";
 import { Player } from "./Player";
 import { ConfigSidebar } from "./ConfigSidebar";
@@ -83,10 +86,7 @@ function Inner() {
     events.renderFrameEvent.emit({
       frame_number: Math.max(Math.floor(time * FPS), 0),
       fps: FPS,
-      resolution_base: {
-        x: 1920,
-        y: 1080,
-      },
+      resolution_base: OUTPUT_SIZE,
     });
   }, 1000 / 60);
 

apps/desktop/src/routes/editor/Player.tsx

@@ -8,7 +8,7 @@ import { For, Show, Suspense, createEffect, createSignal } from "solid-js";
 import { reconcile } from "solid-js/store";
 
 import { type AspectRatio, commands } from "~/utils/tauri";
-import { FPS, useEditorContext } from "./context";
+import { FPS, OUTPUT_SIZE, useEditorContext } from "./context";
 import { ASPECT_RATIOS } from "./projectConfig";
 import {
   ComingSoonTooltip,
@@ -88,13 +88,13 @@ export function Player() {
         await commands.stopPlayback(videoId);
         setPlaybackTime(0);
         await commands.seekTo(videoId, 0);
-        await commands.startPlayback(videoId, FPS, { x: 1280, y: 720 });
+        await commands.startPlayback(videoId, FPS, OUTPUT_SIZE);
         setPlaying(true);
       } else if (playing()) {
         await commands.stopPlayback(videoId);
         setPlaying(false);
       } else {
-        await commands.startPlayback(videoId, FPS, { x: 1280, y: 720 });
+        await commands.startPlayback(videoId, FPS, OUTPUT_SIZE);
         setPlaying(true);
       }
     } catch (error) {

apps/desktop/src/routes/editor/context.ts

@@ -4,19 +4,27 @@ import { trackStore } from "@solid-primitives/deep";
 import { createEventListener } from "@solid-primitives/event-listener";
 import { createUndoHistory } from "@solid-primitives/history";
 import { debounce } from "@solid-primitives/scheduled";
-import { Accessor, createEffect, createSignal, on } from "solid-js";
+import {
+  Accessor,
+  createEffect,
+  createResource,
+  createSignal,
+  on,
+} from "solid-js";
 import { createStore, reconcile, unwrap } from "solid-js/store";
+import { createElementBounds } from "@solid-primitives/bounds";
 
 import type { PresetsStore } from "../../store";
 import {
   type ProjectConfiguration,
   type SerializedEditorInstance,
   type XY,
   commands,
+  events,
 } from "~/utils/tauri";
-import { useEditorInstanceContext } from "./editorInstanceContext";
 import { DEFAULT_PROJECT_CONFIG } from "./projectConfig";
-import { createElementBounds } from "@solid-primitives/bounds";
+import { createImageDataWS, createLazySignal } from "~/utils/socket";
+import { createPresets } from "~/utils/createPresets";
 
 export type CurrentDialog =
   | { type: "createPreset" }
@@ -26,7 +34,12 @@ export type CurrentDialog =
 
 export type DialogState = { open: false } | ({ open: boolean } & CurrentDialog);
 
-export const FPS = 60;
+export const FPS = 30;
+
+export const OUTPUT_SIZE = {
+  x: 1920,
+  y: 1080,
+};
 
 export const [EditorContextProvider, useEditorContext] = createContextProvider(
   (props: {
@@ -103,6 +116,45 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider(
   null!
 );
 
+export type FrameData = { width: number; height: number; data: ImageData };
+
+export const [EditorInstanceContextProvider, useEditorInstanceContext] =
+  createContextProvider((props: { videoId: string }) => {
+    const [latestFrame, setLatestFrame] = createLazySignal<{
+      width: number;
+      data: ImageData;
+    }>();
+
+    const [editorInstance] = createResource(async () => {
+      const instance = await commands.createEditorInstance(props.videoId);
+
+      const [ws, isConnected] = createImageDataWS(
+        instance.framesSocketUrl,
+        setLatestFrame
+      );
+
+      createEffect(() => {
+        if (isConnected()) {
+          events.renderFrameEvent.emit({
+            frame_number: Math.floor(0),
+            fps: FPS,
+            resolution_base: OUTPUT_SIZE,
+          });
+        }
+      });
+
+      return instance;
+    });
+
+    return {
+      editorInstance,
+      videoId: props.videoId,
+      latestFrame,
+      presets: createPresets(),
+      prettyName: () => editorInstance()?.prettyName ?? "Cap Recording",
+    };
+  }, null!);
+
 function createStoreHistory<T extends Static>(
   ...[state, setState]: ReturnType<typeof createStore<T>>
 ) {

apps/desktop/src/routes/recordings-overlay.tsx

@@ -38,7 +38,7 @@ import {
   checkIsUpgradedAndUpdate,
   canCreateShareableLink,
 } from "~/utils/plans";
-import { FPS } from "./editor/context";
+import { FPS, OUTPUT_SIZE } from "./editor/context";
 
 type MediaEntry = {
   path: string;
@@ -780,7 +780,7 @@ function createRecordingMutations(
             progress,
             false,
             FPS,
-            { x: 1920, y: 1080 }
+            OUTPUT_SIZE
           );
 
           // Show quick progress animation for existing video
@@ -906,7 +906,7 @@ function createRecordingMutations(
             progress,
             true, // Force re-render
             FPS,
-            { x: 1920, y: 1080 }
+            OUTPUT_SIZE
           );
 
           await commands.copyFileToPath(outputPath, savePath);
@@ -1042,7 +1042,7 @@ function createRecordingMutations(
             progress,
             false,
             FPS,
-            { x: 1920, y: 1080 }
+            OUTPUT_SIZE
           );
           console.log("Using existing rendered video");
 

crates/editor/src/editor_instance.rs

@@ -246,26 +246,27 @@ impl EditorInstance {
 
                 let segment = &self.segments[segment.unwrap_or(0) as usize];
 
-                let (screen_frame, camera_frame) = segment
+                if let Some((screen_frame, camera_frame)) = segment
                     .decoders
                     .get_frames(time as f32, !project.camera.hide)
-                    .await;
-
-                self.renderer
-                    .render_frame(
-                        screen_frame,
-                        camera_frame,
-                        project.background.source.clone(),
-                        ProjectUniforms::new(
-                            &self.render_constants,
-                            &project,
+                    .await
+                {
+                    self.renderer
+                        .render_frame(
+                            screen_frame,
+                            camera_frame,
+                            project.background.source.clone(),
+                            ProjectUniforms::new(
+                                &self.render_constants,
+                                &project,
+                                time as f32,
+                                resolution_base,
+                            ),
                             time as f32,
                             resolution_base,
-                        ),
-                        time as f32,
-                        resolution_base,
-                    )
-                    .await;
+                        )
+                        .await;
+                }
             }
         })
     }

crates/editor/src/playback.rs

@@ -94,7 +94,8 @@ impl Playback {
                         _ = stop_rx.changed() => {
                            break;
                         },
-                        (screen_frame, camera_frame) = segment.decoders.get_frames(time as f32, !project.camera.hide) => {
+                        data = segment.decoders.get_frames(time as f32, !project.camera.hide) => {
+                            if let Some((screen_frame, camera_frame)) = data {
                                 let uniforms = ProjectUniforms::new(&self.render_constants, &project, time as f32, resolution_base);
 
                                 self
@@ -108,6 +109,7 @@ impl Playback {
                                         resolution_base
                                     )
                                     .await;
+                            }
                         }
                         else => {
                         }
@@ -126,8 +128,6 @@ impl Playback {
                 frame_number += 1;
             }
 
-            println!("stopped playback");
-
             stop_tx.send(true).ok();
 
             event_tx.send(PlaybackEvent::Stop).ok();

crates/media/src/encoders/h264_avassetwriter.rs

@@ -16,6 +16,7 @@ pub struct H264AVAssetWriterEncoder {
 
 impl H264AVAssetWriterEncoder {
     pub fn init(tag: &'static str, config: VideoInfo, output: Output) -> Result<Self, MediaError> {
+        let fps = config.frame_rate.0 as f32 / config.frame_rate.1 as f32;
         let Output::File(destination) = output;
 
         let mut asset_writer = av::AssetWriter::with_url_and_file_type(
@@ -47,11 +48,14 @@ impl H264AVAssetWriterEncoder {
             ns::Number::with_u32(config.height).as_id_ref(),
         );
 
+        let bitrate = config.width as f32 * config.height as f32 / (1920.0 * 1080.0) * 10_000_000.0
+            + fps / 30.0 * 4_000_000.0;
+
         output_settings.insert(
             av::video_settings_keys::compression_props(),
             ns::Dictionary::with_keys_values(
                 &[unsafe { AVVideoAverageBitRateKey }],
-                &[ns::Number::with_u32(10_000_000).as_id_ref()],
+                &[ns::Number::with_f32(bitrate).as_id_ref()],
             )
             .as_id_ref(),
         );