better dynamic resolution control

CapSoftware · Jan 15, 2025 · 0c7eb34 · 0c7eb34
1 parent a435e1d
commit 0c7eb34
Show file tree

Hide file tree

Showing 17 changed files with 118 additions and 89 deletions.
diff --git a/apps/cli/src/main.rs b/apps/cli/src/main.rs
@@ -121,6 +121,7 @@ async fn main() -> Result<(), String> {
                 render_constants,
                 &segments,
                 fps,
+                XY::new(1920, 1080),
             )
             .unwrap();
 

diff --git a/apps/desktop/src-tauri/src/export.rs b/apps/desktop/src-tauri/src/export.rs
@@ -2,7 +2,7 @@ use crate::{
     general_settings::GeneralSettingsStore, get_video_metadata, upsert_editor_instance,
     windows::ShowCapWindow, RenderProgress, VideoRecordingMetadata, VideoType,
 };
-use cap_project::ProjectConfiguration;
+use cap_project::{ProjectConfiguration, XY};
 use std::path::PathBuf;
 use tauri::AppHandle;
 
@@ -15,6 +15,7 @@ pub async fn export_video(
     progress: tauri::ipc::Channel<RenderProgress>,
     force: bool,
     fps: u32,
+    resolution_base: XY<u32>,
 ) -> Result<PathBuf, String> {
     let screen_metadata =
         match get_video_metadata(app.clone(), video_id.clone(), Some(VideoType::Screen)).await {
@@ -84,6 +85,7 @@ pub async fn export_video(
         editor_instance.render_constants.clone(),
         &editor_instance.segments,
         fps,
+        resolution_base,
     )
     .map_err(|e| {
         sentry::capture_message(&e.to_string(), sentry::Level::Error);

diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs
@@ -25,6 +25,7 @@ use cap_media::feeds::{AudioInputFeed, AudioInputSamplesSender};
 use cap_media::frame_ws::WSFrame;
 use cap_media::sources::CaptureScreen;
 use cap_media::{feeds::CameraFeed, sources::ScreenCaptureTarget};
+use cap_project::XY;
 use cap_project::{Content, ProjectConfiguration, RecordingMeta, Resolution, SharingMeta};
 use cap_recording::RecordingOptions;
 use cap_rendering::ProjectRecordings;
@@ -850,6 +851,7 @@ async fn open_file_path(_app: AppHandle, path: PathBuf) -> Result<(), String> {
 struct RenderFrameEvent {
     frame_number: u32,
     fps: u32,
+    resolution_base: XY<u32>,
 }
 
 #[derive(Serialize, specta::Type, tauri_specta::Event, Debug, Clone)]
@@ -867,10 +869,10 @@ impl EditorStateChanged {
 
 #[tauri::command]
 #[specta::specta]
-async fn start_playback(app: AppHandle, video_id: String, fps: u32) {
+async fn start_playback(app: AppHandle, video_id: String, fps: u32, resolution_base: XY<u32>) {
     upsert_editor_instance(&app, video_id)
         .await
-        .start_playback(fps)
+        .start_playback(fps, resolution_base)
         .await
 }
 
@@ -2270,7 +2272,11 @@ async fn create_editor_instance_impl(app: &AppHandle, video_id: String) -> Arc<E
         let preview_tx = instance.preview_tx.clone();
         move |e| {
             preview_tx
-                .send(Some((e.payload.frame_number, e.payload.fps)))
+                .send(Some((
+                    e.payload.frame_number,
+                    e.payload.fps,
+                    e.payload.resolution_base,
+                )))
                 .ok();
         }
     });

diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs
@@ -17,7 +17,7 @@ use cap_flags::FLAGS;
 use cap_media::feeds::CameraFeed;
 use cap_media::sources::{CaptureScreen, CaptureWindow};
 use cap_project::{
-    Content, ProjectConfiguration, TimelineConfiguration, TimelineSegment, ZoomSegment,
+    Content, ProjectConfiguration, TimelineConfiguration, TimelineSegment, ZoomSegment, XY,
 };
 use cap_recording::CompletedRecording;
 use cap_rendering::ProjectRecordings;
@@ -253,6 +253,7 @@ pub async fn stop_recording(app: AppHandle, state: MutableState<'_, App>) -> Res
                         tauri::ipc::Channel::new(|_| Ok(())),
                         true,
                         completed_recording.meta.content.max_fps(),
+                        XY::new(1920, 1080),
                     )
                     .await
                     .ok();
@@ -369,8 +370,6 @@ fn project_config_from_recording(
                 })
                 .collect(),
             zoom_segments: generate_zoom_segments_from_clicks(&completed_recording, &recordings),
-            output_width: None,
-            output_height: None,
         }),
         ..Default::default()
     }

diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx
@@ -83,6 +83,10 @@ function Inner() {
     events.renderFrameEvent.emit({
       frame_number: Math.max(Math.floor(time * FPS), 0),
       fps: FPS,
+      resolution_base: {
+        x: 1920,
+        y: 1080,
+      },
     });
   }, 1000 / 60);
 

diff --git a/apps/desktop/src/routes/editor/Header.tsx b/apps/desktop/src/routes/editor/Header.tsx
@@ -168,29 +168,16 @@ export function Header() {
     };
 
     try {
-      const updatedProject = {
-        ...project,
-        timeline: project.timeline
-          ? {
-              ...project.timeline,
-              outputWidth: selectedResolution().width,
-              outputHeight: selectedResolution().height,
-            }
-          : {
-              segments: [],
-              zoomSegments: [],
-              outputWidth: selectedResolution().width,
-              outputHeight: selectedResolution().height,
-            },
-        fps: selectedFps(),
-      };
-
       const videoPath = await commands.exportVideo(
         videoId,
-        updatedProject,
+        project,
         progress,
         true,
-        selectedFps()
+        selectedFps(),
+        {
+          x: selectedResolution().width,
+          y: selectedResolution().height,
+        }
       );
       await commands.copyFileToPath(videoPath, path);
 
@@ -574,13 +561,6 @@ function ShareButton(props: ShareButtonProps) {
         });
 
         console.log("Starting actual upload...");
-        const projectConfig = {
-          ...(project ?? presets.getDefaultConfig() ?? DEFAULT_PROJECT_CONFIG),
-          outputResolution: {
-            width: props.selectedResolution().width,
-            height: props.selectedResolution().height,
-          },
-        };
 
         setProgressState({
           type: "uploading",
@@ -623,10 +603,14 @@ function ShareButton(props: ShareButtonProps) {
 
         await commands.exportVideo(
           videoId,
-          projectConfig,
+          project,
           progress,
           true,
-          props.selectedFps()
+          props.selectedFps(),
+          {
+            x: props.selectedResolution().width,
+            y: props.selectedResolution().height,
+          }
         );
 
         // Now proceed with upload

diff --git a/apps/desktop/src/routes/editor/Player.tsx b/apps/desktop/src/routes/editor/Player.tsx
@@ -88,13 +88,13 @@ export function Player() {
         await commands.stopPlayback(videoId);
         setPlaybackTime(0);
         await commands.seekTo(videoId, 0);
-        await commands.startPlayback(videoId, FPS);
+        await commands.startPlayback(videoId, FPS, { x: 1280, y: 720 });
         setPlaying(true);
       } else if (playing()) {
         await commands.stopPlayback(videoId);
         setPlaying(false);
       } else {
-        await commands.startPlayback(videoId, FPS);
+        await commands.startPlayback(videoId, FPS, { x: 1280, y: 720 });
         setPlaying(true);
       }
     } catch (error) {

diff --git a/apps/desktop/src/routes/editor/Timeline.tsx b/apps/desktop/src/routes/editor/Timeline.tsx
@@ -82,6 +82,7 @@ export function Timeline() {
                 recordingSegment: null,
               },
             ],
+            zoomSegments: [],
           };
         })
       );

diff --git a/apps/desktop/src/routes/editor/editorInstanceContext.ts b/apps/desktop/src/routes/editor/editorInstanceContext.ts
@@ -33,6 +33,10 @@ export const [EditorInstanceContextProvider, useEditorInstanceContext] =
           events.renderFrameEvent.emit({
             frame_number: Math.floor(0),
             fps: FPS,
+            resolution_base: {
+              x: 1920,
+              y: 1080,
+            },
           });
         }
       });

diff --git a/apps/desktop/src/routes/recordings-overlay.tsx b/apps/desktop/src/routes/recordings-overlay.tsx
@@ -779,7 +779,8 @@ function createRecordingMutations(
             presets.getDefaultConfig() ?? DEFAULT_PROJECT_CONFIG,
             progress,
             false,
-            FPS
+            FPS,
+            { x: 1920, y: 1080 }
           );
 
           // Show quick progress animation for existing video
@@ -904,7 +905,8 @@ function createRecordingMutations(
             presets.getDefaultConfig() ?? DEFAULT_PROJECT_CONFIG,
             progress,
             true, // Force re-render
-            FPS
+            FPS,
+            { x: 1920, y: 1080 }
           );
 
           await commands.copyFileToPath(outputPath, savePath);
@@ -1039,7 +1041,8 @@ function createRecordingMutations(
             presets.getDefaultConfig() ?? DEFAULT_PROJECT_CONFIG,
             progress,
             false,
-            FPS
+            FPS,
+            { x: 1920, y: 1080 }
           );
           console.log("Using existing rendered video");
 

diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts
@@ -53,8 +53,8 @@ async focusCapturesPanel() : Promise<void> {
 async getCurrentRecording() : Promise<JsonValue<RecordingInfo | null>> {
     return await TAURI_INVOKE("get_current_recording");
 },
-async exportVideo(videoId: string, project: ProjectConfiguration, progress: TAURI_CHANNEL<RenderProgress>, force: boolean, fps: number) : Promise<string> {
-    return await TAURI_INVOKE("export_video", { videoId, project, progress, force, fps });
+async exportVideo(videoId: string, project: ProjectConfiguration, progress: TAURI_CHANNEL<RenderProgress>, force: boolean, fps: number, resolutionBase: XY<number>) : Promise<string> {
+    return await TAURI_INVOKE("export_video", { videoId, project, progress, force, fps, resolutionBase });
 },
 async copyFileToPath(src: string, dst: string) : Promise<null> {
     return await TAURI_INVOKE("copy_file_to_path", { src, dst });
@@ -74,8 +74,8 @@ async getVideoMetadata(videoId: string, videoType: VideoType | null) : Promise<V
 async createEditorInstance(videoId: string) : Promise<SerializedEditorInstance> {
     return await TAURI_INVOKE("create_editor_instance", { videoId });
 },
-async startPlayback(videoId: string, fps: number) : Promise<void> {
-    await TAURI_INVOKE("start_playback", { videoId, fps });
+async startPlayback(videoId: string, fps: number, resolutionBase: XY<number>) : Promise<void> {
+    await TAURI_INVOKE("start_playback", { videoId, fps, resolutionBase });
 },
 async stopPlayback(videoId: string) : Promise<void> {
     await TAURI_INVOKE("stop_playback", { videoId });
@@ -267,7 +267,7 @@ export type RecordingOptions = { captureTarget: ScreenCaptureTarget; cameraLabel
 export type RecordingOptionsChanged = null
 export type RecordingStarted = null
 export type RecordingStopped = { path: string }
-export type RenderFrameEvent = { frame_number: number; fps: number }
+export type RenderFrameEvent = { frame_number: number; fps: number; resolution_base: XY<number> }
 export type RenderProgress = { type: "Starting"; total_frames: number } | { type: "EstimatedTotalFrames"; total_frames: number } | { type: "FrameRendered"; current_frame: number }
 export type RequestNewScreenshot = null
 export type RequestOpenSettings = { page: string }
@@ -282,7 +282,7 @@ export type SerializedEditorInstance = { framesSocketUrl: string; recordingDurat
 export type SharingMeta = { id: string; link: string }
 export type ShowCapWindow = "Setup" | "Main" | { Settings: { page: string | null } } | { Editor: { project_id: string } } | "PrevRecordings" | "WindowCaptureOccluder" | { Camera: { ws_port: number } } | { InProgressRecording: { position: [number, number] | null } } | "Upgrade" | "SignIn"
 export type SingleSegment = { display: Display; camera?: CameraMeta | null; audio?: AudioMeta | null; cursor?: string | null }
-export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments: ZoomSegment[]; outputWidth?: number | null; outputHeight?: number | null }
+export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments: ZoomSegment[] }
 export type TimelineSegment = { recordingSegment: number | null; timescale: number; start: number; end: number }
 export type UploadMode = { Initial: { pre_created_video: PreCreatedVideo | null } } | "Reupload"
 export type UploadProgress = { stage: string; progress: number; message: string }

diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs
@@ -1,7 +1,7 @@
 use std::{sync::Arc, time::Instant};
 
 use cap_media::frame_ws::WSFrame;
-use cap_project::{BackgroundSource, ProjectConfiguration, RecordingMeta};
+use cap_project::{BackgroundSource, ProjectConfiguration, RecordingMeta, XY};
 use cap_rendering::{
     decoder::DecodedFrame, produce_frame, ProjectRecordings, ProjectUniforms, RenderVideoConstants,
 };
@@ -18,6 +18,7 @@ pub enum RendererMessage {
         uniforms: ProjectUniforms,
         time: f32, // Add this field
         finished: oneshot::Sender<()>,
+        resolution_base: XY<u32>,
     },
     Stop {
         finished: oneshot::Sender<()>,
@@ -80,6 +81,7 @@ impl Renderer {
                         uniforms,
                         time,
                         finished,
+                        resolution_base,
                     } => {
                         if let Some(task) = frame_task.as_ref() {
                             if task.is_finished() {
@@ -102,6 +104,7 @@ impl Renderer {
                                 &uniforms,
                                 time,
                                 total_frames,
+                                resolution_base,
                             )
                             .await
                             .unwrap();
@@ -145,6 +148,7 @@ impl RendererHandle {
         background: BackgroundSource,
         uniforms: ProjectUniforms,
         time: f32, // Add this parameter
+        resolution_base: XY<u32>,
     ) {
         let (finished_tx, finished_rx) = oneshot::channel();
 
@@ -155,6 +159,7 @@ impl RendererHandle {
             uniforms,
             time, // Pass the time
             finished: finished_tx,
+            resolution_base,
         })
         .await;
 

diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs
@@ -173,7 +173,7 @@ impl EditorInstance {
         (self.on_state_change)(&state);
     }
 
-    pub async fn start_playback(self: Arc<Self>, fps: u32) {
+    pub async fn start_playback(self: Arc<Self>, fps: u32, resolution_base: XY<u32>) {
         let (mut handle, prev) = {
             let Ok(mut state) = self.state.try_lock() else {
                 return;
@@ -188,7 +188,7 @@ impl EditorInstance {
                 start_frame_number,
                 project: self.project_config.0.subscribe(),
             }
-            .start(fps)
+            .start(fps, resolution_base)
             .await;
 
             let prev = state.playback_task.replace(playback_handle.clone());
@@ -223,12 +223,13 @@ impl EditorInstance {
 
     fn spawn_preview_renderer(
         self: Arc<Self>,
-        mut preview_rx: watch::Receiver<Option<(u32, u32)>>,
+        mut preview_rx: watch::Receiver<Option<(u32, u32, XY<u32>)>>,
     ) -> tokio::task::JoinHandle<()> {
         tokio::spawn(async move {
             loop {
                 preview_rx.changed().await.unwrap();
-                let Some((frame_number, fps)) = *preview_rx.borrow().deref() else {
+                let Some((frame_number, fps, resolution_base)) = *preview_rx.borrow().deref()
+                else {
                     continue;
                 };
 
@@ -255,8 +256,14 @@ impl EditorInstance {
                         screen_frame,
                         camera_frame,
                         project.background.source.clone(),
-                        ProjectUniforms::new(&self.render_constants, &project, time as f32),
-                        time as f32, // Add the time parameter
+                        ProjectUniforms::new(
+                            &self.render_constants,
+                            &project,
+                            time as f32,
+                            resolution_base,
+                        ),
+                        time as f32,
+                        resolution_base,
                     )
                     .await;
             }
@@ -283,7 +290,7 @@ impl Drop for EditorInstance {
     }
 }
 
-type PreviewFrameInstruction = (u32, u32);
+type PreviewFrameInstruction = (u32, u32, XY<u32>);
 
 pub struct EditorState {
     pub playhead_position: u32,
-Original file line number
+Diff line change
@@ Expand Up / @@ -82,6 +82,7 @@ export function Timeline() { @@
                     recordingSegment: null,
                   },
                 ],
+                zoomSegments: [],
               };
             })
           );
@@ Expand Down @@