diff --git a/apps/cli/src/main.rs b/apps/cli/src/main.rs index 7d5156a2..815525c0 100644 --- a/apps/cli/src/main.rs +++ b/apps/cli/src/main.rs @@ -121,6 +121,7 @@ async fn main() -> Result<(), String> { render_constants, &segments, fps, + XY::new(1920, 1080), ) .unwrap(); diff --git a/apps/desktop/src-tauri/src/export.rs b/apps/desktop/src-tauri/src/export.rs index 7235822d..0189b235 100644 --- a/apps/desktop/src-tauri/src/export.rs +++ b/apps/desktop/src-tauri/src/export.rs @@ -2,7 +2,7 @@ use crate::{ general_settings::GeneralSettingsStore, get_video_metadata, upsert_editor_instance, windows::ShowCapWindow, RenderProgress, VideoRecordingMetadata, VideoType, }; -use cap_project::ProjectConfiguration; +use cap_project::{ProjectConfiguration, XY}; use std::path::PathBuf; use tauri::AppHandle; @@ -15,6 +15,7 @@ pub async fn export_video( progress: tauri::ipc::Channel, force: bool, fps: u32, + resolution_base: XY, ) -> Result { let screen_metadata = match get_video_metadata(app.clone(), video_id.clone(), Some(VideoType::Screen)).await { @@ -84,6 +85,7 @@ pub async fn export_video( editor_instance.render_constants.clone(), &editor_instance.segments, fps, + resolution_base, ) .map_err(|e| { sentry::capture_message(&e.to_string(), sentry::Level::Error); diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 23544acd..30ed6e00 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -25,6 +25,7 @@ use cap_media::feeds::{AudioInputFeed, AudioInputSamplesSender}; use cap_media::frame_ws::WSFrame; use cap_media::sources::CaptureScreen; use cap_media::{feeds::CameraFeed, sources::ScreenCaptureTarget}; +use cap_project::XY; use cap_project::{Content, ProjectConfiguration, RecordingMeta, Resolution, SharingMeta}; use cap_recording::RecordingOptions; use cap_rendering::ProjectRecordings; @@ -850,6 +851,7 @@ async fn open_file_path(_app: AppHandle, path: PathBuf) -> Result<(), String> { struct RenderFrameEvent { frame_number: u32, fps: u32, + resolution_base: XY, } #[derive(Serialize, specta::Type, tauri_specta::Event, Debug, Clone)] @@ -867,10 +869,10 @@ impl EditorStateChanged { #[tauri::command] #[specta::specta] -async fn start_playback(app: AppHandle, video_id: String, fps: u32) { +async fn start_playback(app: AppHandle, video_id: String, fps: u32, resolution_base: XY) { upsert_editor_instance(&app, video_id) .await - .start_playback(fps) + .start_playback(fps, resolution_base) .await } @@ -2270,7 +2272,11 @@ async fn create_editor_instance_impl(app: &AppHandle, video_id: String) -> Arc) -> Res return Err("Recording not in progress".to_string())?; }; - let now = Instant::now(); let completed_recording = current_recording.stop().await.map_err(|e| e.to_string())?; if let Some(window) = CapWindowId::InProgressRecording.get(&app) { @@ -253,6 +252,7 @@ pub async fn stop_recording(app: AppHandle, state: MutableState<'_, App>) -> Res tauri::ipc::Channel::new(|_| Ok(())), true, completed_recording.meta.content.max_fps(), + XY::new(1920, 1080), ) .await .ok(); diff --git a/apps/desktop/src/routes/editor/ConfigSidebar.tsx b/apps/desktop/src/routes/editor/ConfigSidebar.tsx index 7f144626..e5a0eb37 100644 --- a/apps/desktop/src/routes/editor/ConfigSidebar.tsx +++ b/apps/desktop/src/routes/editor/ConfigSidebar.tsx @@ -703,7 +703,7 @@ export function ConfigSidebar() { ) } minValue={1} - maxValue={2.5} + maxValue={4.5} step={0.001} /> diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index a2d3afe6..673b719a 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -21,7 +21,14 @@ import { createEventListenerMap } from "@solid-primitives/event-listener"; import { convertFileSrc } from "@tauri-apps/api/core"; import { events } from "~/utils/tauri"; -import { EditorContextProvider, FPS, useEditorContext } from "./context"; +import { + EditorContextProvider, + EditorInstanceContextProvider, + FPS, + OUTPUT_SIZE, + useEditorContext, + useEditorInstanceContext, +} from "./context"; import { Dialog, DialogContent, @@ -30,10 +37,6 @@ import { Subfield, Toggle, } from "./ui"; -import { - EditorInstanceContextProvider, - useEditorInstanceContext, -} from "./editorInstanceContext"; import { Header } from "./Header"; import { Player } from "./Player"; import { ConfigSidebar } from "./ConfigSidebar"; @@ -83,6 +86,7 @@ function Inner() { events.renderFrameEvent.emit({ frame_number: Math.max(Math.floor(time * FPS), 0), fps: FPS, + resolution_base: OUTPUT_SIZE, }); }, 1000 / 60); diff --git a/apps/desktop/src/routes/editor/Header.tsx b/apps/desktop/src/routes/editor/Header.tsx index 0b778382..beaada1b 100644 --- a/apps/desktop/src/routes/editor/Header.tsx +++ b/apps/desktop/src/routes/editor/Header.tsx @@ -7,31 +7,76 @@ import { batch, createEffect, createResource, + createSignal, onCleanup, onMount, } from "solid-js"; import { type as ostype } from "@tauri-apps/plugin-os"; import { Tooltip } from "@kobalte/core"; +import { Select as KSelect } from "@kobalte/core/select"; +import { createMutation } from "@tanstack/solid-query"; +import { getRequestEvent } from "solid-js/web"; +import { save } from "@tauri-apps/plugin-dialog"; +import { Channel } from "@tauri-apps/api/core"; +import type { UnlistenFn } from "@tauri-apps/api/event"; +import { getCurrentWindow, ProgressBarStatus } from "@tauri-apps/api/window"; import { type RenderProgress, commands } from "~/utils/tauri"; -import { canCreateShareableLink } from "~/utils/plans"; - +import { + canCreateShareableLink, + checkIsUpgradedAndUpdate, +} from "~/utils/plans"; import { FPS, useEditorContext } from "./context"; -import { Dialog, DialogContent } from "./ui"; +import { + Dialog, + DialogContent, + MenuItem, + MenuItemList, + PopperContent, + topLeftAnimateClasses, +} from "./ui"; +import { DEFAULT_PROJECT_CONFIG } from "./projectConfig"; import { type ProgressState, progressState, setProgressState, } from "~/store/progress"; - import { events } from "~/utils/tauri"; import Titlebar from "~/components/titlebar/Titlebar"; import { initializeTitlebar, setTitlebar } from "~/utils/titlebar-state"; -import type { UnlistenFn } from "@tauri-apps/api/event"; -import { getCurrentWindow, ProgressBarStatus } from "@tauri-apps/api/window"; + +type ResolutionOption = { + label: string; + value: string; + width: number; + height: number; +}; + +const RESOLUTION_OPTIONS: ResolutionOption[] = [ + { label: "720p", value: "720p", width: 1280, height: 720 }, + { label: "1080p", value: "1080p", width: 1920, height: 1080 }, + { label: "4K", value: "4k", width: 3840, height: 2160 }, +]; + +const FPS_OPTIONS = [ + { label: "30 FPS", value: 30 }, + { label: "60 FPS", value: 60 }, +] satisfies Array<{ label: string; value: number }>; export function Header() { const currentWindow = getCurrentWindow(); + const { videoId, project, prettyName } = useEditorContext(); + + const [showExportOptions, setShowExportOptions] = createSignal(false); + const [selectedFps, setSelectedFps] = createSignal( + Number(localStorage.getItem("cap-export-fps")) || 30 + ); + const [selectedResolution, setSelectedResolution] = + createSignal( + RESOLUTION_OPTIONS.find( + (opt) => opt.value === localStorage.getItem("cap-export-resolution") + ) || RESOLUTION_OPTIONS[0] + ); let unlistenTitlebar: UnlistenFn | undefined; onMount(async () => { @@ -39,6 +84,12 @@ export function Header() { }); onCleanup(() => unlistenTitlebar?.()); + // Save settings when they change + createEffect(() => { + localStorage.setItem("cap-export-fps", selectedFps().toString()); + localStorage.setItem("cap-export-resolution", selectedResolution().value); + }); + createEffect(() => { const state = progressState; if (state === undefined || state.type === "idle") { @@ -61,6 +112,91 @@ export function Header() { currentWindow.setProgressBar({ progress: Math.round(percentage) }); }); + const exportWithSettings = async () => { + setShowExportOptions(false); + + const path = await save({ + filters: [{ name: "mp4 filter", extensions: ["mp4"] }], + defaultPath: `~/Desktop/${prettyName()}.mp4`, + }); + if (!path) return; + + setProgressState({ + type: "saving", + progress: 0, + renderProgress: 0, + totalFrames: 0, + message: "Preparing to render...", + mediaPath: path, + stage: "rendering", + }); + + const progress = new Channel(); + progress.onmessage = (p) => { + if (p.type === "FrameRendered" && progressState.type === "saving") { + const percentComplete = Math.min( + Math.round( + (p.current_frame / (progressState.totalFrames || 1)) * 100 + ), + 100 + ); + + setProgressState({ + ...progressState, + renderProgress: p.current_frame, + message: `Rendering video - ${percentComplete}%`, + }); + + // If rendering is complete, update to finalizing state + if (percentComplete === 100) { + setProgressState({ + ...progressState, + message: "Finalizing export...", + }); + } + } + if ( + p.type === "EstimatedTotalFrames" && + progressState.type === "saving" + ) { + setProgressState({ + ...progressState, + totalFrames: p.total_frames, + message: "Starting render...", + }); + } + }; + + try { + const videoPath = await commands.exportVideo( + videoId, + project, + progress, + true, + selectedFps(), + { + x: selectedResolution().width, + y: selectedResolution().height, + } + ); + await commands.copyFileToPath(videoPath, path); + + setProgressState({ + type: "saving", + progress: 100, + message: "Saved successfully!", + mediaPath: path, + }); + + setTimeout(() => { + setProgressState({ type: "idle" }); + }, 1500); + } catch (error) { + setProgressState({ type: "idle" }); + throw error; + } + }; + batch(() => { setTitlebar("border", false); setTitlebar("height", "4rem"); @@ -76,8 +212,125 @@ export function Header() { >
- - + +
+ + +
+
+
+ + + options={RESOLUTION_OPTIONS} + optionValue="value" + optionTextValue="label" + placeholder="Select Resolution" + value={selectedResolution()} + onChange={setSelectedResolution} + itemComponent={(props) => ( + + as={KSelect.Item} + item={props.item} + > + + {props.item.rawValue.label} + + + )} + > + + class="flex-1 text-sm text-left truncate"> + {(state) => ( + {state.selectedOption()?.label} + )} + + + + + + + + as={KSelect.Content} + class={cx(topLeftAnimateClasses, "z-50")} + > + + class="max-h-32 overflow-y-auto" + as={KSelect.Listbox} + /> + + + +
+
+ + + options={FPS_OPTIONS} + optionValue="value" + optionTextValue="label" + placeholder="Select FPS" + value={FPS_OPTIONS.find( + (opt) => opt.value === selectedFps() + )} + onChange={(option) => setSelectedFps(option?.value ?? 30)} + itemComponent={(props) => ( + + as={KSelect.Item} + item={props.item} + > + + {props.item.rawValue.label} + + + )} + > + + class="flex-1 text-sm text-left truncate"> + {(state) => ( + {state.selectedOption()?.label} + )} + + + + + + + + as={KSelect.Content} + class={cx(topLeftAnimateClasses, "z-50")} + > + + class="max-h-32 overflow-y-auto" + as={KSelect.Listbox} + /> + + + +
+ +
+
+ +
); @@ -235,111 +488,12 @@ export function Header() { ); } -import { Channel } from "@tauri-apps/api/core"; -import { save } from "@tauri-apps/plugin-dialog"; -import { DEFAULT_PROJECT_CONFIG } from "./projectConfig"; -import { createMutation } from "@tanstack/solid-query"; -import { getRequestEvent } from "solid-js/web"; -import { checkIsUpgradedAndUpdate } from "~/utils/plans"; - -function ExportButton() { - const { videoId, project, prettyName } = useEditorContext(); +type ShareButtonProps = { + selectedResolution: () => ResolutionOption; + selectedFps: () => number; +}; - const exportVideo = createMutation(() => ({ - mutationFn: async (useCustomMuxer: boolean) => { - const path = await save({ - filters: [{ name: "mp4 filter", extensions: ["mp4"] }], - defaultPath: `~/Desktop/${prettyName()}.mp4`, - }); - if (!path) return; - - setProgressState({ - type: "saving", - progress: 0, - renderProgress: 0, - totalFrames: 0, - message: "Preparing to render...", - mediaPath: path, - stage: "rendering", - }); - - const progress = new Channel(); - progress.onmessage = (p) => { - if (p.type === "FrameRendered" && progressState.type === "saving") { - const percentComplete = Math.min( - Math.round( - (p.current_frame / (progressState.totalFrames || 1)) * 100 - ), - 100 - ); - - setProgressState({ - ...progressState, - renderProgress: p.current_frame, - message: `Rendering video - ${percentComplete}%`, - }); - - // If rendering is complete, update to finalizing state - if (percentComplete === 100) { - setProgressState({ - ...progressState, - message: "Finalizing export...", - }); - } - } - if ( - p.type === "EstimatedTotalFrames" && - progressState.type === "saving" - ) { - setProgressState({ - ...progressState, - totalFrames: p.total_frames, - message: "Starting render...", - }); - } - }; - - try { - const videoPath = await commands.exportVideo( - videoId, - project, - progress, - true, - FPS - ); - await commands.copyFileToPath(videoPath, path); - - setProgressState({ - type: "saving", - progress: 100, - message: "Saved successfully!", - mediaPath: path, - }); - - setTimeout(() => { - setProgressState({ type: "idle" }); - }, 1500); - } catch (error) { - setProgressState({ type: "idle" }); - throw error; - } - }, - })); - - return ( - - ); -} - -function ShareButton() { +function ShareButton(props: ShareButtonProps) { const { videoId, project, presets } = useEditorContext(); const [recordingMeta, metaActions] = createResource(() => commands.getRecordingMeta(videoId, "recording") @@ -407,8 +561,6 @@ function ShareButton() { }); console.log("Starting actual upload..."); - const projectConfig = - project ?? presets.getDefaultConfig() ?? DEFAULT_PROJECT_CONFIG; setProgressState({ type: "uploading", @@ -449,7 +601,17 @@ function ShareButton() { getRequestEvent()?.nativeEvent; - await commands.exportVideo(videoId, projectConfig, progress, true, FPS); + await commands.exportVideo( + videoId, + project, + progress, + true, + props.selectedFps(), + { + x: props.selectedResolution().width, + y: props.selectedResolution().height, + } + ); // Now proceed with upload const result = recordingMeta()?.sharing diff --git a/apps/desktop/src/routes/editor/Player.tsx b/apps/desktop/src/routes/editor/Player.tsx index 6f4556a2..89645869 100644 --- a/apps/desktop/src/routes/editor/Player.tsx +++ b/apps/desktop/src/routes/editor/Player.tsx @@ -8,7 +8,7 @@ import { For, Show, Suspense, createEffect, createSignal } from "solid-js"; import { reconcile } from "solid-js/store"; import { type AspectRatio, commands } from "~/utils/tauri"; -import { FPS, useEditorContext } from "./context"; +import { FPS, OUTPUT_SIZE, useEditorContext } from "./context"; import { ASPECT_RATIOS } from "./projectConfig"; import { ComingSoonTooltip, @@ -88,13 +88,13 @@ export function Player() { await commands.stopPlayback(videoId); setPlaybackTime(0); await commands.seekTo(videoId, 0); - await commands.startPlayback(videoId, FPS); + await commands.startPlayback(videoId, FPS, OUTPUT_SIZE); setPlaying(true); } else if (playing()) { await commands.stopPlayback(videoId); setPlaying(false); } else { - await commands.startPlayback(videoId, FPS); + await commands.startPlayback(videoId, FPS, OUTPUT_SIZE); setPlaying(true); } } catch (error) { diff --git a/apps/desktop/src/routes/editor/Timeline.tsx b/apps/desktop/src/routes/editor/Timeline.tsx index 4f9ac224..8ce4a2cd 100644 --- a/apps/desktop/src/routes/editor/Timeline.tsx +++ b/apps/desktop/src/routes/editor/Timeline.tsx @@ -82,6 +82,7 @@ export function Timeline() { recordingSegment: null, }, ], + zoomSegments: [], }; }) ); diff --git a/apps/desktop/src/routes/editor/context.ts b/apps/desktop/src/routes/editor/context.ts index fd717d9e..fee5ae9f 100644 --- a/apps/desktop/src/routes/editor/context.ts +++ b/apps/desktop/src/routes/editor/context.ts @@ -4,8 +4,15 @@ import { trackStore } from "@solid-primitives/deep"; import { createEventListener } from "@solid-primitives/event-listener"; import { createUndoHistory } from "@solid-primitives/history"; import { debounce } from "@solid-primitives/scheduled"; -import { Accessor, createEffect, createSignal, on } from "solid-js"; +import { + Accessor, + createEffect, + createResource, + createSignal, + on, +} from "solid-js"; import { createStore, reconcile, unwrap } from "solid-js/store"; +import { createElementBounds } from "@solid-primitives/bounds"; import type { PresetsStore } from "../../store"; import { @@ -13,10 +20,11 @@ import { type SerializedEditorInstance, type XY, commands, + events, } from "~/utils/tauri"; -import { useEditorInstanceContext } from "./editorInstanceContext"; import { DEFAULT_PROJECT_CONFIG } from "./projectConfig"; -import { createElementBounds } from "@solid-primitives/bounds"; +import { createImageDataWS, createLazySignal } from "~/utils/socket"; +import { createPresets } from "~/utils/createPresets"; export type CurrentDialog = | { type: "createPreset" } @@ -26,7 +34,12 @@ export type CurrentDialog = export type DialogState = { open: false } | ({ open: boolean } & CurrentDialog); -export const FPS = 60; +export const FPS = 30; + +export const OUTPUT_SIZE = { + x: 1920, + y: 1080, +}; export const [EditorContextProvider, useEditorContext] = createContextProvider( (props: { @@ -103,6 +116,45 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( null! ); +export type FrameData = { width: number; height: number; data: ImageData }; + +export const [EditorInstanceContextProvider, useEditorInstanceContext] = + createContextProvider((props: { videoId: string }) => { + const [latestFrame, setLatestFrame] = createLazySignal<{ + width: number; + data: ImageData; + }>(); + + const [editorInstance] = createResource(async () => { + const instance = await commands.createEditorInstance(props.videoId); + + const [ws, isConnected] = createImageDataWS( + instance.framesSocketUrl, + setLatestFrame + ); + + createEffect(() => { + if (isConnected()) { + events.renderFrameEvent.emit({ + frame_number: Math.floor(0), + fps: FPS, + resolution_base: OUTPUT_SIZE, + }); + } + }); + + return instance; + }); + + return { + editorInstance, + videoId: props.videoId, + latestFrame, + presets: createPresets(), + prettyName: () => editorInstance()?.prettyName ?? "Cap Recording", + }; + }, null!); + function createStoreHistory( ...[state, setState]: ReturnType> ) { diff --git a/apps/desktop/src/routes/editor/editorInstanceContext.ts b/apps/desktop/src/routes/editor/editorInstanceContext.ts deleted file mode 100644 index a190bfc0..00000000 --- a/apps/desktop/src/routes/editor/editorInstanceContext.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { createContextProvider } from "@solid-primitives/context"; -import { createEffect, createResource } from "solid-js"; - -import { events, commands } from "~/utils/tauri"; -import { createPresets } from "~/utils/createPresets"; -import { createImageDataWS, createLazySignal } from "~/utils/socket"; -import { FPS } from "./context"; - -export const OUTPUT_SIZE = { - width: 1920, - height: 1080, -}; - -export type FrameData = { width: number; height: number; data: ImageData }; - -export const [EditorInstanceContextProvider, useEditorInstanceContext] = - createContextProvider((props: { videoId: string }) => { - const [latestFrame, setLatestFrame] = createLazySignal<{ - width: number; - data: ImageData; - }>(); - - const [editorInstance] = createResource(async () => { - const instance = await commands.createEditorInstance(props.videoId); - - const [ws, isConnected] = createImageDataWS( - instance.framesSocketUrl, - setLatestFrame - ); - - createEffect(() => { - if (isConnected()) { - events.renderFrameEvent.emit({ - frame_number: Math.floor(0), - fps: FPS, - }); - } - }); - - return instance; - }); - - return { - editorInstance, - videoId: props.videoId, - latestFrame, - presets: createPresets(), - prettyName: () => editorInstance()?.prettyName ?? "Cap Recording", - }; - }, null!); diff --git a/apps/desktop/src/routes/recordings-overlay.tsx b/apps/desktop/src/routes/recordings-overlay.tsx index 203ba7e8..dd7cff3d 100644 --- a/apps/desktop/src/routes/recordings-overlay.tsx +++ b/apps/desktop/src/routes/recordings-overlay.tsx @@ -38,7 +38,7 @@ import { checkIsUpgradedAndUpdate, canCreateShareableLink, } from "~/utils/plans"; -import { FPS } from "./editor/context"; +import { FPS, OUTPUT_SIZE } from "./editor/context"; type MediaEntry = { path: string; @@ -779,7 +779,8 @@ function createRecordingMutations( presets.getDefaultConfig() ?? DEFAULT_PROJECT_CONFIG, progress, false, - FPS + FPS, + OUTPUT_SIZE ); // Show quick progress animation for existing video @@ -904,7 +905,8 @@ function createRecordingMutations( presets.getDefaultConfig() ?? DEFAULT_PROJECT_CONFIG, progress, true, // Force re-render - FPS + FPS, + OUTPUT_SIZE ); await commands.copyFileToPath(outputPath, savePath); @@ -1039,7 +1041,8 @@ function createRecordingMutations( presets.getDefaultConfig() ?? DEFAULT_PROJECT_CONFIG, progress, false, - FPS + FPS, + OUTPUT_SIZE ); console.log("Using existing rendered video"); diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index ed4ba54d..2d415703 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -53,8 +53,8 @@ async focusCapturesPanel() : Promise { async getCurrentRecording() : Promise> { return await TAURI_INVOKE("get_current_recording"); }, -async exportVideo(videoId: string, project: ProjectConfiguration, progress: TAURI_CHANNEL, force: boolean, fps: number) : Promise { - return await TAURI_INVOKE("export_video", { videoId, project, progress, force, fps }); +async exportVideo(videoId: string, project: ProjectConfiguration, progress: TAURI_CHANNEL, force: boolean, fps: number, resolutionBase: XY) : Promise { + return await TAURI_INVOKE("export_video", { videoId, project, progress, force, fps, resolutionBase }); }, async copyFileToPath(src: string, dst: string) : Promise { return await TAURI_INVOKE("copy_file_to_path", { src, dst }); @@ -74,8 +74,8 @@ async getVideoMetadata(videoId: string, videoType: VideoType | null) : Promise { return await TAURI_INVOKE("create_editor_instance", { videoId }); }, -async startPlayback(videoId: string, fps: number) : Promise { - await TAURI_INVOKE("start_playback", { videoId, fps }); +async startPlayback(videoId: string, fps: number, resolutionBase: XY) : Promise { + await TAURI_INVOKE("start_playback", { videoId, fps, resolutionBase }); }, async stopPlayback(videoId: string) : Promise { await TAURI_INVOKE("stop_playback", { videoId }); @@ -267,7 +267,7 @@ export type RecordingOptions = { captureTarget: ScreenCaptureTarget; cameraLabel export type RecordingOptionsChanged = null export type RecordingStarted = null export type RecordingStopped = { path: string } -export type RenderFrameEvent = { frame_number: number; fps: number } +export type RenderFrameEvent = { frame_number: number; fps: number; resolution_base: XY } export type RenderProgress = { type: "Starting"; total_frames: number } | { type: "EstimatedTotalFrames"; total_frames: number } | { type: "FrameRendered"; current_frame: number } export type RequestNewScreenshot = null export type RequestOpenSettings = { page: string } @@ -282,7 +282,7 @@ export type SerializedEditorInstance = { framesSocketUrl: string; recordingDurat export type SharingMeta = { id: string; link: string } export type ShowCapWindow = "Setup" | "Main" | { Settings: { page: string | null } } | { Editor: { project_id: string } } | "PrevRecordings" | "WindowCaptureOccluder" | { Camera: { ws_port: number } } | { InProgressRecording: { position: [number, number] | null } } | "Upgrade" | "SignIn" export type SingleSegment = { display: Display; camera?: CameraMeta | null; audio?: AudioMeta | null; cursor?: string | null } -export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments?: ZoomSegment[] } +export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments: ZoomSegment[] } export type TimelineSegment = { recordingSegment: number | null; timescale: number; start: number; end: number } export type UploadMode = { Initial: { pre_created_video: PreCreatedVideo | null } } | "Reupload" export type UploadProgress = { stage: string; progress: number; message: string } diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index b090a466..30f5ff80 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -1,7 +1,7 @@ use std::{sync::Arc, time::Instant}; use cap_media::frame_ws::WSFrame; -use cap_project::{BackgroundSource, ProjectConfiguration, RecordingMeta}; +use cap_project::{BackgroundSource, ProjectConfiguration, RecordingMeta, XY}; use cap_rendering::{ decoder::DecodedFrame, produce_frame, ProjectRecordings, ProjectUniforms, RenderVideoConstants, }; @@ -18,6 +18,7 @@ pub enum RendererMessage { uniforms: ProjectUniforms, time: f32, // Add this field finished: oneshot::Sender<()>, + resolution_base: XY, }, Stop { finished: oneshot::Sender<()>, @@ -80,6 +81,7 @@ impl Renderer { uniforms, time, finished, + resolution_base, } => { if let Some(task) = frame_task.as_ref() { if task.is_finished() { @@ -102,6 +104,7 @@ impl Renderer { &uniforms, time, total_frames, + resolution_base, ) .await .unwrap(); @@ -145,6 +148,7 @@ impl RendererHandle { background: BackgroundSource, uniforms: ProjectUniforms, time: f32, // Add this parameter + resolution_base: XY, ) { let (finished_tx, finished_rx) = oneshot::channel(); @@ -155,6 +159,7 @@ impl RendererHandle { uniforms, time, // Pass the time finished: finished_tx, + resolution_base, }) .await; diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index fc375987..6944d3e1 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -173,7 +173,7 @@ impl EditorInstance { (self.on_state_change)(&state); } - pub async fn start_playback(self: Arc, fps: u32) { + pub async fn start_playback(self: Arc, fps: u32, resolution_base: XY) { let (mut handle, prev) = { let Ok(mut state) = self.state.try_lock() else { return; @@ -188,7 +188,7 @@ impl EditorInstance { start_frame_number, project: self.project_config.0.subscribe(), } - .start(fps) + .start(fps, resolution_base) .await; let prev = state.playback_task.replace(playback_handle.clone()); @@ -223,12 +223,13 @@ impl EditorInstance { fn spawn_preview_renderer( self: Arc, - mut preview_rx: watch::Receiver>, + mut preview_rx: watch::Receiver)>>, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { loop { preview_rx.changed().await.unwrap(); - let Some((frame_number, fps)) = *preview_rx.borrow().deref() else { + let Some((frame_number, fps, resolution_base)) = *preview_rx.borrow().deref() + else { continue; }; @@ -245,20 +246,27 @@ impl EditorInstance { let segment = &self.segments[segment.unwrap_or(0) as usize]; - let (screen_frame, camera_frame) = segment + if let Some((screen_frame, camera_frame)) = segment .decoders .get_frames(time as f32, !project.camera.hide) - .await; - - self.renderer - .render_frame( - screen_frame, - camera_frame, - project.background.source.clone(), - ProjectUniforms::new(&self.render_constants, &project, time as f32), - time as f32, // Add the time parameter - ) - .await; + .await + { + self.renderer + .render_frame( + screen_frame, + camera_frame, + project.background.source.clone(), + ProjectUniforms::new( + &self.render_constants, + &project, + time as f32, + resolution_base, + ), + time as f32, + resolution_base, + ) + .await; + } } }) } @@ -283,7 +291,7 @@ impl Drop for EditorInstance { } } -type PreviewFrameInstruction = (u32, u32); +type PreviewFrameInstruction = (u32, u32, XY); pub struct EditorState { pub playhead_position: u32, diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index c0ea758d..a61bd995 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration}; use cap_media::data::{AudioInfo, AudioInfoError, FromSampleBytes}; use cap_media::feeds::{AudioData, AudioPlaybackBuffer}; -use cap_project::ProjectConfiguration; +use cap_project::{ProjectConfiguration, XY}; use cap_rendering::{ProjectUniforms, RenderVideoConstants}; use cpal::{ traits::{DeviceTrait, HostTrait, StreamTrait}, @@ -35,7 +35,7 @@ pub struct PlaybackHandle { } impl Playback { - pub async fn start(self, fps: u32) -> PlaybackHandle { + pub async fn start(self, fps: u32, resolution_base: XY) -> PlaybackHandle { let (stop_tx, mut stop_rx) = watch::channel(false); stop_rx.borrow_and_update(); @@ -76,7 +76,7 @@ impl Playback { }; loop { - if frame_number as f64 > fps as f64 * duration { + if frame_number as f64 >= fps as f64 * duration { break; }; @@ -94,8 +94,9 @@ impl Playback { _ = stop_rx.changed() => { break; }, - (screen_frame, camera_frame) = segment.decoders.get_frames(time as f32, !project.camera.hide) => { - let uniforms = ProjectUniforms::new(&self.render_constants, &project, time as f32); + data = segment.decoders.get_frames(time as f32, !project.camera.hide) => { + if let Some((screen_frame, camera_frame)) = data { + let uniforms = ProjectUniforms::new(&self.render_constants, &project, time as f32, resolution_base); self .renderer @@ -104,9 +105,11 @@ impl Playback { camera_frame, project.background.source.clone(), uniforms.clone(), - time as f32 // Add the time parameter + time as f32, + resolution_base ) .await; + } } else => { } @@ -125,8 +128,6 @@ impl Playback { frame_number += 1; } - println!("stopped playback"); - stop_tx.send(true).ok(); event_tx.send(PlaybackEvent::Stop).ok(); diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index ed295c90..11b61a53 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -5,7 +5,7 @@ use cap_media::{ feeds::{AudioData, AudioFrameBuffer}, MediaError, }; -use cap_project::{ProjectConfiguration, RecordingMeta}; +use cap_project::{ProjectConfiguration, RecordingMeta, XY}; use cap_rendering::{ ProjectUniforms, RecordingSegmentDecoders, RenderSegment, RenderVideoConstants, RenderedFrame, SegmentVideoPaths, @@ -43,6 +43,7 @@ pub struct Exporter { meta: RecordingMeta, render_constants: Arc, fps: u32, + resolution_base: XY, } impl Exporter @@ -58,11 +59,13 @@ where render_constants: Arc, segments: &[Segment], fps: u32, + resolution_base: XY, ) -> Result { let output_folder = output_path.parent().unwrap(); std::fs::create_dir_all(output_folder)?; - let output_size = ProjectUniforms::get_output_size(&render_constants.options, &project); + let output_size = + ProjectUniforms::get_output_size(&render_constants.options, &project, resolution_base); let (render_segments, audio_segments): (Vec<_>, Vec<_>) = segments .iter() @@ -104,6 +107,7 @@ where audio_segments, output_size, fps, + resolution_base, }) } @@ -228,7 +232,7 @@ where RawVideoFormat::Rgba, self.output_size.0, self.output_size.1, - 30, + self.fps, ) .wrap_frame( &frame.data, @@ -294,6 +298,7 @@ where &self.meta, self.render_segments, self.fps, + self.resolution_base, ) .then(|f| async { f.map_err(Into::into) }); diff --git a/crates/media/src/encoders/h264_avassetwriter.rs b/crates/media/src/encoders/h264_avassetwriter.rs index b2bc13fb..dc75cdaa 100644 --- a/crates/media/src/encoders/h264_avassetwriter.rs +++ b/crates/media/src/encoders/h264_avassetwriter.rs @@ -16,6 +16,7 @@ pub struct H264AVAssetWriterEncoder { impl H264AVAssetWriterEncoder { pub fn init(tag: &'static str, config: VideoInfo, output: Output) -> Result { + let fps = config.frame_rate.0 as f32 / config.frame_rate.1 as f32; let Output::File(destination) = output; let mut asset_writer = av::AssetWriter::with_url_and_file_type( @@ -47,11 +48,14 @@ impl H264AVAssetWriterEncoder { ns::Number::with_u32(config.height).as_id_ref(), ); + let bitrate = config.width as f32 * config.height as f32 / (1920.0 * 1080.0) * 10_000_000.0 + + fps / 30.0 * 4_000_000.0; + output_settings.insert( av::video_settings_keys::compression_props(), ns::Dictionary::with_keys_values( &[unsafe { AVVideoAverageBitRateKey }], - &[ns::Number::with_u32(10_000_000).as_id_ref()], + &[ns::Number::with_f32(bitrate).as_id_ref()], ) .as_id_ref(), ); diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index 022ceeff..9891f78c 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -299,11 +299,10 @@ pub enum ZoomMode { Manual { x: f32, y: f32 }, } -#[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] +#[derive(Type, Serialize, Deserialize, Clone, Debug)] #[serde(rename_all = "camelCase")] pub struct TimelineConfiguration { pub segments: Vec, - #[serde(default)] pub zoom_segments: Vec, } diff --git a/crates/recording/src/actor.rs b/crates/recording/src/actor.rs index 5327114e..33050ac1 100644 --- a/crates/recording/src/actor.rs +++ b/crates/recording/src/actor.rs @@ -193,8 +193,6 @@ pub async fn spawn_recording_actor( let cursors = if let Some(cursor) = pipeline.cursor.take() { let res = cursor.actor.stop().await; - dbg!(&res.cursors); - std::fs::write( &cursor.output_path, serde_json::to_string_pretty(&CursorEvents { diff --git a/crates/rendering/src/decoder.rs b/crates/rendering/src/decoder.rs index af108485..efc44b16 100644 --- a/crates/rendering/src/decoder.rs +++ b/crates/rendering/src/decoder.rs @@ -156,7 +156,7 @@ impl AsyncVideoDecoder { let mut peekable_requests = PeekableReceiver { rx, peeked: None }; - let mut packets = input.packets(); + let mut packets = input.packets().peekable(); let width = decoder.width(); let height = decoder.height(); @@ -194,15 +194,12 @@ impl AsyncVideoDecoder { * 1_000_000.0) as i64; let position = timestamp_us.rescale((1, 1_000_000), rescale::TIME_BASE); - println!("seeking to {position} for frame {requested_frame}, last sent frame: {:?}", last_sent_frame.map(|(f, _)| f)); - decoder.flush(); input.seek(position, ..position).unwrap(); - cache.clear(); last_decoded_frame = None; last_sent_frame = None; - packets = input.packets(); + packets = input.packets().peekable(); } last_active_frame = Some(requested_frame); @@ -212,7 +209,13 @@ impl AsyncVideoDecoder { break; } let Some((stream, packet)) = packets.next() else { - println!("sending black frame as end of stream"); + // handles the case where the cache doesn't contain a frame so we fallback to the previously sent one + if let Some(last_sent_frame) = &last_sent_frame { + if last_sent_frame.0 < requested_frame { + sender.take().map(|s| s.send(last_sent_frame.1.clone())); + } + } + sender.take().map(|s| s.send(black_frame.clone())); break; }; @@ -313,14 +316,10 @@ impl AsyncVideoDecoder { } } - if let Some(s) = sender.take() { - s.send( - last_sent_frame - .clone() - .map(|f| f.1) - .unwrap_or_else(|| black_frame.clone()), - ) - .ok(); + if let Some((sender, last_sent_frame)) = + sender.take().zip(last_sent_frame.clone()) + { + sender.send(last_sent_frame.1).ok(); } } } @@ -337,13 +336,12 @@ pub struct AsyncVideoDecoderHandle { } impl AsyncVideoDecoderHandle { - pub async fn get_frame(&self, time: f32) -> DecodedFrame { + pub async fn get_frame(&self, time: f32) -> Option { let (tx, rx) = tokio::sync::oneshot::channel(); self.sender .send(VideoDecoderMessage::GetFrame(time, tx)) .unwrap(); - let res = rx.await.unwrap(); - res + rx.await.ok() } } diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 469dd8ae..ce413924 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -28,7 +28,7 @@ pub mod decoder; mod project_recordings; mod zoom; pub use decoder::DecodedFrame; -pub use project_recordings::{ProjectRecordings, SegmentRecordings}; +pub use project_recordings::{ProjectRecordings, SegmentRecordings, Video}; use zoom::*; @@ -126,15 +126,17 @@ impl RecordingSegmentDecoders { &self, frame_time: f32, needs_camera: bool, - ) -> (DecodedFrame, Option) { - tokio::join!( + ) -> Option<(DecodedFrame, Option)> { + let (screen, camera) = tokio::join!( self.screen.get_frame(frame_time), OptionFuture::from( needs_camera .then(|| self.camera.as_ref().map(|d| d.get_frame(frame_time))) .flatten() ) - ) + ); + + Some((screen?, camera.flatten())) } } @@ -159,11 +161,12 @@ pub struct RenderSegment { pub async fn render_video_to_channel( options: RenderOptions, - mut project: ProjectConfiguration, + project: ProjectConfiguration, sender: mpsc::Sender, meta: &RecordingMeta, segments: Vec, fps: u32, + resolution_base: XY, ) -> Result<(), RenderingError> { let constants = RenderVideoConstants::new(options, meta).await?; let recordings = ProjectRecordings::new(meta); @@ -177,8 +180,8 @@ pub async fn render_video_to_channel( let total_frames = (fps as f64 * duration).ceil() as u32; println!( - "Final export duration: {} seconds ({} frames at 30fps)", - duration, total_frames + "Final export duration: {} seconds ({} frames at {}fps)", + duration, total_frames, fps ); let mut frame_number = 0; @@ -189,40 +192,54 @@ pub async fn render_video_to_channel( break; } - let (time, segment_i) = if let Some(timeline) = &project.timeline { + let source_time = if let Some(timeline) = &project.timeline { match timeline.get_recording_time(frame_number as f64 / fps as f64) { - Some(value) => (value.0, value.1), - None => (frame_number as f64 / fps as f64, Some(0u32)), + Some(value) => value.0, + None => frame_number as f64 / fps as f64, } } else { - (frame_number as f64 / fps as f64, Some(0u32)) + frame_number as f64 / fps as f64 + }; + + let segment_i = if let Some(timeline) = &project.timeline { + timeline + .get_recording_time(frame_number as f64 / fps as f64) + .map(|value| value.1) + .flatten() + .unwrap_or(0u32) + } else { + 0u32 }; - let segment = &segments[segment_i.unwrap() as usize]; - // let frame_time = frame_number; - let (screen_frame, camera_frame) = segment + let segment = &segments[segment_i as usize]; + + if let Some((screen_frame, camera_frame)) = segment .decoders - .get_frames(time as f32, !project.camera.hide) - .await; - - let uniforms = ProjectUniforms::new(&constants, &project, time as f32); - - let frame = produce_frame( - &constants, - &screen_frame, - &camera_frame, - background, - &uniforms, - time as f32, - total_frames, - ) - .await?; + .get_frames(source_time as f32, !project.camera.hide) + .await + { + let uniforms = + ProjectUniforms::new(&constants, &project, source_time as f32, resolution_base); + + let frame = produce_frame( + &constants, + &screen_frame, + &camera_frame, + background, + &uniforms, + source_time as f32, + total_frames, + resolution_base, + ) + .await?; - if frame.width == 0 || frame.height == 0 { - continue; + if frame.width == 0 || frame.height == 0 { + continue; + } + + sender.send(frame).await?; } - sender.send(frame).await?; frame_number += 1; } @@ -564,43 +581,76 @@ impl ProjectUniforms { basis as f64 * padding_factor } - pub fn get_output_size(options: &RenderOptions, project: &ProjectConfiguration) -> (u32, u32) { + pub fn get_output_size( + options: &RenderOptions, + project: &ProjectConfiguration, + resolution_base: XY, + ) -> (u32, u32) { let crop = Self::get_crop(options, project); - let crop_aspect = crop.aspect_ratio(); - let padding = Self::get_padding(options, project) * 2.0; - let aspect = match &project.aspect_ratio { + let (base_width, base_height) = match &project.aspect_ratio { None => { let width = ((crop.size.x as f64 + padding) as u32 + 1) & !1; let height = ((crop.size.y as f64 + padding) as u32 + 1) & !1; - return (width, height); + (width, height) + } + Some(AspectRatio::Square) => { + let size = if crop_aspect > 1.0 { + crop.size.y + } else { + crop.size.x + }; + (size, size) + } + Some(AspectRatio::Wide) => { + if crop_aspect > 16.0 / 9.0 { + (((crop.size.y as f32 * 16.0 / 9.0) as u32), crop.size.y) + } else { + (crop.size.x, ((crop.size.x as f32 * 9.0 / 16.0) as u32)) + } + } + Some(AspectRatio::Vertical) => { + if crop_aspect > 9.0 / 16.0 { + ((crop.size.y as f32 * 9.0 / 16.0) as u32, crop.size.y) + } else { + (crop.size.x, ((crop.size.x as f32 * 16.0 / 9.0) as u32)) + } + } + Some(AspectRatio::Classic) => { + if crop_aspect > 4.0 / 3.0 { + ((crop.size.y as f32 * 4.0 / 3.0) as u32, crop.size.y) + } else { + (crop.size.x, ((crop.size.x as f32 * 3.0 / 4.0) as u32)) + } + } + Some(AspectRatio::Tall) => { + if crop_aspect > 3.0 / 4.0 { + ((crop.size.y as f32 * 3.0 / 4.0) as u32, crop.size.y) + } else { + (crop.size.x, ((crop.size.x as f32 * 4.0 / 3.0) as u32)) + } } - Some(AspectRatio::Square) => 1.0, - Some(AspectRatio::Wide) => 16.0 / 9.0, - Some(AspectRatio::Vertical) => 9.0 / 16.0, - Some(AspectRatio::Classic) => 4.0 / 3.0, - Some(AspectRatio::Tall) => 3.0 / 4.0, }; - let (width, height) = if crop_aspect > aspect { - (crop.size.x, (crop.size.x as f32 / aspect) as u32) - } else if crop_aspect < aspect { - ((crop.size.y as f32 * aspect) as u32, crop.size.y) - } else { - (crop.size.x, crop.size.y) - }; + let width_scale = resolution_base.x as f32 / base_width as f32; + let height_scale = resolution_base.y as f32 / base_height as f32; + let scale = width_scale.min(height_scale); + + let scaled_width = ((base_width as f32 * scale) as u32 + 1) & !1; + let scaled_height = ((base_height as f32 * scale) as u32 + 1) & !1; + return (scaled_width, scaled_height); - // Ensure width and height are divisible by 2 - ((width + 1) & !1, (height + 1) & !1) + // ((base_width + 1) & !1, (base_height + 1) & !1) } pub fn get_display_offset( options: &RenderOptions, project: &ProjectConfiguration, + resolution_base: XY, ) -> Coord { - let output_size = Self::get_output_size(options, project); + let output_size = Self::get_output_size(options, project, resolution_base); let output_size = XY::new(output_size.0 as f64, output_size.1 as f64); let output_aspect = output_size.x / output_size.y; @@ -642,9 +692,10 @@ impl ProjectUniforms { constants: &RenderVideoConstants, project: &ProjectConfiguration, time: f32, + resolution_base: XY, ) -> Self { let options = &constants.options; - let output_size = Self::get_output_size(options, project); + let output_size = Self::get_output_size(options, project, resolution_base); let cursor_position = interpolate_cursor_position( &Default::default(), /*constants.cursor*/ @@ -717,12 +768,12 @@ impl ProjectUniforms { (crop.position.y + crop.size.y) as f64, )); - let display_offset = Self::get_display_offset(options, project); + let display_offset = Self::get_display_offset(options, project, resolution_base); let end = Coord::new(output_size) - display_offset; let screen_scale_origin = zoom_origin - .to_frame_space(options, project) + .to_frame_space(options, project, resolution_base) .clamp(display_offset.coord, end.coord); let zoom = Zoom { @@ -879,6 +930,7 @@ pub async fn produce_frame( uniforms: &ProjectUniforms, time: f32, total_frames: u32, + resolution_base: XY, ) -> Result { let mut encoder = constants.device.create_command_encoder( &(wgpu::CommandEncoderDescriptor { @@ -1011,6 +1063,7 @@ pub async fn produce_frame( time, &mut encoder, get_either(texture_views, !output_is_left), + resolution_base, ); } @@ -1273,6 +1326,7 @@ fn draw_cursor( time: f32, encoder: &mut CommandEncoder, view: &wgpu::TextureView, + resolution_base: XY, ) { let Some(cursor_position) = interpolate_cursor_position( &Default::default(), // constants.cursor, @@ -1291,8 +1345,10 @@ fn draw_cursor( // Calculate velocity in screen space let velocity = if let Some(prev_pos) = prev_position { - let curr_frame_pos = cursor_position.to_frame_space(&constants.options, &uniforms.project); - let prev_frame_pos = prev_pos.to_frame_space(&constants.options, &uniforms.project); + let curr_frame_pos = + cursor_position.to_frame_space(&constants.options, &uniforms.project, resolution_base); + let prev_frame_pos = + prev_pos.to_frame_space(&constants.options, &uniforms.project, resolution_base); let frame_velocity = curr_frame_pos.coord - prev_frame_pos.coord; // Convert to pixels per frame @@ -1335,7 +1391,8 @@ fn draw_cursor( STANDARD_CURSOR_HEIGHT * cursor_size_percentage, ]; - let frame_position = cursor_position.to_frame_space(&constants.options, &uniforms.project); + let frame_position = + cursor_position.to_frame_space(&constants.options, &uniforms.project, resolution_base); let position = uniforms.zoom.apply_scale(frame_position); let relative_position = [position.x as f32, position.y as f32]; @@ -2074,10 +2131,11 @@ impl Coord { &self, options: &RenderOptions, project: &ProjectConfiguration, + resolution_base: XY, ) -> Coord { self.to_raw_display_space(options) .to_cropped_display_space(options, project) - .to_frame_space(options, project) + .to_frame_space(options, project, resolution_base) } } @@ -2097,8 +2155,9 @@ impl Coord { &self, options: &RenderOptions, project: &ProjectConfiguration, + resolution_base: XY, ) -> Coord { - let padding = ProjectUniforms::get_display_offset(options, project); + let padding = ProjectUniforms::get_display_offset(options, project, resolution_base); Coord::new(self.coord + *padding) } }