Skip to content

Commit

Permalink
account for rendered frame stride when exporting
Browse files Browse the repository at this point in the history
  • Loading branch information
@Brendonovich
Brendonovich committed Dec 18, 2024
1 parent f8d5421 commit ec4f88a
Show file tree
Hide file tree
Showing 8 changed files with 95 additions and 40 deletions.
9 changes: 4 additions & 5 deletions crates/editor/src/editor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,24 +76,23 @@ impl Renderer {
let frame_tx = self.frame_tx.clone();

frame_task = Some(tokio::spawn(async move {
let now = Instant::now();
let (frame, stride) = produce_frame(
let frame = produce_frame(
&render_constants,
&screen_frame,
&camera_frame,
cap_rendering::Background::from(background),
&uniforms,
time, // Pass the actual time value
time,
)
.await
.unwrap();

frame_tx
.try_send(WSFrame {
data: frame,
data: frame.data,
width: uniforms.output_size.0,
height: uniforms.output_size.1,
stride,
stride: frame.padded_bytes_per_row,
})
.ok();
finished.send(()).ok();
Expand Down
56 changes: 41 additions & 15 deletions crates/export/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use cap_media::{
};
use cap_project::{ProjectConfiguration, RecordingMeta};
use cap_rendering::{
ProjectUniforms, RecordingSegmentDecoders, RenderSegment, RenderVideoConstants,
ProjectUniforms, RecordingSegmentDecoders, RenderSegment, RenderVideoConstants, RenderedFrame,
SegmentVideoPaths,
};
use futures::FutureExt;
Expand Down Expand Up @@ -111,7 +111,7 @@ where
buffer: AudioFrameBuffer,
}

let (tx_image_data, mut rx_image_data) = tokio::sync::mpsc::channel::<Vec<u8>>(4);
let (tx_image_data, mut rx_image_data) = tokio::sync::mpsc::channel::<RenderedFrame>(4);

let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::<MP4Input>(4);

Expand Down Expand Up @@ -225,7 +225,11 @@ where
self.output_size.1,
30,
)
.wrap_frame(&frame, 0);
.wrap_frame(
&frame.data,
0,
frame.padded_bytes_per_row as usize,
);
video_frame.set_pts(Some(frame_count as i64));

frame_tx
Expand All @@ -239,16 +243,14 @@ where
}

// Save the first frame as a screenshot and thumbnail
if let Some(frame_data) = first_frame {
let width = self.output_size.0;
let height = self.output_size.1;
if let Some(frame) = first_frame {
let rgba_img: ImageBuffer<Rgba<u8>, Vec<u8>> =
ImageBuffer::from_raw(width, height, frame_data)
ImageBuffer::from_raw(frame.width, frame.height, frame.data)
.expect("Failed to create image from frame data");

// Convert RGBA to RGB
let rgb_img: ImageBuffer<image::Rgb<u8>, Vec<u8>> =
ImageBuffer::from_fn(width, height, |x, y| {
ImageBuffer::from_fn(frame.width, frame.height, |x, y| {
let rgba = rgba_img.get_pixel(x, y);
image::Rgb([rgba[0], rgba[1], rgba[2]])
});
Expand Down Expand Up @@ -306,7 +308,7 @@ where
pipe_tx: tokio::sync::mpsc::Sender<Vec<u8>>,
}

let (tx_image_data, mut rx_image_data) = tokio::sync::mpsc::channel::<Vec<u8>>(4);
let (tx_image_data, mut rx_image_data) = tokio::sync::mpsc::channel::<RenderedFrame>(4);

let ffmpeg_handle = tokio::spawn({
let project = self.project.clone();
Expand All @@ -325,6 +327,17 @@ where
let pipe_path = cap_utils::create_channel_named_pipe(
rx,
audio_dir.path().join("audio.pipe"),
{
let mut done = false;
move |value| {
if done {
None
} else {
done = true;
Some(&value)
}
}
},
);

ffmpeg.add_input(cap_ffmpeg_cli::FFmpegRawAudioInput {
Expand All @@ -348,11 +361,26 @@ where
};

let video_tx = {
let (tx, rx) = tokio::sync::mpsc::channel::<Vec<u8>>(30);
let (tx, rx) = tokio::sync::mpsc::channel::<RenderedFrame>(30);

let pipe_path = cap_utils::create_channel_named_pipe(
rx,
video_dir.path().join("video.pipe"),
{
let mut i = 0;

move |frame| {
if i < frame.data.len() as u32 {
let value =
&frame.data[i as usize..(i + frame.width * 4) as usize];

i += frame.padded_bytes_per_row;
Some(value)
} else {
None
}
}
},
);

ffmpeg.add_input(cap_ffmpeg_cli::FFmpegRawVideoInput {
Expand Down Expand Up @@ -442,16 +470,14 @@ where
ffmpeg_process.stop().await;

// Save the first frame as a screenshot and thumbnail
if let Some(frame_data) = first_frame {
let width = self.output_size.0;
let height = self.output_size.1;
if let Some(frame) = first_frame {
let rgba_img: ImageBuffer<Rgba<u8>, Vec<u8>> =
ImageBuffer::from_raw(width, height, frame_data)
ImageBuffer::from_raw(frame.width, frame.height, frame.data)
.expect("Failed to create image from frame data");

// Convert RGBA to RGB
let rgb_img: ImageBuffer<image::Rgb<u8>, Vec<u8>> =
ImageBuffer::from_fn(width, height, |x, y| {
ImageBuffer::from_fn(frame.width, frame.height, |x, y| {
let rgba = rgba_img.get_pixel(x, y);
image::Rgb([rgba[0], rgba[1], rgba[2]])
});
Expand Down
4 changes: 2 additions & 2 deletions crates/flags/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ pub struct Flags {
pub const FLAGS: Flags = Flags {
record_mouse: false, // cfg!(debug_assertions),
split: false, // cfg!(debug_assertions),
pause_resume: false, // cfg!(debug_assertions),
zoom: false, // cfg!(debug_assertions),
pause_resume: cfg!(debug_assertions),
zoom: false, // cfg!(debug_assertions),
custom_s3: true,
};
13 changes: 11 additions & 2 deletions crates/media/src/data.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,11 +262,20 @@ impl VideoInfo {
av_pix_fmt as i32
}

pub fn wrap_frame(&self, data: &[u8], timestamp: i64) -> FFVideo {
pub fn wrap_frame(&self, data: &[u8], timestamp: i64, stride: usize) -> FFVideo {
let mut frame = FFVideo::new(self.pixel_format, self.width, self.height);

frame.set_pts(Some(timestamp));
frame.data_mut(0)[0..data.len()].copy_from_slice(data);

if frame.stride(0) == self.width as usize {
frame.data_mut(0)[0..data.len()].copy_from_slice(data);
} else {
let ffmpeg_stride = frame.stride(0) as usize;
for (line, chunk) in data.chunks(stride).enumerate() {
frame.data_mut(0)[line * ffmpeg_stride..(line + 1) * ffmpeg_stride]
.copy_from_slice(&chunk[0..ffmpeg_stride]);
}
}

frame
}
Expand Down
12 changes: 10 additions & 2 deletions crates/media/src/feeds/camera.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,11 @@ impl FrameConverter {
converter.convert(buffer.buffer(), resolution.width(), resolution.height())
}
None => {
let input_frame = self.video_info.wrap_frame(buffer.buffer(), 0);
let input_frame = self.video_info.wrap_frame(
buffer.buffer(),
0,
buffer.buffer().len() / buffer.resolution().height() as usize,
);
let mut rgba_frame = FFVideo::empty();

self.context.run(&input_frame, &mut rgba_frame).unwrap();
Expand All @@ -439,7 +443,11 @@ impl FrameConverter {
}

fn raw(&mut self, buffer: &nokhwa::Buffer) -> FFVideo {
self.video_info.wrap_frame(buffer.buffer(), 0)
self.video_info.wrap_frame(
buffer.buffer(),
0,
buffer.buffer_bytes().len() / buffer.resolution().height() as usize,
)
}
}

Expand Down
2 changes: 1 addition & 1 deletion crates/recording/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ pub mod actor;
mod cursor;
pub mod segmented_actor;

pub use actor::{spawn_recording_actor, ActorHandle, CompletedRecording, RecordingError};
pub use segmented_actor::{spawn_recording_actor, ActorHandle, CompletedRecording, RecordingError};

use cap_media::sources::*;
use serde::{Deserialize, Serialize};
Expand Down
23 changes: 18 additions & 5 deletions crates/rendering/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ pub enum RenderingError {
#[error(transparent)]
BufferMapFailed(#[from] wgpu::BufferAsyncError),
#[error("Sending frame to channel failed")]
ChannelSendFrameFailed(#[from] mpsc::error::SendError<Vec<u8>>),
ChannelSendFrameFailed(#[from] mpsc::error::SendError<RenderedFrame>),
}

pub struct RenderSegment {
Expand All @@ -141,7 +141,7 @@ pub struct RenderSegment {
pub async fn render_video_to_channel(
options: RenderOptions,
project: ProjectConfiguration,
sender: tokio::sync::mpsc::Sender<Vec<u8>>,
sender: mpsc::Sender<RenderedFrame>,
meta: &RecordingMeta,
segments: Vec<RenderSegment>,
) -> Result<(), RenderingError> {
Expand Down Expand Up @@ -198,7 +198,7 @@ pub async fn render_video_to_channel(
)
.await?;

sender.send(frame.0).await?;
sender.send(frame).await?;
} else {
println!("no decoder frames: {:?}", (time, segment_i));
};
Expand Down Expand Up @@ -1252,14 +1252,22 @@ fn do_easing(t: f64) -> f64 {
1.0 - (1.0 - t) * (1.0 - t)
}

#[derive(Clone)]
pub struct RenderedFrame {
pub data: Vec<u8>,
pub width: u32,
pub height: u32,
pub padded_bytes_per_row: u32,
}

pub async fn produce_frame(
constants: &RenderVideoConstants,
screen_frame: &Vec<u8>,
camera_frame: &Option<DecodedFrame>,
background: Background,
uniforms: &ProjectUniforms,
time: f32,
) -> Result<(Vec<u8>, u32), RenderingError> {
) -> Result<RenderedFrame, RenderingError> {
let mut encoder = constants.device.create_command_encoder(
&(wgpu::CommandEncoderDescriptor {
label: Some("Render Encoder"),
Expand Down Expand Up @@ -1530,7 +1538,12 @@ pub async fn produce_frame(
drop(data);
output_buffer.unmap();

Ok((image_data, padded_bytes_per_row))
Ok(RenderedFrame {
data: image_data,
padded_bytes_per_row,
width: uniforms.output_size.0,
height: uniforms.output_size.1,
})
}

fn draw_cursor(
Expand Down
16 changes: 8 additions & 8 deletions crates/utils/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,10 @@ fn create_named_pipe(path: &std::path::Path) -> Result<(), Box<dyn std::error::E
Ok(())
}

pub fn create_channel_named_pipe(
mut rx: tokio::sync::mpsc::Receiver<Vec<u8>>,
pub fn create_channel_named_pipe<T: Send + 'static>(
mut rx: tokio::sync::mpsc::Receiver<T>,
unix_path: PathBuf,
get_bytes: impl FnMut(&T) -> Option<&[u8]> + Clone + Send + 'static,
) -> OsString {
#[cfg(unix)]
{
Expand All @@ -62,13 +63,12 @@ pub fn create_channel_named_pipe(
.unwrap();
println!("video pipe opened");

println!("receiving frame for channel");
while let Some(bytes) = rx.recv().await {
println!("received frame, writing bytes");
file.write_all(&bytes)
// .await
.unwrap();
println!("bytes written");
let mut get_bytes = get_bytes.clone();

while let Some(bytes) = get_bytes(&bytes) {
file.write_all(&bytes).unwrap();
}
}

println!("done writing to video pipe");
Expand Down

1 comment on commit ec4f88a

@vercel
Copy link

@vercel vercel bot commented on ec4f88a Dec 18, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

cap-web – ./

cap-web-nu.vercel.app
cap-web-mc-ilroy.vercel.app
cap.so
www.cap.so
cap-web-git-main-mc-ilroy.vercel.app

Please sign in to comment.