diff --git a/nokhwa-bindings-linux/src/lib.rs b/nokhwa-bindings-linux/src/lib.rs index 23647a6..2bd9c58 100644 --- a/nokhwa-bindings-linux/src/lib.rs +++ b/nokhwa-bindings-linux/src/lib.rs @@ -540,6 +540,7 @@ mod internal { FrameFormat::YUYV => FourCC::new(b"YUYV"), FrameFormat::GRAY => FourCC::new(b"GRAY"), FrameFormat::RAWRGB => FourCC::new(b"RGB3"), + FrameFormat::RAWBGR => FourCC::new(b"BGR3"), FrameFormat::NV12 => FourCC::new(b"NV12"), }; @@ -912,6 +913,7 @@ mod internal { "MJPG" => Some(FrameFormat::MJPEG), "GRAY" => Some(FrameFormat::GRAY), "RGB3" => Some(FrameFormat::RAWRGB), + "BGR3" => Some(FrameFormat::RAWBGR), "NV12" => Some(FrameFormat::NV12), _ => None, } @@ -923,6 +925,7 @@ mod internal { FrameFormat::YUYV => FourCC::new(b"YUYV"), FrameFormat::GRAY => FourCC::new(b"GRAY"), FrameFormat::RAWRGB => FourCC::new(b"RGB3"), + FrameFormat::RAWBGR => FourCC::new(b"BGR3"), FrameFormat::NV12 => FourCC::new(b"NV12"), } } diff --git a/nokhwa-bindings-macos/src/lib.rs b/nokhwa-bindings-macos/src/lib.rs index bd419bb..15b250b 100644 --- a/nokhwa-bindings-macos/src/lib.rs +++ b/nokhwa-bindings-macos/src/lib.rs @@ -2285,6 +2285,13 @@ mod internal { FrameFormat::GRAY => kCMPixelFormat_8IndexedGray_WhiteIsZero, FrameFormat::NV12 => kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, FrameFormat::RAWRGB => kCMPixelFormat_24RGB, + FrameFormat::RAWBGR => { + return Err(NokhwaError::SetPropertyError { + property: "setVideoSettings".to_string(), + value: "set frame format".to_string(), + error: "Unsupported frame format BGR".to_string(), + }); + } }; let obj = CFNumber::from(cmpixelfmt as i32); let obj = obj.as_CFTypeRef() as *mut Object; diff --git a/nokhwa-bindings-windows/src/lib.rs b/nokhwa-bindings-windows/src/lib.rs index 88682b5..d65aec0 100644 --- a/nokhwa-bindings-windows/src/lib.rs +++ b/nokhwa-bindings-windows/src/lib.rs @@ -48,7 +48,7 @@ pub mod wmf { }; use windows::Win32::Media::DirectShow::{CameraControl_Flags_Auto, CameraControl_Flags_Manual}; use windows::Win32::Media::MediaFoundation::{ - IMFMediaType, MFCreateSample, MF_SOURCE_READER_FIRST_VIDEO_STREAM, + MFCreateSample, MF_SOURCE_READER_FIRST_VIDEO_STREAM, }; use windows::{ core::{Interface, GUID, PWSTR}, @@ -65,14 +65,14 @@ pub mod wmf { KernelStreaming::GUID_NULL, MediaFoundation::{ IMFActivate, IMFAttributes, IMFMediaSource, IMFSample, IMFSourceReader, - MFCreateAttributes, MFCreateMediaType, MFCreateSourceReaderFromMediaSource, - MFEnumDeviceSources, MFMediaType_Video, MFShutdown, MFStartup, + MFCreateAttributes, MFCreateSourceReaderFromMediaSource, + MFEnumDeviceSources, MFShutdown, MFStartup, MFSTARTUP_NOSOCKET, MF_API_VERSION, MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, MF_MT_FRAME_RATE, MF_MT_FRAME_RATE_RANGE_MAX, MF_MT_FRAME_RATE_RANGE_MIN, MF_MT_FRAME_SIZE, - MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE, MF_READWRITE_DISABLE_CONVERTERS, + MF_MT_SUBTYPE, MF_READWRITE_DISABLE_CONVERTERS, }, }, System::Com::{CoInitializeEx, CoUninitialize, COINIT}, @@ -165,7 +165,7 @@ pub mod wmf { fn guid_to_frameformat(guid: GUID) -> Option { match guid { MF_VIDEO_FORMAT_NV12 => Some(FrameFormat::NV12), - MF_VIDEO_FORMAT_RGB24 => Some(FrameFormat::RAWRGB), + MF_VIDEO_FORMAT_RGB24 => Some(FrameFormat::RAWBGR), MF_VIDEO_FORMAT_GRAY => Some(FrameFormat::GRAY), MF_VIDEO_FORMAT_YUY2 => Some(FrameFormat::YUYV), MF_VIDEO_FORMAT_MJPEG => Some(FrameFormat::MJPEG), @@ -173,16 +173,6 @@ pub mod wmf { } } - fn frameformat_to_guid(frameformat: FrameFormat) -> GUID { - match frameformat { - FrameFormat::MJPEG => MF_VIDEO_FORMAT_MJPEG, - FrameFormat::YUYV => MF_VIDEO_FORMAT_YUY2, - FrameFormat::NV12 => MF_VIDEO_FORMAT_NV12, - FrameFormat::GRAY => MF_VIDEO_FORMAT_GRAY, - FrameFormat::RAWRGB => MF_VIDEO_FORMAT_RGB24, - } - } - pub fn initialize_mf() -> Result<(), NokhwaError> { if !(INITIALIZED.load(Ordering::SeqCst)) { if let Err(why) = unsafe { @@ -607,16 +597,6 @@ pub mod wmf { } framerates.push(numerator); }; - if let Ok(fraction_u64) = - unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX) } - { - let mut numerator = (fraction_u64 >> 32) as u32; - let denominator = fraction_u64 as u32; - if denominator != 1 { - numerator = 0; - } - framerates.push(numerator); - }; if let Ok(fraction_u64) = unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE) } { let mut numerator = (fraction_u64 >> 32) as u32; let denominator = fraction_u64 as u32; @@ -1006,88 +986,125 @@ pub mod wmf { } pub fn set_format(&mut self, format: CameraFormat) -> Result<(), NokhwaError> { - // convert to media_type - let media_type: IMFMediaType = match unsafe { MFCreateMediaType() } { - Ok(mt) => mt, - Err(why) => { - return Err(NokhwaError::StructureError { - structure: "IMFMediaType".to_string(), - error: why.to_string(), - }) + // We need to make sure to use all the original attributes of the IMFMediaType to avoid problems. + // Otherwise, constructing IMFMediaType from scratch can sometimes fail due to not exactly matching. + // Therefore, we search for the first media_type that matches and also works correctly. + + let mut last_error : Option = None; + + let mut index = 0; + while let Ok(media_type) = unsafe { + self.source_reader + .GetNativeMediaType(MEDIA_FOUNDATION_FIRST_VIDEO_STREAM, index) + } { + index += 1; + let fourcc = match unsafe { media_type.GetGUID(&MF_MT_SUBTYPE) } { + Ok(fcc) => fcc, + Err(why) => { + return Err(NokhwaError::GetPropertyError { + property: "MF_MT_SUBTYPE".to_string(), + error: why.to_string(), + }) + } + }; + + let frame_fmt = match guid_to_frameformat(fourcc) { + Some(fcc) => fcc, + None => continue, + }; + + if frame_fmt != format.format() { + continue; } - }; - // set relevant things - let resolution = (u64::from(format.resolution().width_x) << 32_u64) - + u64::from(format.resolution().height_y); - let fps = { - let frame_rate_u64 = 0_u64; - let mut bytes: [u8; 8] = frame_rate_u64.to_le_bytes(); - bytes[7] = format.frame_rate() as u8; - bytes[3] = 0x01; - u64::from_le_bytes(bytes) - }; - let fourcc = frameformat_to_guid(format.format()); - // setting to the new media_type - if let Err(why) = unsafe { media_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video) } { - return Err(NokhwaError::SetPropertyError { - property: "MF_MT_MAJOR_TYPE".to_string(), - value: "MFMediaType_Video".to_string(), - error: why.to_string(), - }); - } - if let Err(why) = unsafe { media_type.SetGUID(&MF_MT_SUBTYPE, &fourcc) } { - return Err(NokhwaError::SetPropertyError { - property: "MF_MT_SUBTYPE".to_string(), - value: format!("{:?}", fourcc), - error: why.to_string(), - }); - } - if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_SIZE, resolution) } { - return Err(NokhwaError::SetPropertyError { - property: "MF_MT_FRAME_SIZE".to_string(), - value: resolution.to_string(), - error: why.to_string(), - }); - } - if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE, fps) } { - return Err(NokhwaError::SetPropertyError { - property: "MF_MT_FRAME_RATE".to_string(), - value: fps.to_string(), - error: why.to_string(), - }); - } - if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE_RANGE_MIN, fps) } { - return Err(NokhwaError::SetPropertyError { - property: "MF_MT_FRAME_RATE_RANGE_MIN".to_string(), - value: fps.to_string(), - error: why.to_string(), - }); - } - if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX, fps) } { - return Err(NokhwaError::SetPropertyError { - property: "MF_MT_FRAME_RATE_RANGE_MAX".to_string(), - value: fps.to_string(), - error: why.to_string(), - }); + let (width, height) = match unsafe { media_type.GetUINT64(&MF_MT_FRAME_SIZE) } { + Ok(res_u64) => { + let width = (res_u64 >> 32) as u32; + let height = res_u64 as u32; // the cast will truncate the upper bits + (width, height) + } + Err(why) => { + return Err(NokhwaError::GetPropertyError { + property: "MF_MT_FRAME_SIZE".to_string(), + error: why.to_string(), + }) + } + }; + + if (Resolution { width_x: width, height_y: height }) != format.resolution() { + continue; + } + + // MFRatio is represented as 2 u32s in memory. This means we can convert it to 2 + let framerate_list = { + let mut framerates = vec![0_u32; 3]; + if let Ok(fraction_u64) = + unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX) } + { + let mut numerator = (fraction_u64 >> 32) as u32; + let denominator = fraction_u64 as u32; + if denominator != 1 { + numerator = 0; + } + framerates.push(numerator); + }; + if let Ok(fraction_u64) = unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE) } { + let mut numerator = (fraction_u64 >> 32) as u32; + let denominator = fraction_u64 as u32; + if denominator != 1 { + numerator = 0; + } + framerates.push(numerator); + }; + if let Ok(fraction_u64) = + unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MIN) } + { + let mut numerator = (fraction_u64 >> 32) as u32; + let denominator = fraction_u64 as u32; + if denominator != 1 { + numerator = 0; + } + framerates.push(numerator); + }; + framerates + }; + + for frame_rate in framerate_list { + if frame_rate == format.frame_rate() { + let result = unsafe { + self.source_reader.SetCurrentMediaType( + MEDIA_FOUNDATION_FIRST_VIDEO_STREAM, + None, + &media_type, + ) + }; + + match result { + Ok(_) => { + self.device_format = format; + self.format_refreshed()?; + return Ok(()); + }, + Err(why) => { + last_error = Some(NokhwaError::SetPropertyError { + property: "MEDIA_FOUNDATION_FIRST_VIDEO_STREAM".to_string(), + value: format!("{media_type:?}"), + error: why.to_string(), + }); + } + } + } + } } - if let Err(why) = unsafe { - self.source_reader.SetCurrentMediaType( - MEDIA_FOUNDATION_FIRST_VIDEO_STREAM, - None, - &media_type, - ) - } { - return Err(NokhwaError::SetPropertyError { - property: "MEDIA_FOUNDATION_FIRST_VIDEO_STREAM".to_string(), - value: format!("{media_type:?}"), - error: why.to_string(), - }); + if let Some(err) = last_error { + return Err(err); } - self.device_format = format; - self.format_refreshed()?; - Ok(()) + + Err(NokhwaError::InitializeError { + backend: ApiBackend::MediaFoundation, + error: "Failed to fulfill requested format".to_string(), + }) } pub fn is_stream_open(&self) -> bool { diff --git a/nokhwa-core/src/pixel_format.rs b/nokhwa-core/src/pixel_format.rs index c197d36..0e8c43e 100644 --- a/nokhwa-core/src/pixel_format.rs +++ b/nokhwa-core/src/pixel_format.rs @@ -15,7 +15,7 @@ */ use crate::error::NokhwaError; use crate::types::{ - buf_mjpeg_to_rgb, buf_nv12_to_rgb, buf_yuyv422_to_rgb, color_frame_formats, frame_formats, + buf_bgr_to_rgb, buf_mjpeg_to_rgb, buf_nv12_to_rgb, buf_yuyv422_to_rgb, color_frame_formats, frame_formats, mjpeg_to_rgb, nv12_to_rgb, yuyv422_to_rgb, FrameFormat, Resolution, }; use image::{Luma, LumaA, Pixel, Rgb, Rgba}; @@ -76,6 +76,16 @@ impl FormatDecoder for RgbFormat { }) .collect()), FrameFormat::RAWRGB => Ok(data.to_vec()), + FrameFormat::RAWBGR => { + let mut rgb = vec![0u8; data.len()]; + data.chunks_exact(3).enumerate().for_each(|(idx, px)| { + let index = idx * 3; + rgb[index] = px[2]; + rgb[index + 1] = px[1]; + rgb[index + 2] = px[0]; + }); + Ok(rgb) + }, FrameFormat::NV12 => nv12_to_rgb(resolution, data, false), } } @@ -111,6 +121,7 @@ impl FormatDecoder for RgbFormat { dest.copy_from_slice(data); Ok(()) } + FrameFormat::RAWBGR => buf_bgr_to_rgb(resolution, data, dest), FrameFormat::NV12 => buf_nv12_to_rgb(resolution, data, dest, false), } } @@ -150,6 +161,10 @@ impl FormatDecoder for RgbAFormat { .chunks_exact(3) .flat_map(|x| [x[0], x[1], x[2], 255]) .collect()), + FrameFormat::RAWBGR => Ok(data + .chunks_exact(3) + .flat_map(|x| [x[2], x[1], x[0], 255]) + .collect()), FrameFormat::NV12 => nv12_to_rgb(resolution, data, true), } } @@ -193,6 +208,16 @@ impl FormatDecoder for RgbAFormat { }); Ok(()) } + FrameFormat::RAWBGR => { + data.chunks_exact(3).enumerate().for_each(|(idx, px)| { + let index = idx * 4; + dest[index] = px[2]; + dest[index + 1] = px[1]; + dest[index + 2] = px[0]; + dest[index + 3] = 255; + }); + Ok(()) + } FrameFormat::NV12 => buf_nv12_to_rgb(resolution, data, dest, true), } } @@ -253,6 +278,10 @@ impl FormatDecoder for LumaFormat { .chunks(3) .map(|px| ((i32::from(px[0]) + i32::from(px[1]) + i32::from(px[2])) / 3) as u8) .collect()), + FrameFormat::RAWBGR => Ok(data + .chunks(3) + .map(|px| ((i32::from(px[2]) + i32::from(px[1]) + i32::from(px[0])) / 3) as u8) + .collect()), } } @@ -268,11 +297,10 @@ impl FormatDecoder for LumaFormat { FrameFormat::MJPEG | FrameFormat::YUYV | FrameFormat::NV12 => { Err(NokhwaError::ProcessFrameError { src: fcc, - destination: "Luma => RGB".to_string(), + destination: "RGB => Luma".to_string(), error: "Conversion Error".to_string(), }) } - FrameFormat::GRAY => { data.iter().zip(dest.iter_mut()).for_each(|(pxv, d)| { *d = *pxv; @@ -281,7 +309,12 @@ impl FormatDecoder for LumaFormat { } FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError { src: fcc, - destination: "RGB => RGB".to_string(), + destination: "RGB => Luma".to_string(), + error: "Conversion Error".to_string(), + }), + FrameFormat::RAWBGR => Err(NokhwaError::ProcessFrameError { + src: fcc, + destination: "BGR => Luma".to_string(), error: "Conversion Error".to_string(), }), } @@ -340,7 +373,12 @@ impl FormatDecoder for LumaAFormat { FrameFormat::GRAY => Ok(data.iter().flat_map(|x| [*x, 255]).collect()), FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError { src: fcc, - destination: "RGB => RGB".to_string(), + destination: "RGB => LumaA".to_string(), + error: "Conversion Error".to_string(), + }), + FrameFormat::RAWBGR => Err(NokhwaError::ProcessFrameError { + src: fcc, + destination: "BGR => LumaA".to_string(), error: "Conversion Error".to_string(), }), } @@ -393,7 +431,12 @@ impl FormatDecoder for LumaAFormat { } FrameFormat::RAWRGB => Err(NokhwaError::ProcessFrameError { src: fcc, - destination: "RGB => RGB".to_string(), + destination: "RGB => LumaA".to_string(), + error: "Conversion Error".to_string(), + }), + FrameFormat::RAWBGR => Err(NokhwaError::ProcessFrameError { + src: fcc, + destination: "BGR => LumaA".to_string(), error: "Conversion Error".to_string(), }), } diff --git a/nokhwa-core/src/traits.rs b/nokhwa-core/src/traits.rs index 79d4db0..9de70ed 100644 --- a/nokhwa-core/src/traits.rs +++ b/nokhwa-core/src/traits.rs @@ -173,7 +173,7 @@ CaptureBackendTrait { let cfmt = self.camera_format(); let resolution = cfmt.resolution(); let pxwidth = match cfmt.format() { - FrameFormat::MJPEG | FrameFormat::YUYV | FrameFormat::RAWRGB | FrameFormat::NV12 => 3, + FrameFormat::MJPEG | FrameFormat::YUYV | FrameFormat::RAWRGB | FrameFormat::RAWBGR | FrameFormat::NV12 => 3, FrameFormat::GRAY => 1, }; if alpha { diff --git a/nokhwa-core/src/types.rs b/nokhwa-core/src/types.rs index fb39dac..975c9e4 100644 --- a/nokhwa-core/src/types.rs +++ b/nokhwa-core/src/types.rs @@ -299,6 +299,7 @@ pub enum FrameFormat { NV12, GRAY, RAWRGB, + RAWBGR, } impl Display for FrameFormat { @@ -316,6 +317,9 @@ impl Display for FrameFormat { FrameFormat::RAWRGB => { write!(f, "RAWRGB") } + FrameFormat::RAWBGR => { + write!(f, "RAWBGR") + } FrameFormat::NV12 => { write!(f, "NV12") } @@ -331,6 +335,7 @@ impl FromStr for FrameFormat { "YUYV" => Ok(FrameFormat::YUYV), "GRAY" => Ok(FrameFormat::GRAY), "RAWRGB" => Ok(FrameFormat::RAWRGB), + "RAWBGR" => Ok(FrameFormat::RAWBGR), "NV12" => Ok(FrameFormat::NV12), _ => Err(NokhwaError::StructureError { structure: "FrameFormat".to_string(), @@ -349,6 +354,7 @@ pub const fn frame_formats() -> &'static [FrameFormat] { FrameFormat::NV12, FrameFormat::GRAY, FrameFormat::RAWRGB, + FrameFormat::RAWBGR, ] } @@ -360,6 +366,7 @@ pub const fn color_frame_formats() -> &'static [FrameFormat] { FrameFormat::YUYV, FrameFormat::NV12, FrameFormat::RAWRGB, + FrameFormat::RAWBGR, ] } @@ -1812,3 +1819,55 @@ pub fn buf_nv12_to_rgb( Ok(()) } + +#[allow(clippy::similar_names)] +#[inline] +pub fn buf_bgr_to_rgb( + resolution: Resolution, + data: &[u8], + out: &mut [u8], +) -> Result<(), NokhwaError> { + let width = resolution.width(); + let height = resolution.height(); + + if width % 2 != 0 || height % 2 != 0 { + return Err(NokhwaError::ProcessFrameError { + src: FrameFormat::RAWBGR, + destination: "RGB".to_string(), + error: "bad resolution".to_string(), + }); + } + + let input_size = (width * height * 3) as usize; // BGR is 3 bytes per pixel + let output_size = (width * height * 3) as usize; // RGB is 3 bytes per pixel + + if data.len() != input_size { + return Err(NokhwaError::ProcessFrameError { + src: FrameFormat::RAWBGR, + destination: "RGB".to_string(), + error: "bad input buffer size".to_string(), + }); + } + + if out.len() != output_size { + return Err(NokhwaError::ProcessFrameError { + src: FrameFormat::RAWBGR, + destination: "RGB".to_string(), + error: "bad output buffer size".to_string(), + }); + } + + for (idx, chunk) in data.chunks_exact(3).enumerate() { + // BGR Format: [Blue, Green, Red] + let b = chunk[0]; + let g = chunk[1]; + let r = chunk[2]; + + let out_idx = idx * 3; // 3 bytes per pixel in RGB + out[out_idx] = r; // Red + out[out_idx + 1] = g; // Green + out[out_idx + 2] = b; // Blue + } + + Ok(()) +}