Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix bug in MSFoundation backend where media formats are created not exactly as expected by the device and fail to open #197

Merged
merged 5 commits into from
Jan 4, 2025
Merged

Fix bug in MSFoundation backend where media formats are created not exactly as expected by the device and fail to open #197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ef19477
Fix infinite loop when guid_to_frameformat fails
eduramiba Dec 4, 2024
bbb5bb4
Merge branch 'l1npengtul:0.10' into 0.10
eduramiba Dec 5, 2024
b60bf85
Merge branch 'l1npengtul:0.10' into 0.10
eduramiba Jan 3, 2025
ff190f1
Fix bug in MSFoundation backend where media formats are created not e…
eduramiba Jan 3, 2025
73eee19
Fix compilation issue
eduramiba Jan 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions nokhwa-bindings-linux/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -540,6 +540,7 @@ mod internal {
FrameFormat::YUYV => FourCC::new(b"YUYV"),
FrameFormat::GRAY => FourCC::new(b"GRAY"),
FrameFormat::RAWRGB => FourCC::new(b"RGB3"),
FrameFormat::RAWBGR => FourCC::new(b"BGR3"),
FrameFormat::NV12 => FourCC::new(b"NV12"),
};

Expand Down Expand Up @@ -912,6 +913,7 @@ mod internal {
"MJPG" => Some(FrameFormat::MJPEG),
"GRAY" => Some(FrameFormat::GRAY),
"RGB3" => Some(FrameFormat::RAWRGB),
"BGR3" => Some(FrameFormat::RAWBGR),
"NV12" => Some(FrameFormat::NV12),
_ => None,
}
Expand All @@ -923,6 +925,7 @@ mod internal {
FrameFormat::YUYV => FourCC::new(b"YUYV"),
FrameFormat::GRAY => FourCC::new(b"GRAY"),
FrameFormat::RAWRGB => FourCC::new(b"RGB3"),
FrameFormat::RAWBGR => FourCC::new(b"BGR3"),
FrameFormat::NV12 => FourCC::new(b"NV12"),
}
}
Expand Down
7 changes: 7 additions & 0 deletions nokhwa-bindings-macos/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2285,6 +2285,13 @@ mod internal {
FrameFormat::GRAY => kCMPixelFormat_8IndexedGray_WhiteIsZero,
FrameFormat::NV12 => kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange,
FrameFormat::RAWRGB => kCMPixelFormat_24RGB,
FrameFormat::RAWBGR => {
return Err(NokhwaError::SetPropertyError {
property: "setVideoSettings".to_string(),
value: "set frame format".to_string(),
error: "Unsupported frame format BGR".to_string(),
});
}
};
let obj = CFNumber::from(cmpixelfmt as i32);
let obj = obj.as_CFTypeRef() as *mut Object;
Expand Down
221 changes: 119 additions & 102 deletions nokhwa-bindings-windows/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ pub mod wmf {
};
use windows::Win32::Media::DirectShow::{CameraControl_Flags_Auto, CameraControl_Flags_Manual};
use windows::Win32::Media::MediaFoundation::{
IMFMediaType, MFCreateSample, MF_SOURCE_READER_FIRST_VIDEO_STREAM,
MFCreateSample, MF_SOURCE_READER_FIRST_VIDEO_STREAM,
};
use windows::{
core::{Interface, GUID, PWSTR},
Expand All @@ -65,14 +65,14 @@ pub mod wmf {
KernelStreaming::GUID_NULL,
MediaFoundation::{
IMFActivate, IMFAttributes, IMFMediaSource, IMFSample, IMFSourceReader,
MFCreateAttributes, MFCreateMediaType, MFCreateSourceReaderFromMediaSource,
MFEnumDeviceSources, MFMediaType_Video, MFShutdown, MFStartup,
MFCreateAttributes, MFCreateSourceReaderFromMediaSource,
MFEnumDeviceSources, MFShutdown, MFStartup,
MFSTARTUP_NOSOCKET, MF_API_VERSION, MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME,
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE,
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID,
MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, MF_MT_FRAME_RATE,
MF_MT_FRAME_RATE_RANGE_MAX, MF_MT_FRAME_RATE_RANGE_MIN, MF_MT_FRAME_SIZE,
MF_MT_MAJOR_TYPE, MF_MT_SUBTYPE, MF_READWRITE_DISABLE_CONVERTERS,
MF_MT_SUBTYPE, MF_READWRITE_DISABLE_CONVERTERS,
},
},
System::Com::{CoInitializeEx, CoUninitialize, COINIT},
Expand Down Expand Up @@ -165,24 +165,14 @@ pub mod wmf {
fn guid_to_frameformat(guid: GUID) -> Option<FrameFormat> {
match guid {
MF_VIDEO_FORMAT_NV12 => Some(FrameFormat::NV12),
MF_VIDEO_FORMAT_RGB24 => Some(FrameFormat::RAWRGB),
MF_VIDEO_FORMAT_RGB24 => Some(FrameFormat::RAWBGR),
MF_VIDEO_FORMAT_GRAY => Some(FrameFormat::GRAY),
MF_VIDEO_FORMAT_YUY2 => Some(FrameFormat::YUYV),
MF_VIDEO_FORMAT_MJPEG => Some(FrameFormat::MJPEG),
_ => None,
}
}

fn frameformat_to_guid(frameformat: FrameFormat) -> GUID {
match frameformat {
FrameFormat::MJPEG => MF_VIDEO_FORMAT_MJPEG,
FrameFormat::YUYV => MF_VIDEO_FORMAT_YUY2,
FrameFormat::NV12 => MF_VIDEO_FORMAT_NV12,
FrameFormat::GRAY => MF_VIDEO_FORMAT_GRAY,
FrameFormat::RAWRGB => MF_VIDEO_FORMAT_RGB24,
}
}

pub fn initialize_mf() -> Result<(), NokhwaError> {
if !(INITIALIZED.load(Ordering::SeqCst)) {
if let Err(why) = unsafe {
Expand Down Expand Up @@ -607,16 +597,6 @@ pub mod wmf {
}
framerates.push(numerator);
};
if let Ok(fraction_u64) =
unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX) }
{
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
if denominator != 1 {
numerator = 0;
}
framerates.push(numerator);
};
if let Ok(fraction_u64) = unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE) } {
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
Expand Down Expand Up @@ -1006,88 +986,125 @@ pub mod wmf {
}

pub fn set_format(&mut self, format: CameraFormat) -> Result<(), NokhwaError> {
// convert to media_type
let media_type: IMFMediaType = match unsafe { MFCreateMediaType() } {
Ok(mt) => mt,
Err(why) => {
return Err(NokhwaError::StructureError {
structure: "IMFMediaType".to_string(),
error: why.to_string(),
})
// We need to make sure to use all the original attributes of the IMFMediaType to avoid problems.
// Otherwise, constructing IMFMediaType from scratch can sometimes fail due to not exactly matching.
// Therefore, we search for the first media_type that matches and also works correctly.

let mut last_error : Option<NokhwaError> = None;

let mut index = 0;
while let Ok(media_type) = unsafe {
self.source_reader
.GetNativeMediaType(MEDIA_FOUNDATION_FIRST_VIDEO_STREAM, index)
} {
index += 1;
let fourcc = match unsafe { media_type.GetGUID(&MF_MT_SUBTYPE) } {
Ok(fcc) => fcc,
Err(why) => {
return Err(NokhwaError::GetPropertyError {
property: "MF_MT_SUBTYPE".to_string(),
error: why.to_string(),
})
}
};

let frame_fmt = match guid_to_frameformat(fourcc) {
Some(fcc) => fcc,
None => continue,
};

if frame_fmt != format.format() {
continue;
}
};

// set relevant things
let resolution = (u64::from(format.resolution().width_x) << 32_u64)
+ u64::from(format.resolution().height_y);
let fps = {
let frame_rate_u64 = 0_u64;
let mut bytes: [u8; 8] = frame_rate_u64.to_le_bytes();
bytes[7] = format.frame_rate() as u8;
bytes[3] = 0x01;
u64::from_le_bytes(bytes)
};
let fourcc = frameformat_to_guid(format.format());
// setting to the new media_type
if let Err(why) = unsafe { media_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_MAJOR_TYPE".to_string(),
value: "MFMediaType_Video".to_string(),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetGUID(&MF_MT_SUBTYPE, &fourcc) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_SUBTYPE".to_string(),
value: format!("{:?}", fourcc),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_SIZE, resolution) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_FRAME_SIZE".to_string(),
value: resolution.to_string(),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE, fps) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_FRAME_RATE".to_string(),
value: fps.to_string(),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE_RANGE_MIN, fps) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_FRAME_RATE_RANGE_MIN".to_string(),
value: fps.to_string(),
error: why.to_string(),
});
}
if let Err(why) = unsafe { media_type.SetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX, fps) } {
return Err(NokhwaError::SetPropertyError {
property: "MF_MT_FRAME_RATE_RANGE_MAX".to_string(),
value: fps.to_string(),
error: why.to_string(),
});
let (width, height) = match unsafe { media_type.GetUINT64(&MF_MT_FRAME_SIZE) } {
Ok(res_u64) => {
let width = (res_u64 >> 32) as u32;
let height = res_u64 as u32; // the cast will truncate the upper bits
(width, height)
}
Err(why) => {
return Err(NokhwaError::GetPropertyError {
property: "MF_MT_FRAME_SIZE".to_string(),
error: why.to_string(),
})
}
};

if (Resolution { width_x: width, height_y: height }) != format.resolution() {
continue;
}

// MFRatio is represented as 2 u32s in memory. This means we can convert it to 2
let framerate_list = {
let mut framerates = vec![0_u32; 3];
if let Ok(fraction_u64) =
unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MAX) }
{
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
if denominator != 1 {
numerator = 0;
}
framerates.push(numerator);
};
if let Ok(fraction_u64) = unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE) } {
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
if denominator != 1 {
numerator = 0;
}
framerates.push(numerator);
};
if let Ok(fraction_u64) =
unsafe { media_type.GetUINT64(&MF_MT_FRAME_RATE_RANGE_MIN) }
{
let mut numerator = (fraction_u64 >> 32) as u32;
let denominator = fraction_u64 as u32;
if denominator != 1 {
numerator = 0;
}
framerates.push(numerator);
};
framerates
};

for frame_rate in framerate_list {
if frame_rate == format.frame_rate() {
let result = unsafe {
self.source_reader.SetCurrentMediaType(
MEDIA_FOUNDATION_FIRST_VIDEO_STREAM,
None,
&media_type,
)
};

match result {
Ok(_) => {
self.device_format = format;
self.format_refreshed()?;
return Ok(());
},
Err(why) => {
last_error = Some(NokhwaError::SetPropertyError {
property: "MEDIA_FOUNDATION_FIRST_VIDEO_STREAM".to_string(),
value: format!("{media_type:?}"),
error: why.to_string(),
});
}
}
}
}
}

if let Err(why) = unsafe {
self.source_reader.SetCurrentMediaType(
MEDIA_FOUNDATION_FIRST_VIDEO_STREAM,
None,
&media_type,
)
} {
return Err(NokhwaError::SetPropertyError {
property: "MEDIA_FOUNDATION_FIRST_VIDEO_STREAM".to_string(),
value: format!("{media_type:?}"),
error: why.to_string(),
});
if let Some(err) = last_error {
return Err(err);
}
self.device_format = format;
self.format_refreshed()?;
Ok(())

Err(NokhwaError::InitializeError {
backend: ApiBackend::MediaFoundation,
error: "Failed to fulfill requested format".to_string(),
})
}

pub fn is_stream_open(&self) -> bool {
Expand Down
Loading
Loading