Skip to content

Commit

Permalink
fix: vision crash when computer awake from sleep (and for some reason…
Browse files Browse the repository at this point in the history
… get weird screen capture size)
  • Loading branch information
louis030195 committed Aug 12, 2024
1 parent e29b224 commit 417a499
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 12 deletions.
11 changes: 9 additions & 2 deletions screenpipe-vision/src/core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -124,14 +124,21 @@ pub async fn continuous_capture(
);
let arc_monitor = arc_monitor.clone();
let (image, image_hash, _capture_duration) = capture_screenshot(arc_monitor).await;
let current_average = compare_with_previous_image(
let current_average = match compare_with_previous_image(
&previous_image,
&image,
&mut max_average,
frame_counter,
&mut max_avg_value,
)
.await;
.await
{
Ok(avg) => avg,
Err(e) => {
error!("Error comparing images: {}", e);
0.0 // or some default value
}
};

// Account for situation when there is no previous image
let current_average = if previous_image.is_none() {
Expand Down
21 changes: 11 additions & 10 deletions screenpipe-vision/src/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,14 @@ pub fn calculate_hash(image: &DynamicImage) -> u64 {
hasher.finish()
}

pub fn compare_images_histogram(image1: &DynamicImage, image2: &DynamicImage) -> f64 {
pub fn compare_images_histogram(
image1: &DynamicImage,
image2: &DynamicImage,
) -> anyhow::Result<f64> {
let image_one = image1.to_luma8();
let image_two = image2.to_luma8();
let result =
image_compare::gray_similarity_histogram(Metric::Hellinger, &image_one, &image_two)
.expect("Images had different dimensions");
result
image_compare::gray_similarity_histogram(Metric::Hellinger, &image_one, &image_two)
.map_err(|e| anyhow::anyhow!("Failed to compare images: {}", e))
}

pub fn compare_images_ssim(image1: &DynamicImage, image2: &DynamicImage) -> f64 {
Expand Down Expand Up @@ -151,14 +152,14 @@ pub async fn capture_screenshot(monitor: Arc<Monitor>) -> (DynamicImage, u64, Du
pub async fn compare_with_previous_image(
previous_image: &Option<Arc<DynamicImage>>,
current_image: &DynamicImage,
max_average: &mut Option<MaxAverageFrame>, // Prefix with underscore if not used
max_average: &mut Option<MaxAverageFrame>,
frame_number: u64,
max_avg_value: &mut f64,
) -> f64 {
) -> anyhow::Result<f64> {
let mut current_average = 0.0;
if let Some(prev_image) = previous_image {
let histogram_diff = compare_images_histogram(prev_image, &current_image);
let ssim_diff = 1.0 - compare_images_ssim(prev_image, &current_image);
let histogram_diff = compare_images_histogram(prev_image, current_image)?;
let ssim_diff = 1.0 - compare_images_ssim(prev_image, current_image);
current_average = (histogram_diff + ssim_diff) / 2.0;
let max_avg_frame_number = max_average.as_ref().map_or(0, |frame| frame.frame_number);
debug!(
Expand All @@ -168,7 +169,7 @@ pub async fn compare_with_previous_image(
} else {
debug!("No previous image to compare for frame {}", frame_number);
}
current_average
Ok(current_average)
}

pub async fn save_text_files(
Expand Down

1 comment on commit 417a499

@louis030195
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@m13v i was not so familiar with error handling in rust until now, we should start being more careful on error handling, basically ".expect" crash the function and parent, which was ending the recording loop (i made same mistake in the audio recording, fixed)

the error was just some random things when computer was sleeping for long time and reawake

Please sign in to comment.