diff --git a/crates/egui/src/containers/scroll_area.rs b/crates/egui/src/containers/scroll_area.rs
index 3b3925c9dcd..ee6bf86ee01 100644
--- a/crates/egui/src/containers/scroll_area.rs
+++ b/crates/egui/src/containers/scroll_area.rs
@@ -1,8 +1,8 @@
 #![allow(clippy::needless_range_loop)]
 
 use crate::{
-    emath, epaint, lerp, pass_state, pos2, remap, remap_clamp, vec2, Context, Id, NumExt, Pos2,
-    Rangef, Rect, Sense, Ui, UiBuilder, UiKind, UiStackInfo, Vec2, Vec2b,
+    emath, epaint, lerp, pass_state, pos2, remap, remap_clamp, Context, Id, NumExt, Pos2, Rangef,
+    Rect, Sense, Ui, UiBuilder, UiKind, UiStackInfo, Vec2, Vec2b,
 };
 
 #[derive(Clone, Copy, Debug)]
@@ -1067,21 +1067,35 @@ impl Prepared {
                 )
             };
 
-            let handle_rect = if d == 0 {
-                Rect::from_min_max(
-                    pos2(from_content(state.offset.x), cross.min),
-                    pos2(from_content(state.offset.x + inner_rect.width()), cross.max),
-                )
-            } else {
-                Rect::from_min_max(
-                    pos2(cross.min, from_content(state.offset.y)),
-                    pos2(
-                        cross.max,
-                        from_content(state.offset.y + inner_rect.height()),
-                    ),
-                )
+            let calculate_handle_rect = |d, offset: &Vec2| {
+                let handle_size = if d == 0 {
+                    from_content(offset.x + inner_rect.width()) - from_content(offset.x)
+                } else {
+                    from_content(offset.y + inner_rect.height()) - from_content(offset.y)
+                }
+                .max(scroll_style.handle_min_length);
+
+                let handle_start_point = remap_clamp(
+                    offset[d],
+                    0.0..=max_offset[d],
+                    scroll_bar_rect.min[d]..=(scroll_bar_rect.max[d] - handle_size),
+                );
+
+                if d == 0 {
+                    Rect::from_min_max(
+                        pos2(handle_start_point, cross.min),
+                        pos2(handle_start_point + handle_size, cross.max),
+                    )
+                } else {
+                    Rect::from_min_max(
+                        pos2(cross.min, handle_start_point),
+                        pos2(cross.max, handle_start_point + handle_size),
+                    )
+                }
             };
 
+            let handle_rect = calculate_handle_rect(d, &state.offset);
+
             let interact_id = id.with(d);
             let sense = if self.scrolling_enabled {
                 Sense::click_and_drag()
@@ -1110,8 +1124,8 @@ impl Prepared {
                 let new_handle_top = pointer_pos[d] - *scroll_start_offset_from_top_left;
                 state.offset[d] = remap(
                     new_handle_top,
-                    scroll_bar_rect.min[d]..=scroll_bar_rect.max[d],
-                    0.0..=content_size[d],
+                    scroll_bar_rect.min[d]..=(scroll_bar_rect.max[d] - handle_rect.size()[d]),
+                    0.0..=max_offset[d],
                 );
 
                 // some manual action taken, scroll not stuck
@@ -1131,31 +1145,7 @@ impl Prepared {
 
             if ui.is_rect_visible(outer_scroll_bar_rect) {
                 // Avoid frame-delay by calculating a new handle rect:
-                let mut handle_rect = if d == 0 {
-                    Rect::from_min_max(
-                        pos2(from_content(state.offset.x), cross.min),
-                        pos2(from_content(state.offset.x + inner_rect.width()), cross.max),
-                    )
-                } else {
-                    Rect::from_min_max(
-                        pos2(cross.min, from_content(state.offset.y)),
-                        pos2(
-                            cross.max,
-                            from_content(state.offset.y + inner_rect.height()),
-                        ),
-                    )
-                };
-                let min_handle_size = scroll_style.handle_min_length;
-                if handle_rect.size()[d] < min_handle_size {
-                    handle_rect = Rect::from_center_size(
-                        handle_rect.center(),
-                        if d == 0 {
-                            vec2(min_handle_size, handle_rect.size().y)
-                        } else {
-                            vec2(handle_rect.size().x, min_handle_size)
-                        },
-                    );
-                }
+                let handle_rect = calculate_handle_rect(d, &state.offset);
 
                 let visuals = if scrolling_enabled {
                     // Pick visuals based on interaction with the handle.