diff --git a/src/components/ImageView/ImageView.js b/src/components/ImageView/ImageView.js
index 08b82bedf..5e56190bd 100644
--- a/src/components/ImageView/ImageView.js
+++ b/src/components/ImageView/ImageView.js
@@ -506,7 +506,9 @@ export default observer(
     crosshairRef = createRef();
     handleDeferredMouseDown = null;
     deferredClickTimeout = [];
-    skipMouseUp = false;
+    skipNextMouseDown = false;
+    skipNextClick = false;
+    skipNextMouseUp = false;
     mouseDownPoint = null;
 
     constructor(props) {
@@ -520,10 +522,10 @@ export default observer(
       const { item } = this.props;
 
       if (isFF(FF_DEV_1442)) {
-        this.handleDeferredMouseDown?.();
+        this.handleDeferredMouseDown?.(true);
       }
-      if (this.skipMouseUp) {
-        this.skipMouseUp = false;
+      if (this.skipNextClick) {
+        this.skipNextClick = false;
         return;
       }
 
@@ -556,8 +558,8 @@ export default observer(
     };
 
     handleDeferredClick = (handleDeferredMouseDownCallback, handleDeselection, eligibleToDeselect = false) => {
-      this.handleDeferredMouseDown = () => {
-        if (eligibleToDeselect) {
+      this.handleDeferredMouseDown = (wasClicked) => {
+        if (wasClicked && eligibleToDeselect) {
           handleDeselection();
         }
         handleDeferredMouseDownCallback();
@@ -566,7 +568,7 @@ export default observer(
       };
       this.resetDeferredClickTimeout();
       this.deferredClickTimeout.push(setTimeout(() => {
-        this.handleDeferredMouseDown?.();
+        this.handleDeferredMouseDown?.(false);
       }, this.props.item.annotation.isDrawing ? 0 : 100));
     };
 
@@ -575,6 +577,7 @@ export default observer(
       const isPanTool = item.getToolsManager().findSelectedTool()?.fullName === 'ZoomPanTool';
       const isMoveTool = item.getToolsManager().findSelectedTool()?.fullName === 'MoveTool';
 
+      this.skipNextMouseDown = this.skipNextMouseUp = this.skipNextClick = false;
       if (isFF(FF_LSDV_4930)) {
         this.mouseDownPoint = { x: e.evt.offsetX, y: e.evt.offsetY };
       }
@@ -626,6 +629,10 @@ export default observer(
 
           this.canvasX = left;
           this.canvasY = top;
+          if (this.skipNextMouseDown) {
+            this.skipNextMouseDown = false;
+            return true;
+          }
           item.event('mousedown', e, x, y);
 
           return true;
@@ -652,7 +659,9 @@ export default observer(
 
         const handleDeselection = () => {
           item.annotation.unselectAll();
-          this.skipMouseUp = true;
+          this.skipNextMouseDown = true;
+          this.skipNextMouseUp = true;
+          this.skipNextClick = true;
         };
 
         this.handleDeferredClick(handleMouseDown, handleDeselection, eligibleToDeselect);
@@ -680,7 +689,7 @@ export default observer(
 
       item.freezeHistory();
 
-      return item.event('mouseup', e, x - this.canvasX, y - this.canvasY);
+      return this.triggerMouseUp(e, x - this.canvasX, y - this.canvasY);
     };
 
     handleGlobalMouseMove = e => {
@@ -705,7 +714,17 @@ export default observer(
       item.freezeHistory();
       item.setSkipInteractions(false);
 
-      return item.event('mouseup', e, e.evt.offsetX, e.evt.offsetY);
+      return this.triggerMouseUp(e, e.evt.offsetX, e.evt.offsetY);
+    };
+
+    triggerMouseUp = (e, x, y) => {
+      if (this.skipNextMouseUp) {
+        this.skipNextMouseUp = false;
+        return;
+      }
+      const { item } = this.props;
+
+      return item.event('mouseup', e, x, y);
     };
 
     handleMouseMove = e => {
@@ -721,7 +740,7 @@ export default observer(
 
       if (isFF(FF_DEV_1442) && isDragging) {
         this.resetDeferredClickTimeout();
-        this.handleDeferredMouseDown?.();
+        this.handleDeferredMouseDown?.(false);
       }
 
       if ((isMouseWheelClick || isShiftDrag) && item.zoomScale > 1) {
diff --git a/tests/functional/data/image_segmentation/tools/rect.ts b/tests/functional/data/image_segmentation/tools/rect.ts
new file mode 100644
index 000000000..55b1b0818
--- /dev/null
+++ b/tests/functional/data/image_segmentation/tools/rect.ts
@@ -0,0 +1,32 @@
+export const rectangleToolAndLabelsConfig = `<View>
+  <Image name="img" value="$image" />
+  <Rectangle name="rect" toName="img" />
+  <Labels name="labels" toName="img">
+    <Label value="Label 1" background="blue" />
+    <Label value="Label 2" background="red" />
+  </Labels>
+ </View>`;
+
+export const simpleImageData = {
+  image: 'https://htx-misc.s3.amazonaws.com/opensource/label-studio/examples/images/nick-owuor-astro-nic-visuals-wDifg5xc9Z4-unsplash.jpg',
+};
+
+export const simpleRectangleResult = [
+  {
+    'id': 'rect_1',
+    'original_width': 2242,
+    'original_height': 2802,
+    'image_rotation': 0,
+    'value': {
+      'x': 20,
+      'y': 20,
+      'width': 20,
+      'height': 20,
+      'rotation': 0,
+    },
+    'from_name': 'rect',
+    'to_name': 'img',
+    'type': 'rectangle',
+    'origin': 'manual',
+  },
+];
diff --git a/tests/functional/specs/image_segmentation/tools/rect.cy.ts b/tests/functional/specs/image_segmentation/tools/rect.cy.ts
new file mode 100644
index 000000000..ef176662b
--- /dev/null
+++ b/tests/functional/specs/image_segmentation/tools/rect.cy.ts
@@ -0,0 +1,99 @@
+import { ImageView, Labels, LabelStudio, Sidebar } from '@heartexlabs/ls-test/helpers/LSF';
+import {
+  rectangleToolAndLabelsConfig,
+  simpleImageData,
+  simpleRectangleResult
+} from 'data/image_segmentation/tools/rect';
+import { FF_DEV_1442 } from '../../../../../src/utils/feature-flags';
+
+describe('Rectangle tool', () => {
+  it('should not draw rectangle when clicking outside to unselect (FF_DEV_1442 = true)', () => {
+    LabelStudio.addFeatureFlagsOnPageLoad({
+      [FF_DEV_1442]: true,
+    });
+
+    LabelStudio.params()
+      .config(rectangleToolAndLabelsConfig)
+      .data(simpleImageData)
+      .withResult(simpleRectangleResult)
+      .init();
+
+    LabelStudio.waitForObjectsReady();
+
+    Sidebar.hasRegions(1);
+    Sidebar.toggleRegionSelection(0);
+    Labels.select('Label 1');
+
+    ImageView.clickAtRelative(0.5, 0.5);
+    ImageView.clickAtRelative(0.7, 0.7);
+    Sidebar.hasRegions(1);
+    Labels.selectedLabel.should('have.length', 0);
+  });
+
+  it('should draw rectangle by dragging (FF_DEV_1442 = true)', () => {
+    LabelStudio.addFeatureFlagsOnPageLoad({
+      [FF_DEV_1442]: true,
+    });
+
+    LabelStudio.params()
+      .config(rectangleToolAndLabelsConfig)
+      .data(simpleImageData)
+      .withResult(simpleRectangleResult)
+      .init();
+
+    LabelStudio.waitForObjectsReady();
+
+    Sidebar.hasRegions(1);
+    Sidebar.toggleRegionSelection(0);
+    Labels.select('Label 1');
+
+    ImageView.drawRectRelative(0.5, 0.5, 0.2, 0.2);
+    Sidebar.hasRegions(2);
+    Labels.selectedLabel.should('contain.text', 'Label 1');
+  });
+
+  it('should draw rectangle when clicking outside (FF_DEV_1442 = false)', () => {
+    LabelStudio.addFeatureFlagsOnPageLoad({
+      [FF_DEV_1442]: false,
+    });
+
+    LabelStudio.params()
+      .config(rectangleToolAndLabelsConfig)
+      .data(simpleImageData)
+      .withResult(simpleRectangleResult)
+      .init();
+
+    LabelStudio.waitForObjectsReady();
+
+    Sidebar.hasRegions(1);
+    Sidebar.toggleRegionSelection(0);
+    Labels.select('Label 1');
+
+    ImageView.clickAtRelative(0.5, 0.5);
+    ImageView.clickAtRelative(0.7, 0.7);
+    Sidebar.hasRegions(2);
+    Labels.selectedLabel.should('contain.text', 'Label 1');
+  });
+
+  it('should draw rectangle by dragging (FF_DEV_1442 = false)', () => {
+    LabelStudio.addFeatureFlagsOnPageLoad({
+      [FF_DEV_1442]: false,
+    });
+
+    LabelStudio.params()
+      .config(rectangleToolAndLabelsConfig)
+      .data(simpleImageData)
+      .withResult(simpleRectangleResult)
+      .init();
+
+    LabelStudio.waitForObjectsReady();
+
+    Sidebar.hasRegions(1);
+    Sidebar.toggleRegionSelection(0);
+    Labels.select('Label 1');
+
+    ImageView.drawRectRelative(0.5, 0.5, 0.2, 0.2);
+    Sidebar.hasRegions(2);
+    Labels.selectedLabel.should('contain.text', 'Label 1');
+  });
+});