diff --git a/.changeset/lemon-files-fold.md b/.changeset/lemon-files-fold.md
new file mode 100644
index 000000000..93a6b41e5
--- /dev/null
+++ b/.changeset/lemon-files-fold.md
@@ -0,0 +1,5 @@
+---
+"@vue-flow/core": patch
+---
+
+Don't set user selection flags on pointer down, only setting them when pointer is moved to allow pane click events to occur when `selectionKeyCode` is `true` (i.e. selection on drag).
diff --git a/packages/core/src/container/Pane/Pane.vue b/packages/core/src/container/Pane/Pane.vue
index 52d2db5ba..3a7090531 100644
--- a/packages/core/src/container/Pane/Pane.vue
+++ b/packages/core/src/container/Pane/Pane.vue
@@ -150,9 +150,6 @@ function onPointerDown(event: PointerEvent) {
     y,
   }
 
-  userSelectionActive.value = true
-  nodesSelectionActive.value = false
-
   emits.selectionStart(event)
 }
 
diff --git a/packages/core/src/container/Viewport/Viewport.vue b/packages/core/src/container/Viewport/Viewport.vue
index 1571777aa..ba0aff8b6 100644
--- a/packages/core/src/container/Viewport/Viewport.vue
+++ b/packages/core/src/container/Viewport/Viewport.vue
@@ -66,7 +66,7 @@ const selectionKeyPressed = useKeyPress(selectionKeyCode)
 
 const zoomKeyPressed = useKeyPress(zoomActivationKeyCode)
 
-const shouldPanOnDrag = toRef(() => panKeyPressed.value || panOnDrag.value)
+const shouldPanOnDrag = toRef(() => !selectionKeyPressed.value && (panKeyPressed.value || panOnDrag.value))
 
 const shouldPanOnScroll = toRef(() => panKeyPressed.value || panOnScroll.value)
 
diff --git a/packages/core/src/types/store.ts b/packages/core/src/types/store.ts
index c1b3844f7..d3a0354f1 100644
--- a/packages/core/src/types/store.ts
+++ b/packages/core/src/types/store.ts
@@ -279,10 +279,7 @@ export interface Actions extends Omit<ViewportHelper, 'viewportInitialized'> {
   removeSelectedEdges: (edges: GraphEdge[]) => void
   /** manually unselect nodes and remove from state */
   removeSelectedNodes: (nodes: GraphNode[]) => void
-  /**
-   * @deprecated will be replaced in the next major
-   * unselect selected elements (if none are passed, all elements are unselected)
-   */
+  /** unselect selected elements (if none are passed, all elements are unselected) */
   removeSelectedElements: (elements?: Elements) => void
   /** apply min zoom value to d3 */
   setMinZoom: (zoom: number) => void