diff --git a/src/scikit_build_core/build/_file_processor.py b/src/scikit_build_core/build/_file_processor.py
index ed48fa7f..7b669fdc 100644
--- a/src/scikit_build_core/build/_file_processor.py
+++ b/src/scikit_build_core/build/_file_processor.py
@@ -32,6 +32,7 @@ def each_unignored_file(
     starting_path: Path,
     include: Sequence[str] = (),
     exclude: Sequence[str] = (),
+    build_dir: str = "",
 ) -> Generator[Path, None, None]:
     """
     Runs through all non-ignored files. Must be run from the root directory.
@@ -50,9 +51,19 @@ def each_unignored_file(
         if p != Path(".gitignore")
     }
 
+    exclude_build_dir = build_dir.format(
+        cache_tag="*",
+        wheel_tag="*",
+        build_type="*",
+        state="*",
+    )
+    exclude_lines = (
+        f"{EXCLUDE_LINES}\n{exclude_build_dir}" if exclude_build_dir else EXCLUDE_LINES
+    )
+
     user_exclude_spec = pathspec.GitIgnoreSpec.from_lines(list(exclude))
     global_exclude_spec = pathspec.GitIgnoreSpec.from_lines(global_exclude_lines)
-    builtin_exclude_spec = pathspec.GitIgnoreSpec.from_lines(EXCLUDE_LINES)
+    builtin_exclude_spec = pathspec.GitIgnoreSpec.from_lines(exclude_lines)
 
     include_spec = pathspec.GitIgnoreSpec.from_lines(include)
 
diff --git a/src/scikit_build_core/build/_pathutil.py b/src/scikit_build_core/build/_pathutil.py
index cc1ff5e4..9242290a 100644
--- a/src/scikit_build_core/build/_pathutil.py
+++ b/src/scikit_build_core/build/_pathutil.py
@@ -41,6 +41,7 @@ def packages_to_file_mapping(
     include: Sequence[str],
     src_exclude: Sequence[str],
     target_exclude: Sequence[str],
+    build_dir: str,
 ) -> dict[str, str]:
     """
     This will output a mapping of source files to target files.
@@ -55,6 +56,7 @@ def packages_to_file_mapping(
             source_dir,
             include=include,
             exclude=src_exclude,
+            build_dir=build_dir,
         ):
             rel_path = filepath.relative_to(source_dir)
             target_path = platlib_dir / package_dir / rel_path
diff --git a/src/scikit_build_core/build/sdist.py b/src/scikit_build_core/build/sdist.py
index 554e807e..9a06274e 100644
--- a/src/scikit_build_core/build/sdist.py
+++ b/src/scikit_build_core/build/sdist.py
@@ -154,6 +154,7 @@ def build_sdist(
                 Path(),
                 include=settings.sdist.include,
                 exclude=settings.sdist.exclude,
+                build_dir=settings.build_dir,
             )
         )
         for filepath in paths:
diff --git a/src/scikit_build_core/build/wheel.py b/src/scikit_build_core/build/wheel.py
index a389df04..aa3f44cb 100644
--- a/src/scikit_build_core/build/wheel.py
+++ b/src/scikit_build_core/build/wheel.py
@@ -448,6 +448,7 @@ def _build_wheel_impl_impl(
             include=settings.sdist.include,
             src_exclude=settings.sdist.exclude,
             target_exclude=settings.wheel.exclude,
+            build_dir=settings.build_dir,
         )
 
         if not editable:
diff --git a/tests/test_editable_unit.py b/tests/test_editable_unit.py
index 48584368..ba15f9d0 100644
--- a/tests/test_editable_unit.py
+++ b/tests/test_editable_unit.py
@@ -141,6 +141,7 @@ def test_navigate_editable_pkg(editable_package: EditablePackage, virtualenv: VE
         include=[],
         src_exclude=[],
         target_exclude=[],
+        build_dir="",
     )
     assert mapping == {
         str(Path("pkg/__init__.py")): str(pkg_dir / "__init__.py"),