Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

stabilize stage management for rustc tools, especially on cargo #135990

Draft
wants to merge 10 commits into
base: master
Choose a base branch
from
+452 −370
Draft

stabilize stage management for rustc tools, especially on cargo #135990

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
2558413
refactor `ToolBuild` to handle compiler tool stages
onur-ozkan Jan 24, 2025
234a2b5
make `LlvmBitcodeLinker` and `Rustdoc` to use `ToolBuild`
onur-ozkan Jan 24, 2025
4ddb1ac
let `ToolBuild` to handle compiler tool stages automatically
onur-ozkan Jan 24, 2025
01d7711
apply rustdoc lto in `ToolBuild`
onur-ozkan Feb 3, 2025
9896694
use target compiler for `LlvmBitcodeLinker`
onur-ozkan Feb 3, 2025
28d8ac0
keep `Rustc` private and use `Assemble` instead
onur-ozkan Feb 3, 2025
44cc009
add doc-comments on `compile::Rustc`
onur-ozkan Feb 3, 2025
79b6c4b
document bootstrap `tool` module
onur-ozkan Feb 3, 2025
b222ef8
improve staging api
onur-ozkan Feb 4, 2025
cc6f786
apply some nits and add coverage for `downgraded_from`
onur-ozkan Feb 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions src/bootstrap/src/core/build_steps/check.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ impl Step for Std {
builder.require_submodule("library/stdarch", None);

let target = self.target;
let compiler = builder.compiler(builder.top_stage, builder.config.build);
let compiler = builder.compiler(builder.top_stage, builder.config.build, false);

let mut cargo = builder::Cargo::new(
builder,
Expand Down Expand Up @@ -192,7 +192,7 @@ impl Step for Rustc {
/// the `compiler` targeting the `target` architecture. The artifacts
/// created will also be linked into the sysroot directory.
fn run(self, builder: &Builder<'_>) {
let compiler = builder.compiler(builder.top_stage, builder.config.build);
let compiler = builder.compiler(builder.top_stage, builder.config.build, false);
let target = self.target;

if compiler.stage != 0 {
Expand Down Expand Up @@ -274,7 +274,7 @@ impl Step for CodegenBackend {
return;
}

let compiler = builder.compiler(builder.top_stage, builder.config.build);
let compiler = builder.compiler(builder.top_stage, builder.config.build, false);
let target = self.target;
let backend = self.backend;

Expand Down Expand Up @@ -329,7 +329,7 @@ impl Step for RustAnalyzer {
}

fn run(self, builder: &Builder<'_>) {
let compiler = builder.compiler(builder.top_stage, builder.config.build);
let compiler = builder.compiler(builder.top_stage, builder.config.build, true);
let target = self.target;

builder.ensure(Rustc::new(target, builder));
Expand Down Expand Up @@ -411,7 +411,7 @@ fn run_tool_check_step(
path: &str,
) {
let display_name = path.rsplit('/').next().unwrap();
let compiler = builder.compiler(builder.top_stage, builder.config.build);
let compiler = builder.compiler(builder.top_stage, builder.config.build, true);

builder.ensure(Rustc::new(target, builder));

Expand Down
2 changes: 1 addition & 1 deletion src/bootstrap/src/core/build_steps/clean.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ macro_rules! clean_crate_tree {

fn make_run(run: RunConfig<'_>) {
let builder = run.builder;
let compiler = builder.compiler(builder.top_stage, run.target);
let compiler = builder.compiler(builder.top_stage, run.target, false);
builder.ensure(Self { crates: run.cargo_crates_in_set(), compiler });
}

Expand Down
64 changes: 32 additions & 32 deletions src/bootstrap/src/core/build_steps/clippy.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ impl Step for Std {
builder.require_submodule("library/stdarch", None);

let target = self.target;
let compiler = builder.compiler(builder.top_stage, builder.config.build);
let compiler = builder.compiler(builder.top_stage, builder.config.build, false);

let mut cargo = builder::Cargo::new(
builder,
Expand Down Expand Up @@ -204,7 +204,7 @@ impl Step for Rustc {
/// This will lint the compiler for a particular stage of the build using
/// the `compiler` targeting the `target` architecture.
fn run(self, builder: &Builder<'_>) {
let compiler = builder.compiler(builder.top_stage, builder.config.build);
let compiler = builder.compiler(builder.top_stage, builder.config.build, false);
let target = self.target;

if compiler.stage != 0 {
Expand Down Expand Up @@ -254,7 +254,7 @@ impl Step for Rustc {

macro_rules! lint_any {
($(
$name:ident, $path:expr, $readable_name:expr
$name:ident, $path:expr, $readable_name:expr, $mode:expr
$(,lint_by_default = $lint_by_default:expr)*
;
)+) => {
Expand Down Expand Up @@ -283,15 +283,15 @@ macro_rules! lint_any {
}

fn run(self, builder: &Builder<'_>) -> Self::Output {
let compiler = builder.compiler(builder.top_stage, builder.config.build);
let compiler = builder.compiler(builder.top_stage, builder.config.build, builder.top_stage > 0 && $mode == Mode::ToolRustc);
let target = self.target;

builder.ensure(check::Rustc::new(target, builder).build_kind(Some(Kind::Check)));

let cargo = prepare_tool_cargo(
builder,
compiler,
Mode::ToolRustc,
$mode,
target,
Kind::Clippy,
$path,
Expand All @@ -301,15 +301,15 @@ macro_rules! lint_any {

let _guard = builder.msg_tool(
Kind::Clippy,
Mode::ToolRustc,
$mode,
$readable_name,
compiler.stage,
&compiler.host,
&target,
);

let stringified_name = stringify!($name).to_lowercase();
let stamp = BuildStamp::new(&builder.cargo_out(compiler, Mode::ToolRustc, target))
let stamp = BuildStamp::new(&builder.cargo_out(compiler, $mode, target))
.with_prefix(&format!("{}-check", stringified_name));

run_cargo(
Expand All @@ -328,31 +328,31 @@ macro_rules! lint_any {
}

lint_any!(
Bootstrap, "src/bootstrap", "bootstrap";
BuildHelper, "src/build_helper", "build_helper";
BuildManifest, "src/tools/build-manifest", "build-manifest";
CargoMiri, "src/tools/miri/cargo-miri", "cargo-miri";
Clippy, "src/tools/clippy", "clippy";
CollectLicenseMetadata, "src/tools/collect-license-metadata", "collect-license-metadata";
CodegenGcc, "compiler/rustc_codegen_gcc", "rustc-codegen-gcc";
Compiletest, "src/tools/compiletest", "compiletest";
CoverageDump, "src/tools/coverage-dump", "coverage-dump";
Jsondocck, "src/tools/jsondocck", "jsondocck";
Jsondoclint, "src/tools/jsondoclint", "jsondoclint";
LintDocs, "src/tools/lint-docs", "lint-docs";
LlvmBitcodeLinker, "src/tools/llvm-bitcode-linker", "llvm-bitcode-linker";
Miri, "src/tools/miri", "miri";
MiroptTestTools, "src/tools/miropt-test-tools", "miropt-test-tools";
OptDist, "src/tools/opt-dist", "opt-dist";
RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
RemoteTestServer, "src/tools/remote-test-server", "remote-test-server";
Rls, "src/tools/rls", "rls";
RustAnalyzer, "src/tools/rust-analyzer", "rust-analyzer";
Rustdoc, "src/librustdoc", "clippy";
Rustfmt, "src/tools/rustfmt", "rustfmt";
RustInstaller, "src/tools/rust-installer", "rust-installer";
Tidy, "src/tools/tidy", "tidy";
TestFloatParse, "src/etc/test-float-parse", "test-float-parse";
Bootstrap, "src/bootstrap", "bootstrap", Mode::ToolBootstrap;
BuildHelper, "src/build_helper", "build_helper", Mode::ToolBootstrap;
BuildManifest, "src/tools/build-manifest", "build-manifest", Mode::ToolBootstrap;
CargoMiri, "src/tools/miri/cargo-miri", "cargo-miri", Mode::ToolRustc;
Clippy, "src/tools/clippy", "clippy", Mode::ToolRustc;
CollectLicenseMetadata, "src/tools/collect-license-metadata", "collect-license-metadata", Mode::ToolBootstrap;
CodegenGcc, "compiler/rustc_codegen_gcc", "rustc-codegen-gcc", Mode::ToolRustc;
Compiletest, "src/tools/compiletest", "compiletest", Mode::ToolBootstrap;
CoverageDump, "src/tools/coverage-dump", "coverage-dump", Mode::ToolBootstrap;
Jsondocck, "src/tools/jsondocck", "jsondocck", Mode::ToolBootstrap;
Jsondoclint, "src/tools/jsondoclint", "jsondoclint", Mode::ToolBootstrap;
LintDocs, "src/tools/lint-docs", "lint-docs", Mode::ToolBootstrap;
LlvmBitcodeLinker, "src/tools/llvm-bitcode-linker", "llvm-bitcode-linker", Mode::ToolRustc;
Miri, "src/tools/miri", "miri", Mode::ToolRustc;
MiroptTestTools, "src/tools/miropt-test-tools", "miropt-test-tools", Mode::ToolRustc;
OptDist, "src/tools/opt-dist", "opt-dist", Mode::ToolBootstrap;
RemoteTestClient, "src/tools/remote-test-client", "remote-test-client", Mode::ToolBootstrap;
RemoteTestServer, "src/tools/remote-test-server", "remote-test-server", Mode::ToolBootstrap;
Rls, "src/tools/rls", "rls", Mode::ToolBootstrap;
RustAnalyzer, "src/tools/rust-analyzer", "rust-analyzer", Mode::ToolRustc;
Rustdoc, "src/librustdoc", "clippy", Mode::ToolRustc;
Rustfmt, "src/tools/rustfmt", "rustfmt", Mode::ToolRustc;
RustInstaller, "src/tools/rust-installer", "rust-installer", Mode::ToolBootstrap;
Tidy, "src/tools/tidy", "tidy", Mode::ToolBootstrap;
TestFloatParse, "src/etc/test-float-parse", "test-float-parse", Mode::ToolStd;
);

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
Expand Down
58 changes: 39 additions & 19 deletions src/bootstrap/src/core/build_steps/compile.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ impl Step for Std {
&& builder.config.last_modified_commit(&["library"], "download-rustc", true).is_none();

run.builder.ensure(Std {
compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
compiler: run.builder.compiler(run.builder.top_stage, run.build_triple(), false),
target: run.target,
crates,
force_recompile,
Expand Down Expand Up @@ -162,7 +162,7 @@ impl Step for Std {

let mut target_deps = builder.ensure(StartupObjects { compiler, target });

let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target, false);
if compiler_to_use != compiler {
builder.ensure(Std::new(compiler_to_use, target));
let msg = if compiler_to_use.host == target {
Expand Down Expand Up @@ -264,7 +264,7 @@ impl Step for Std {

builder.ensure(StdLink::from_std(
self,
builder.compiler(compiler.stage, builder.config.build),
builder.compiler(compiler.stage, builder.config.build, false),
));
}
}
Expand Down Expand Up @@ -770,7 +770,7 @@ impl Step for StartupObjects {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(StartupObjects {
compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
compiler: run.builder.compiler(run.builder.top_stage, run.build_triple(), false),
target: run.target,
});
}
Expand Down Expand Up @@ -837,11 +837,18 @@ fn cp_rustc_component_to_ci_sysroot(builder: &Builder<'_>, sysroot: &Path, conte
}
}

/// Low-level implementation of the compiler's compilation process.
///
/// DO NOT `pub` the fields of this type and AVOID using it anywhere
/// unless it's in the `Assemble` step.
///
/// If you want to build a compiler for a specific stage and target, use
/// `Builder::compiler` instead.
#[derive(Debug, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)]
pub struct Rustc {
pub target: TargetSelection,
target: TargetSelection,
/// The **previous** compiler used to compile this compiler.
pub compiler: Compiler,
compiler: Compiler,
/// Whether to build a subset of crates, rather than the whole compiler.
///
/// This should only be requested by the user, not used within bootstrap itself.
Expand All @@ -851,6 +858,9 @@ pub struct Rustc {
}

impl Rustc {
/// Serves as a helpful util for unit tests and should NEVER exist
/// for non-test environment.
#[cfg(test)]
pub fn new(compiler: Compiler, target: TargetSelection) -> Self {
Self { target, compiler, crates: Default::default() }
}
Expand Down Expand Up @@ -884,7 +894,7 @@ impl Step for Rustc {
fn make_run(run: RunConfig<'_>) {
let crates = run.cargo_crates_in_set();
run.builder.ensure(Rustc {
compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
compiler: run.builder.compiler(run.builder.top_stage, run.build_triple(), false),
target: run.target,
crates,
});
Expand Down Expand Up @@ -921,9 +931,10 @@ impl Step for Rustc {
return compiler.stage;
}

let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target, false);
if compiler_to_use != compiler {
builder.ensure(Rustc::new(compiler_to_use, target));
let _ = builder.compiler(compiler_to_use.stage, target, false);

let msg = if compiler_to_use.host == target {
format!(
"Uplifting rustc (stage{} -> stage{})",
Expand All @@ -946,7 +957,7 @@ impl Step for Rustc {

// Ensure that build scripts and proc macros have a std / libproc_macro to link against.
builder.ensure(Std::new(
builder.compiler(self.compiler.stage, builder.config.build),
builder.compiler(self.compiler.stage, builder.config.build, false),
builder.config.build,
));

Expand Down Expand Up @@ -1012,7 +1023,7 @@ impl Step for Rustc {

builder.ensure(RustcLink::from_rustc(
self,
builder.compiler(compiler.stage, builder.config.build),
builder.compiler(compiler.stage, builder.config.build, false),
));

compiler.stage
Expand Down Expand Up @@ -1413,7 +1424,7 @@ impl Step for CodegenBackend {

run.builder.ensure(CodegenBackend {
target: run.target,
compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
compiler: run.builder.compiler(run.builder.top_stage, run.build_triple(), false),
backend: backend.clone(),
});
}
Expand All @@ -1424,7 +1435,7 @@ impl Step for CodegenBackend {
let target = self.target;
let backend = self.backend;

builder.ensure(Rustc::new(compiler, target));
let _ = builder.compiler(compiler.stage, target, false);

if builder.config.keep_stage.contains(&compiler.stage) {
builder.info(
Expand All @@ -1436,7 +1447,7 @@ impl Step for CodegenBackend {
return;
}

let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target);
let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target, false);
if compiler_to_use != compiler {
builder.ensure(CodegenBackend { compiler: compiler_to_use, target, backend });
return;
Expand Down Expand Up @@ -1734,7 +1745,7 @@ impl Step for Assemble {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Assemble {
target_compiler: run.builder.compiler(run.builder.top_stage + 1, run.target),
target_compiler: run.builder.compiler(run.builder.top_stage + 1, run.target, false),
});
}

Expand Down Expand Up @@ -1792,7 +1803,7 @@ impl Step for Assemble {
}
}

let maybe_install_llvm_bitcode_linker = |compiler| {
let maybe_install_llvm_bitcode_linker = |compiler: Compiler| {
if builder.config.llvm_bitcode_linker_enabled {
let src_path = builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker {
compiler,
Expand Down Expand Up @@ -1835,7 +1846,8 @@ impl Step for Assemble {
//
// FIXME: It may be faster if we build just a stage 1 compiler and then
// use that to bootstrap this compiler forward.
let mut build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build);
let mut build_compiler =
builder.compiler(target_compiler.stage - 1, builder.config.build, false);

// Build enzyme
let enzyme_install = if builder.config.llvm_enzyme {
Expand All @@ -1861,7 +1873,11 @@ impl Step for Assemble {
// link to these. (FIXME: Is that correct? It seems to be correct most
// of the time but I think we do link to these for stage2/bin compilers
// when not performing a full bootstrap).
let actual_stage = builder.ensure(Rustc::new(build_compiler, target_compiler.host));
let actual_stage = builder.ensure(Rustc {
compiler: build_compiler,
target: target_compiler.host,
crates: vec![],
});
// Current build_compiler.stage might be uplifted instead of being built; so update it
// to not fail while linking the artifacts.
build_compiler.stage = actual_stage;
Expand Down Expand Up @@ -1974,7 +1990,11 @@ impl Step for Assemble {
);
}

maybe_install_llvm_bitcode_linker(build_compiler);
{
let mut compiler_for_rustc_tool = build_compiler;
compiler_for_rustc_tool.downgraded_from = Some(target_compiler.stage);
maybe_install_llvm_bitcode_linker(compiler_for_rustc_tool);
}

// Ensure that `libLLVM.so` ends up in the newly build compiler directory,
// so that it can be found when the newly built `rustc` is run.
Expand Down
Loading
Loading