Skip to content

Commit

Permalink
feat: Migrate to bun.lock (#5112)
Browse files Browse the repository at this point in the history 
* bun: Migrate to bun.lock

(In backwards compatible way)

Read more: https://bun.sh/blog/bun-lock-text-lockfile

* Clean up

* More clean up

* Mount bun.lock to jailed process
  • Loading branch information
@pyranota
pyranota authored Jan 22, 2025
1 parent 669a1ff commit fcfad69
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 41 deletions.
6 changes: 6 additions & 0 deletions backend/windmill-worker/nsjail/run.bun.config.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ mount {
mandatory: false
}

mount {
src: "{JOB_DIR}/bun.lock"
dst: "/tmp/{LANG}/bun.lock"
is_bind: true
mandatory: false
}

mount {
src: "{JOB_DIR}/wrapper.mjs"
Expand Down
103 changes: 62 additions & 41 deletions backend/windmill-worker/src/bun_executor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,24 +62,37 @@ const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js");

const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.config.proto");

pub const BUN_LOCK_SPLIT: &str = "\n//bun.lock\n";
pub const BUN_LOCKB_SPLIT: &str = "\n//bun.lockb\n";
pub const BUN_LOCK_SPLIT_WINDOWS: &str = "\r\n//bun.lock\r\n";
pub const BUN_LOCKB_SPLIT_WINDOWS: &str = "\r\n//bun.lockb\r\n";

pub const EMPTY_FILE: &str = "<empty>";

fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool) {
if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT) {
/// Returns (package.json, bun.lock(b), is_empty, is_binary)
fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool, bool) {
if let Some(index) = lockfile.find(BUN_LOCK_SPLIT) {
// Split using "\n//bun.lock\n"
let (before, after_with_sep) = lockfile.split_at(index);
let after = &after_with_sep[BUN_LOCK_SPLIT.len()..];
(before, Some(after), after == EMPTY_FILE, false)
} else if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT) {
// Split using "\n//bun.lockb\n"
let (before, after_with_sep) = lockfile.split_at(index);
let after = &after_with_sep[BUN_LOCKB_SPLIT.len()..];
(before, Some(after), after == EMPTY_FILE)
(before, Some(after), after == EMPTY_FILE, true)
} else if let Some(index) = lockfile.find(BUN_LOCK_SPLIT_WINDOWS) {
// Split using "\r\n//bun.lock\r\n"
let (before, after_with_sep) = lockfile.split_at(index);
let after = &after_with_sep[BUN_LOCK_SPLIT_WINDOWS.len()..];
(before, Some(after), after == EMPTY_FILE, false)
} else if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT_WINDOWS) {
// Split using "\r\n//bun.lockb\r\n"
let (before, after_with_sep) = lockfile.split_at(index);
let after = &after_with_sep[BUN_LOCKB_SPLIT_WINDOWS.len()..];
(before, Some(after), after == EMPTY_FILE)
(before, Some(after), after == EMPTY_FILE, true)
} else {
(lockfile, None, false)
(lockfile, None, false, false)
}
}

Expand Down Expand Up @@ -199,18 +212,18 @@ pub async fn gen_bun_lockfile(
}
if !npm_mode {
#[cfg(any(target_os = "linux", target_os = "macos"))]
content.push_str(BUN_LOCKB_SPLIT);
content.push_str(BUN_LOCK_SPLIT);

#[cfg(target_os = "windows")]
content.push_str(BUN_LOCKB_SPLIT_WINDOWS);
content.push_str(BUN_LOCK_SPLIT_WINDOWS);

{
let file = format!("{job_dir}/bun.lockb");
let file = format!("{job_dir}/bun.lock");
if !empty_deps && tokio::fs::metadata(&file).await.is_ok() {
let mut file = File::open(&file).await?;
let mut buf = vec![];
file.read_to_end(&mut buf).await?;
content.push_str(&base64::engine::general_purpose::STANDARD.encode(&buf));
let mut buf = String::default();
file.read_to_string(&mut buf).await?;
content.push_str(&buf);
} else {
content.push_str(&EMPTY_FILE);
}
Expand Down Expand Up @@ -277,7 +290,7 @@ pub async fn install_bun_lockfile(
.env_clear()
.envs(PROXY_ENVS.clone())
.envs(common_bun_proc_envs)
.args(vec!["install"])
.args(vec!["install", "--save-text-lockfile"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());

Expand Down Expand Up @@ -785,26 +798,30 @@ async fn compute_bundle_local_and_remote_path(
}

pub async fn prepare_job_dir(reqs: &str, job_dir: &str) -> Result<()> {
let (pkg, lock, empty) = split_lockfile(reqs);
let (pkg, lock, empty, is_binary) = split_lockfile(reqs);
let _ = write_file(job_dir, "package.json", pkg)?;

if !empty {
if let Some(lock) = lock {
let _ = write_lockb(lock, job_dir).await?;
let _ = write_lock(lock, job_dir, is_binary).await?;
}
}

Ok(())
}
async fn write_lockb(splitted_lockb_2: &str, job_dir: &str) -> Result<()> {
write_file_binary(
job_dir,
"bun.lockb",
&base64::engine::general_purpose::STANDARD
.decode(splitted_lockb_2)
.map_err(|_| error::Error::InternalErr("Could not decode bun.lockb".to_string()))?,
)
.await?;
async fn write_lock(splitted_lockb_2: &str, job_dir: &str, is_binary: bool) -> Result<()> {
if is_binary {
write_file_binary(
job_dir,
"bun.lockb",
&base64::engine::general_purpose::STANDARD
.decode(splitted_lockb_2)
.map_err(|_| error::Error::InternalErr(format!("Could not decode bun.lockb")))?,
)
.await?;
} else {
write_file(job_dir, "bun.lock", splitted_lockb_2)?;
};
Ok(())
}

Expand Down Expand Up @@ -900,26 +917,26 @@ pub async fn handle_bun_job(
} else if let Some(codebase) = codebase.as_ref() {
pull_codebase(&job.workspace_id, codebase, job_dir).await?;
} else if let Some(reqs) = requirements_o.as_ref() {
let (pkg, lock, empty) = split_lockfile(reqs);
let (pkg, lock, empty, is_binary) = split_lockfile(reqs);

if lock.is_none() && !annotation.npm {
return Err(error::Error::ExecutionErr(
format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|")
format!("Invalid requirements, expected to find //bun.lock{} split pattern in reqs. Found: |{reqs}|", if is_binary {"b"} else {""})
));
}

let _ = write_file(job_dir, "package.json", pkg)?;
let lockb = if annotation.npm { "" } else { lock.unwrap() };
let lock = if annotation.npm { "" } else { lock.unwrap() };
if !empty {
let mut skip_install = false;
let mut create_buntar = false;
let mut buntar_path = "".to_string();

if !annotation.npm {
let _ = write_lockb(lockb, job_dir).await?;
let _ = write_lock(lock, job_dir, is_binary).await?;

let mut sha_path = sha2::Sha256::new();
sha_path.update(lockb.as_bytes());
sha_path.update(lock.as_bytes());

let buntar_name =
base64::engine::general_purpose::URL_SAFE.encode(sha_path.finalize());
Expand Down Expand Up @@ -962,7 +979,7 @@ pub async fn handle_bun_job(
Some(&vec![
"main.ts".to_string(),
"package.json".to_string(),
"bun.lockb".to_string(),
if is_binary { "bun.lockb" } else { "bun.lock" }.to_string(),
"shared".to_string(),
"bunfig.toml".to_string(),
]),
Expand Down Expand Up @@ -1592,25 +1609,29 @@ pub async fn start_worker(
if let Some(codebase) = codebase.as_ref() {
pull_codebase(w_id, codebase, job_dir).await?;
} else if let Some(reqs) = requirements_o {
let (pkg, lock, empty) = split_lockfile(&reqs);
let (pkg, lock, empty, is_binary) = split_lockfile(&reqs);
if lock.is_none() {
return Err(error::Error::ExecutionErr(
format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|")
));
}
let _ = write_file(job_dir, "package.json", pkg)?;
let lockb = lock.unwrap();
let lock = lock.unwrap();
if !empty {
let _ = write_file_binary(
job_dir,
"bun.lockb",
&base64::engine::general_purpose::STANDARD
.decode(lockb)
.map_err(|_| {
error::Error::InternalErr("Could not decode bun.lockb".to_string())
})?,
)
.await?;
if is_binary {
let _ = write_file_binary(
job_dir,
"bun.lockb",
&base64::engine::general_purpose::STANDARD
.decode(lock)
.map_err(|_| {
error::Error::InternalErr("Could not decode bun.lockb".to_string())
})?,
)
.await?;
} else {
write_file(job_dir, "bun.lock", lock)?;
}

install_bun_lockfile(
&mut mem_peak,
Expand Down

0 comments on commit fcfad69

Please sign in to comment.