Fix build runner (#1029)

* Fix build runner * Add zls module, bump version * Fix path from root issue in modules (thanks daremakemyday from Twitch :)) * Libraryify * remove build_runner backward compatibility * Remove some `= undefined`s in Server.zig Makes library use less dangerous * Consistent mem model + custom build runner possible * Add build runner utils for third-party tooling * Make logs removable in libraries with zls_ prefix * Fix build runner CI * Expose references * Use new addModule
zigtools · Mar 5, 2023 · 121f68b · 121f68b
1 parent a78950c
commit 121f68b
Show file tree

Hide file tree

Showing 19 changed files with 204 additions and 272 deletions.
diff --git a/.github/workflows/build_runner.yml b/.github/workflows/build_runner.yml
@@ -17,7 +17,7 @@ jobs:
   check_build_runner:
     strategy:
       matrix:
-        zig_version: [0.9.1, 0.10.1, master]
+        zig_version: [master]
 
     runs-on: ubuntu-latest
 
@@ -33,16 +33,7 @@ jobs:
         with:
           version: ${{ matrix.zig_version }}
 
-      - name: Create temp zig project
-        run: |
-          mkdir $RUNNER_TEMP/TEMP_ZIG_PROJECT
-          cd $RUNNER_TEMP/TEMP_ZIG_PROJECT
-          zig init-exe
-
       - name: Check build_runner builds on master
-        if: ${{ matrix.zig_version == 'master' }}
-        run: zig build-exe $GITHUB_WORKSPACE/src/special/build_runner.zig --mod @build@::$RUNNER_TEMP/TEMP_ZIG_PROJECT/build.zig --deps @build@
-
-      - name: Check build_runner builds on older tagged releases
-        if: ${{ matrix.zig_version != 'master' }}
-        run: zig build-exe $GITHUB_WORKSPACE/src/special/build_runner.zig --pkg-begin @build@ $RUNNER_TEMP/TEMP_ZIG_PROJECT/build.zig --pkg-end
+        run: |
+          pwd
+          zig build --build-runner src/special/build_runner.zig
diff --git a/README.md b/README.md
@@ -17,6 +17,7 @@ The Zig Language Server (zls) is a tool that implements Microsoft's Language Ser
   - [Per-build Configuration Options](#per-build-configuration-options)
     - [`BuildOption`](#buildoption)
 - [Features](#features)
+- [Using as a library](#using-as-a-library)
 - [Related Projects](#related-projects)
 - [Quick Thanks :)](#quick-thanks-)
 - [License](#license)
@@ -129,6 +130,10 @@ The following LSP features are supported:
 - Selection ranges
 - Folding regions
 
+## Using as a library
+
+You can use zls as a library! [Check out this demo repo](https://github.com/zigtools/zls-as-lib-demo) for a good reference.
+
 ## Related Projects
 
 - [`sublime-zig-language` by @prime31](https://github.com/prime31/sublime-zig-language)
@@ -139,7 +144,7 @@ The following LSP features are supported:
 - [`known-folders` by @ziglibs](https://github.com/ziglibs/known-folders)
   - Provides API to access known folders on Linux, Windows and Mac OS
 - [`zls` by @zigtools](https://github.com/zigtools/zls)
-  - Used by many zls developers to more efficently work on zls
+  - Used by many zls developers to more efficiently work on zls
 
 ## Quick Thanks :)
 

diff --git a/build.zig b/build.zig
@@ -7,7 +7,7 @@ const zls_version = std.builtin.Version{ .major = 0, .minor = 11, .patch = 0 };
 pub fn build(b: *std.build.Builder) !void {
     comptime {
         const current_zig = builtin.zig_version;
-        const min_zig = std.SemanticVersion.parse("0.11.0-dev.1817+f6c934677") catch return; // package manager hashes made consistent on windows
+        const min_zig = std.SemanticVersion.parse("0.11.0-dev.1836+28364166e") catch return; // package manager stuff + --build-runner + zls as a library
         if (current_zig.order(min_zig) == .lt) {
             @compileError(std.fmt.comptimePrint("Your Zig version v{} does not meet the minimum build requirement of v{}", .{ current_zig, min_zig }));
         }
@@ -160,7 +160,7 @@ pub fn build(b: *std.build.Builder) !void {
 
     const build_options_module = exe_options.createModule();
 
-    const zls_module = b.createModule(.{
+    const zls_module = b.addModule("zls", .{
         .source_file = .{ .path = "src/zls.zig" },
         .dependencies = &.{
             .{ .name = "known-folders", .module = known_folders_module },

diff --git a/build.zig.zon b/build.zig.zon
@@ -4,16 +4,16 @@
 
     .dependencies = .{
         .known_folders = .{
-            .url = "https://github.com/ziglibs/known-folders/archive/53fe3b676f32e59d46f4fd201d7ab200e5f6cb98.tar.gz",
-            .hash = "12203e18503cd0fa097a4404b0c4f8535a68536886b536ae51c786455238ba5f183b",
+            .url = "https://github.com/ziglibs/known-folders/archive/d13ba6137084e55f873f6afb67447fe8906cc951.tar.gz",
+            .hash = "122028c00915d9b37296059be8a3883c718dbb5bd174350caedf152fed1f46f99607",
         },
         .tres = .{
-            .url = "https://github.com/ziglibs/tres/archive/d8b0c24a945da02fffdae731edd1903c6889e73c.tar.gz",
-            .hash = "12209914477ef8c4ef99accb293c4a7ec90acdd9e77d3f60f5e056449cbfad3a7fd8",
+            .url = "https://github.com/ziglibs/tres/archive/707a09313b42e05d6ae22d1590499eece5f968ce.tar.gz",
+            .hash = "1220beaae8d152baa941a10b7ef3d3a59d093b257047035e2373c3c2f876ad29ccc8",
         },
         .diffz = .{
-            .url = "https://github.com/ziglibs/diffz/archive/efc91679b000a2d7f86fb40930f0a95a0d349bff.tar.gz",
-            .hash = "122019f94ec81a7cf6e9810983603dbacfc65ed30aea8f277f05ba0ce7c1511fff3d",
+            .url = "https://github.com/ziglibs/diffz/archive/b966296b4489eb082b0831ec9a37d6f5e1906040.tar.gz",
+            .hash = "1220ed4aed884221108ad39f2658b69a91653e0bbc8ce429bc7f1bc4e58f6a751553",
         },
     },
 }
diff --git a/src/ComptimeInterpreter.zig b/src/ComptimeInterpreter.zig
@@ -17,7 +17,7 @@ pub const Index = InternPool.Index;
 pub const Key = InternPool.Key;
 pub const ComptimeInterpreter = @This();
 
-const log = std.log.scoped(.comptime_interpreter);
+const log = std.log.scoped(.zls_comptime_interpreter);
 
 allocator: std.mem.Allocator,
 ip: InternPool,

diff --git a/src/DocumentStore.zig b/src/DocumentStore.zig
@@ -4,7 +4,7 @@ const types = @import("lsp.zig");
 const URI = @import("uri.zig");
 const analysis = @import("analysis.zig");
 const offsets = @import("offsets.zig");
-const log = std.log.scoped(.store);
+const log = std.log.scoped(.zls_store);
 const Ast = std.zig.Ast;
 const BuildAssociatedConfig = @import("BuildAssociatedConfig.zig");
 const BuildConfig = @import("special/build_runner.zig").BuildConfig;
@@ -115,13 +115,13 @@ pub fn deinit(self: *DocumentStore) void {
     self.cimports.deinit(self.allocator);
 }
 
-/// returns a handle to the given document
+/// Returns a handle to the given document
 pub fn getHandle(self: *DocumentStore, uri: Uri) ?*const Handle {
     return self.handles.get(uri);
 }
 
-/// returns a handle to the given document
-/// will load the document from disk if it hasn't been already
+/// Returns a handle to the given document
+/// Will load the document from disk if it hasn't been already
 pub fn getOrLoadHandle(self: *DocumentStore, uri: Uri) ?*const Handle {
     return self.getOrLoadHandleInternal(uri) catch null;
 }
@@ -141,7 +141,9 @@ fn getOrLoadHandleInternal(self: *DocumentStore, uri: Uri) !?*const Handle {
     return gop.value_ptr.*;
 }
 
-pub fn openDocument(self: *DocumentStore, uri: Uri, text: []const u8) error{OutOfMemory}!Handle {
+/// Takes ownership of `new_text` which has to be allocated
+/// with this DocumentStore's allocator
+pub fn openDocument(self: *DocumentStore, uri: Uri, text: [:0]const u8) error{OutOfMemory}!Handle {
     const tracy_zone = tracy.trace(@src());
     defer tracy_zone.end();
 
@@ -157,9 +159,7 @@ pub fn openDocument(self: *DocumentStore, uri: Uri, text: []const u8) error{OutO
     var handle = try self.allocator.create(Handle);
     errdefer self.allocator.destroy(handle);
 
-    const duped_text = try self.allocator.dupeZ(u8, text);
-
-    handle.* = try self.createDocument(uri, duped_text, true);
+    handle.* = try self.createDocument(uri, text, true);
     errdefer handle.deinit(self.allocator);
 
     try self.handles.putNoClobber(self.allocator, handle.uri, handle);
@@ -189,7 +189,8 @@ pub fn closeDocument(self: *DocumentStore, uri: Uri) void {
     self.garbageCollectionBuildFiles() catch {};
 }
 
-/// takes ownership of `new_text` which has to be allocated with `self.allocator`
+/// Takes ownership of `new_text` which has to be allocated
+/// with this DocumentStore's allocator
 pub fn refreshDocument(self: *DocumentStore, uri: Uri, new_text: [:0]const u8) !void {
     const tracy_zone = tracy.trace(@src());
     defer tracy_zone.end();
@@ -430,13 +431,45 @@ fn loadBuildAssociatedConfiguration(allocator: std.mem.Allocator, build_file: Bu
     return try std.json.parse(BuildAssociatedConfig, &token_stream, .{ .allocator = allocator });
 }
 
-/// runs the build.zig and extracts include directories and packages
-/// has to be freed with `std.json.parseFree`
-fn loadBuildConfiguration(
+/// Caller owns returned memory!
+pub fn populateBuildConfigurationArgs(
+    allocator: std.mem.Allocator,
+    args: *std.ArrayListUnmanaged([]const u8),
+    zig_exe_path: []const u8,
+    build_runner_path: []const u8,
+) error{OutOfMemory}!void {
+    try args.appendSlice(allocator, &.{ zig_exe_path, "build", "--build-runner", build_runner_path });
+}
+
+/// Runs the build.zig and returns the run result
+/// Args should be the output of `createBuildConfigurationArgs`
+/// plus any additional custom arguments
+/// Arena recommended
+pub fn executeBuildRunner(
+    allocator: std.mem.Allocator,
+    build_file_path: []const u8,
+    args: []const []const u8,
+) !std.ChildProcess.ExecResult {
+    const tracy_zone = tracy.trace(@src());
+    defer tracy_zone.end();
+
+    const build_file_directory_path = try std.fs.path.resolve(allocator, &.{ build_file_path, "../" });
+    defer allocator.free(build_file_directory_path);
+
+    return try std.ChildProcess.exec(.{
+        .allocator = allocator,
+        .argv = args,
+        .cwd = build_file_directory_path,
+    });
+}
+
+/// Runs the build.zig and extracts include directories and packages
+/// Has to be freed with `std.json.parseFree`
+pub fn loadBuildConfiguration(
     allocator: std.mem.Allocator,
     build_file: BuildFile,
     config: Config,
-    runtime_zig_version: ZigVersionWrapper,
+    _: ZigVersionWrapper,
 ) !BuildConfig {
     const tracy_zone = tracy.trace(@src());
     defer tracy_zone.end();
@@ -446,54 +479,14 @@ fn loadBuildConfiguration(
     const arena_allocator = arena.allocator();
 
     const build_file_path = try URI.parse(arena_allocator, build_file.uri);
-    const directory_path = try std.fs.path.resolve(arena_allocator, &.{ build_file_path, "../" });
-
-    // TODO extract this option from `BuildAssociatedConfig.BuildOption`
-    const zig_cache_root: []const u8 = try std.fs.path.join(arena_allocator, &.{ directory_path, "zig-cache" });
-
-    // introduction of modified module cli arguments https://github.com/ziglang/zig/pull/14664
-    const module_version = comptime std.SemanticVersion.parse("0.11.0-dev.1718+2737dce84") catch unreachable;
-    const use_new_module_cli = runtime_zig_version.version.order(module_version) != .lt;
-
-    const standard_args = if (use_new_module_cli) blk: {
-        const build_module = try std.fmt.allocPrint(arena_allocator, "@build@::{s}", .{build_file_path});
-
-        break :blk [_][]const u8{
-            config.zig_exe_path.?,
-            "run",
-            config.build_runner_path.?,
-            "--cache-dir",
-            config.global_cache_path.?,
-            "--mod",
-            build_module,
-            "--deps",
-            "@build@",
-            "--",
-            config.zig_exe_path.?,
-            directory_path,
-            zig_cache_root,
-            config.build_runner_global_cache_path.?,
-        };
-    } else [_][]const u8{
-        config.zig_exe_path.?,
-        "run",
-        config.build_runner_path.?,
-        "--cache-dir",
-        config.global_cache_path.?,
-        "--pkg-begin",
-        "@build@",
-        build_file_path,
-        "--pkg-end",
-        "--",
-        config.zig_exe_path.?,
-        directory_path,
-        zig_cache_root,
-        config.build_runner_global_cache_path.?,
-    };
 
-    const arg_length = standard_args.len + if (build_file.build_associated_config) |cfg| if (cfg.build_options) |options| options.len else 0 else 0;
+    // NOTE: This used to be backwards compatible
+    // but then I came in like a wrecking ball
+
+    const arg_length = 4 + if (build_file.build_associated_config) |cfg| if (cfg.build_options) |options| options.len else 0 else 0;
     var args = try std.ArrayListUnmanaged([]const u8).initCapacity(arena_allocator, arg_length);
-    args.appendSliceAssumeCapacity(standard_args[0..]);
+    try populateBuildConfigurationArgs(arena_allocator, &args, config.zig_exe_path.?, config.build_runner_path.?);
+
     if (build_file.build_associated_config) |cfg| {
         if (cfg.build_options) |options| {
             for (options) |opt| {
@@ -502,16 +495,7 @@ fn loadBuildConfiguration(
         }
     }
 
-    const zig_run_result = try std.ChildProcess.exec(.{
-        .allocator = arena_allocator,
-        .argv = args.items,
-        .cwd = try std.fs.path.resolve(arena_allocator, &.{ config.zig_exe_path.?, "../" }),
-    });
-
-    defer {
-        arena_allocator.free(zig_run_result.stdout);
-        arena_allocator.free(zig_run_result.stderr);
-    }
+    var zig_run_result = try executeBuildRunner(arena_allocator, build_file_path, args.items);
 
     errdefer blk: {
         const joined = std.mem.join(arena_allocator, " ", args.items) catch break :blk;
@@ -527,13 +511,20 @@ fn loadBuildConfiguration(
         else => return error.RunFailed,
     }
 
-    const parse_options = std.json.ParseOptions{ .allocator = allocator };
+    const parse_options = std.json.ParseOptions{
+        .allocator = allocator,
+        // We ignore unknown fields so people can roll
+        // their own build runners in libraries with
+        // the only requirement being general adherance
+        // to the BuildConfig type
+        .ignore_unknown_fields = true,
+    };
     var token_stream = std.json.TokenStream.init(zig_run_result.stdout);
     var build_config = std.json.parse(BuildConfig, &token_stream, parse_options) catch return error.RunFailed;
     errdefer std.json.parseFree(BuildConfig, build_config, parse_options);
 
     for (build_config.packages) |*pkg| {
-        const pkg_abs_path = try std.fs.path.resolve(allocator, &[_][]const u8{ directory_path, pkg.path });
+        const pkg_abs_path = try std.fs.path.resolve(allocator, &[_][]const u8{ build_file_path, "..", pkg.path });
         allocator.free(pkg.path);
         pkg.path = pkg_abs_path;
     }
@@ -688,7 +679,7 @@ fn uriInImports(
 }
 
 /// takes ownership of the text passed in.
-fn createDocument(self: *DocumentStore, uri: Uri, text: [:0]u8, open: bool) error{OutOfMemory}!Handle {
+fn createDocument(self: *DocumentStore, uri: Uri, text: [:0]const u8, open: bool) error{OutOfMemory}!Handle {
     const tracy_zone = tracy.trace(@src());
     defer tracy_zone.end();