Delay finding and associating a build file with a (handle) uri

src/ComptimeInterpreter.zig

@@ -892,10 +892,10 @@
                     } };
                 }
 
-                const import_uri = (try interpreter.document_store.uriFromImportStr(interpreter.allocator, interpreter.getHandle().*, import_str[1 .. import_str.len - 1])) orelse return error.ImportFailure;
+                const import_uri = (try interpreter.document_store.uriFromImportStr(interpreter.allocator, interpreter.getHandle(), import_str[1 .. import_str.len - 1])) orelse return error.ImportFailure;
                 defer interpreter.allocator.free(import_uri);
 
-                const import_handle = interpreter.document_store.getOrLoadHandle(import_uri) orelse return error.ImportFailure;
+                const import_handle = interpreter.document_store.getOrLoadHandle(import_uri, true) orelse return error.ImportFailure;
                 const import_interpreter = try import_handle.getComptimeInterpreter(interpreter.document_store, interpreter.ip);
 
                 if (import_interpreter.mutex.tryLock()) {

src/Server.zig

@@ -142,14 +142,25 @@
 const Job = union(enum) {
     incoming_message: std.json.Parsed(Message),
     generate_diagnostics: DocumentStore.Uri,
-    load_build_configuration: DocumentStore.Uri,
+    load_handle: DocumentStore.Uri,
+    find_potential_build_files: DocumentStore.Uri,
+    load_build_configuration: struct {
+        build_file_uri: DocumentStore.Uri,
+        maybe_handle_uri: ?DocumentStore.Uri = null,
+    },
     run_build_on_save,
 
     fn deinit(self: Job, allocator: std.mem.Allocator) void {
         switch (self) {
             .incoming_message => |parsed_message| parsed_message.deinit(),
-            .generate_diagnostics => |uri| allocator.free(uri),
-            .load_build_configuration => |uri| allocator.free(uri),
+            .load_handle,
+            .find_potential_build_files,
+            .generate_diagnostics,
+            => |uri| allocator.free(uri),
+            .load_build_configuration => |params| {
+                allocator.free(params.build_file_uri);
+                if (params.maybe_handle_uri) |handle_uri| allocator.free(handle_uri);
+            },
             .run_build_on_save => {},
         }
     }
@@ -168,7 +179,10 @@
     fn syncMode(self: Job) SynchronizationMode {
         return switch (self) {
             .incoming_message => |parsed_message| if (parsed_message.value.isBlocking()) .exclusive else .shared,
-            .generate_diagnostics => .shared,
+            .load_handle,
+            .generate_diagnostics,
+            => .shared,
+            .find_potential_build_files,
             .load_build_configuration,
             .run_build_on_save,
             => .atomic,
@@ -681,7 +695,7 @@
     try server.job_queue.ensureUnusedCapacity(server.document_store.build_files.count());
     for (server.document_store.build_files.keys()) |build_file_uri| {
         server.job_queue.writeItemAssumeCapacity(.{
-            .load_build_configuration = try server.allocator.dupe(u8, build_file_uri),
+            .load_build_configuration = .{ .build_file_uri = try server.allocator.dupe(u8, build_file_uri) },
         });
     }
 }
@@ -1195,7 +1209,7 @@
 
     if (std.process.can_spawn and DocumentStore.isBuildFile(uri)) {
         try server.pushJob(.{
-            .load_build_configuration = try server.allocator.dupe(u8, uri),
+            .load_build_configuration = .{ .build_file_uri = try server.allocator.dupe(u8, uri) },
         });
     }
 
@@ -1777,9 +1791,14 @@
     if (zig_builtin.single_threaded) {
         server.thread_pool = {};
     } else {
+        var num_logical_cpus = std.Thread.getCpuCount() catch 8;
+        if (num_logical_cpus < 4) num_logical_cpus = 4;
         try server.thread_pool.init(.{
             .allocator = allocator,
-            .n_jobs = 4, // what is a good value here?
+            .n_jobs = @min(
+                9, // + 1, main thread = total 10
+                (num_logical_cpus - 3), // < 12 cores => - 1, main thread; - 1, music player; - 1, msgs client :)
+            ),
         });
     }
 
@@ -2023,6 +2042,12 @@
             const response = server.processMessageReportError(parsed_message.value) orelse return;
             server.allocator.free(response);
         },
+        .find_potential_build_files => |handle_uri| {
+            server.document_store.findPotentialBuildFiles(handle_uri) catch return;
+        },
+        .load_handle => |uri| {
+            _ = server.document_store.getOrLoadHandle(uri, true);
+        },
         .generate_diagnostics => |uri| {
             const handle = server.document_store.getHandle(uri) orelse return;
             var arena_allocator = std.heap.ArenaAllocator.init(server.allocator);
@@ -2031,10 +2056,10 @@
             const json_message = server.sendToClientNotification("textDocument/publishDiagnostics", diagnostics) catch return;
             server.allocator.free(json_message);
         },
-        .load_build_configuration => |build_file_uri| {
+        .load_build_configuration => |params| {
             std.debug.assert(std.process.can_spawn);
             if (!std.process.can_spawn) return;
-            server.document_store.invalidateBuildFile(build_file_uri) catch return;
+            server.document_store.invalidateBuildFile(params.build_file_uri, params.maybe_handle_uri) catch return;
         },
         .run_build_on_save => {
             std.debug.assert(std.process.can_spawn);

src/analysis.zig

@@ -28,6 +28,7 @@
 collect_callsite_references: bool,
 /// handle of the doc where the request originated
 root_handle: ?*DocumentStore.Handle,
+load_handles_immediately: bool = true,
 
 const NodeSet = std.HashMapUnmanaged(NodeWithUri, void, NodeWithUri.Context, std.hash_map.default_max_load_percentage);
 
@@ -1476,7 +1477,7 @@
                     else => return null,
                 };
 
-                const new_handle = analyser.store.getOrLoadHandle(builtin_uri) orelse return null;
+                const new_handle = analyser.store.getOrLoadHandle(builtin_uri, true) orelse return null;
                 const new_handle_document_scope = try new_handle.getDocumentScope();
 
                 const decl_index = new_handle_document_scope.getScopeDeclaration(.{
@@ -1507,22 +1508,22 @@
                 const import_str = tree.tokenSlice(main_tokens[import_param]);
                 const import_uri = (try analyser.store.uriFromImportStr(
                     analyser.arena.allocator(),
-                    handle.*,
+                    handle,
                     import_str[1 .. import_str.len - 1],
                 )) orelse (try analyser.store.uriFromImportStr(
                     analyser.arena.allocator(),
-                    if (analyser.root_handle) |root_handle| root_handle.* else return null,
+                    if (analyser.root_handle) |root_handle| root_handle else return null,
                     import_str[1 .. import_str.len - 1],
                 )) orelse return null;
 
-                const new_handle = analyser.store.getOrLoadHandle(import_uri) orelse return null;
+                const new_handle = analyser.store.getOrLoadHandle(import_uri, analyser.load_handles_immediately) orelse return null;
 
                 // reference to node '0' which is root
                 return TypeWithHandle.typeVal(.{ .node = 0, .handle = new_handle });
             } else if (std.mem.eql(u8, call_name, "@cImport")) {
-                const cimport_uri = (try analyser.store.resolveCImport(handle.*, node)) orelse return null;
+                const cimport_uri = (try analyser.store.resolveCImport(handle, node)) orelse return null;
 
-                const new_handle = analyser.store.getOrLoadHandle(cimport_uri) orelse return null;
+                const new_handle = analyser.store.getOrLoadHandle(cimport_uri, analyser.load_handles_immediately) orelse return null;
 
                 // reference to node '0' which is root
                 return TypeWithHandle.typeVal(.{ .node = 0, .handle = new_handle });
@@ -2533,8 +2534,8 @@
                         .start = import_str_tok.loc.start + 1,
                         .end = import_str_tok.loc.end - 1,
                     });
-                    const uri = try analyser.store.uriFromImportStr(analyser.arena.allocator(), curr_handle.*, import_str) orelse return null;
-                    const node_handle = analyser.store.getOrLoadHandle(uri) orelse return null;
+                    const uri = try analyser.store.uriFromImportStr(analyser.arena.allocator(), curr_handle, import_str) orelse return null;
+                    const node_handle = analyser.store.getOrLoadHandle(uri, true) orelse return null;
                     current_type = TypeWithHandle.typeVal(NodeWithHandle{ .handle = node_handle, .node = 0 });
                     _ = tokenizer.next(); // eat the .r_paren
                 } else {
@@ -3168,7 +3169,7 @@
                     var possible = std.ArrayListUnmanaged(Type.EitherEntry){};
 
                     for (refs.items) |ref| {
-                        const handle = analyser.store.getOrLoadHandle(ref.uri).?;
+                        const handle = analyser.store.getOrLoadHandle(ref.uri, analyser.load_handles_immediately).?;
 
                         var call_buf: [1]Ast.Node.Index = undefined;
                         const call = tree.fullCall(&call_buf, ref.call_node).?;

src/features/completions.zig

@@ -550,6 +550,7 @@
     var completions = std.ArrayListUnmanaged(types.CompletionItem){};
 
     const type_handle = (try analyser.getFieldAccessType(handle, source_index, loc)) orelse return null;
+    analyser.load_handles_immediately = false;
     try typeToCompletion(server, analyser, arena, &completions, type_handle, handle, null);
     try formatCompletionDetails(server, arena, completions.items);
 
@@ -738,7 +739,7 @@
     const tracy_zone = tracy.trace(@src());
     defer tracy_zone.end();
 
-    return try server.document_store.errorCompletionItems(arena, handle.*);
+    return try server.document_store.errorCompletionItems(arena, handle);
 }
 
 fn kindToSortScore(kind: types.CompletionItemKind) ?[]const u8 {
@@ -808,7 +809,7 @@
     if (token_tags[dot_token_index - 1] == .number_literal or token_tags[dot_token_index - 1] != .equal) return &.{};
 
     // `var enum_val = .` or the get*Context logic failed because of syntax errors (parser didn't create the necessary node(s))
-    const enum_completions = try document_store.enumCompletionItems(arena, handle.*);
+    const enum_completions = try document_store.enumCompletionItems(arena, handle);
     return enum_completions;
 }
 
@@ -820,7 +821,7 @@
 fn completeFileSystemStringLiteral(
     arena: std.mem.Allocator,
     store: *DocumentStore,
-    handle: DocumentStore.Handle,
+    handle: *DocumentStore.Handle,
     pos_context: Analyser.PositionContext,
 ) ![]types.CompletionItem {
     var completions: DocumentScope.CompletionSet = .{};
@@ -893,10 +894,10 @@
     }
 
     if (completing.len == 0 and pos_context == .import_string_literal) {
-        if (handle.associated_build_file) |uri| blk: {
+        if (handle.getAssociatedBuildFileUri(store, true)) |uri| blk: {
             const build_file = store.getBuildFile(uri).?;
-            const build_config = build_file.tryLockConfig() orelse break :blk;
-            defer build_file.unlockConfig();
+            const build_config = build_file.getConfig() orelse break :blk;
+            defer build_file.unlockShared();
 
             try completions.ensureUnusedCapacity(arena, build_config.packages.len);
             for (build_config.packages) |pkg| {
@@ -908,8 +909,8 @@
             }
         } else if (DocumentStore.isBuildFile(handle.uri)) blk: {
             const build_file = store.getBuildFile(handle.uri) orelse break :blk;
-            const build_config = build_file.tryLockConfig() orelse break :blk;
-            defer build_file.unlockConfig();
+            const build_config = build_file.getConfig() orelse break :blk;
+            defer build_file.unlockShared();
 
             try completions.ensureUnusedCapacity(arena, build_config.deps_build_roots.len);
             for (build_config.deps_build_roots) |dbr| {
@@ -970,7 +971,7 @@
         .string_literal,
         => blk: {
             if (pos_context == .string_literal and !DocumentStore.isBuildFile(handle.uri)) break :blk null;
-            break :blk completeFileSystemStringLiteral(arena, &server.document_store, handle.*, pos_context) catch |err| {
+            break :blk completeFileSystemStringLiteral(arena, &server.document_store, handle, pos_context) catch |err| {
                 log.err("failed to get file system completions: {}", .{err});
                 return null;
             };

src/features/goto.zig

@@ -204,13 +204,13 @@
     const uri = switch (pos_context) {
         .import_string_literal,
         .embedfile_string_literal,
-        => try document_store.uriFromImportStr(arena, handle.*, import_str),
+        => try document_store.uriFromImportStr(arena, handle, import_str),
         .cinclude_string_literal => try URI.fromPath(
             arena,
             blk: {
                 if (std.fs.path.isAbsolute(import_str)) break :blk import_str;
                 var include_dirs: std.ArrayListUnmanaged([]const u8) = .{};
-                _ = document_store.collectIncludeDirs(arena, handle.*, &include_dirs) catch |err| {
+                _ = document_store.collectIncludeDirs(arena, handle, &include_dirs) catch |err| {
                     log.err("failed to resolve include paths: {}", .{err});
                     return null;
                 };

src/features/references.zig

@@ -159,7 +159,7 @@
 
         var handle_dependencies = std.ArrayListUnmanaged([]const u8){};
         defer handle_dependencies.deinit(allocator);
-        try analyser.store.collectDependencies(allocator, handle.*, &handle_dependencies);
+        try analyser.store.collectDependencies(allocator, handle, &handle_dependencies);
 
         try dependencies.ensureUnusedCapacity(allocator, handle_dependencies.items.len);
         for (handle_dependencies.items) |uri| {
@@ -174,7 +174,7 @@
         if (std.mem.eql(u8, uri, curr_handle.uri)) continue;
         const handle = switch (handle_behavior) {
             .get => analyser.store.getHandle(uri),
-            .get_or_load => analyser.store.getOrLoadHandle(uri),
+            .get_or_load => analyser.store.getOrLoadHandle(uri, true),
         } orelse continue;
 
         try builder.collectReferences(handle, 0);