Restore cache on a per-element basis

drift_dev/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.11.1-dev
+
+- Reduce the amount of assets read by drift, improving build performance and enabling faster
+  incremental rebuilds.
+
 ## 2.11.0
 
 - [Nested result columns](https://drift.simonbinder.eu/docs/using-sql/drift_files/#nested-results)

drift_dev/lib/src/analysis/driver/cache.dart

@@ -1,14 +1,18 @@
+import 'package:analyzer/dart/element/element.dart';
+
 import '../results/element.dart';
 import 'state.dart';
 
 /// An in-memory cache of analysis results for drift elements.
 ///
 /// At the moment, the cache is not set up to handle changing files.
 class DriftAnalysisCache {
-  final Map<Uri, CachedSerializationResult> serializationCache = {};
+  final Map<Uri, CachedSerializationResult?> serializationCache = {};
   final Map<Uri, FileState> knownFiles = {};
   final Map<DriftElementId, DriftElementKind> discoveredElements = {};
 
+  final Map<Uri, LibraryElement?> typeHelperLibraries = {};
+
   FileState stateForUri(Uri uri) {
     return knownFiles[uri] ?? notifyFileChanged(uri);
   }

drift_dev/lib/src/analysis/driver/driver.dart

@@ -12,7 +12,6 @@ import '../resolver/drift/sqlparser/mapping.dart';
 import '../resolver/file_analysis.dart';
 import '../resolver/queries/custom_known_functions.dart';
 import '../resolver/resolver.dart';
-import '../results/results.dart';
 import '../serializer.dart';
 import 'cache.dart';
 import 'error.dart';
@@ -59,7 +58,6 @@ class DriftAnalysisDriver {
   final bool _isTesting;
 
   late final TypeMapping typeMapping = TypeMapping(this);
-  late final ElementDeserializer deserializer = ElementDeserializer(this);
 
   AnalysisResultCacheReader? cacheReader;
 
@@ -100,8 +98,9 @@ class DriftAnalysisDriver {
   ///
   /// Returns non-null if analysis results were found and successfully restored.
   Future<Map<String, Object?>?> readStoredAnalysisResult(Uri uri) async {
-    final cached = cache.serializationCache[uri];
-    if (cached != null) return cached.cachedElements;
+    if (cache.serializationCache.containsKey(uri)) {
+      return cache.serializationCache[uri]?.cachedElements;
+    }
 
     // Not available in in-memory cache, so let's read it from the file system.
     final reader = cacheReader;
@@ -122,41 +121,6 @@ class DriftAnalysisDriver {
     return data.cachedElements;
   }
 
-  Future<bool> _recoverFromCache(FileState state) async {
-    final stored = await readStoredAnalysisResult(state.ownUri);
-    if (stored == null) return false;
-
-    var allRecovered = true;
-
-    for (final local in stored.keys) {
-      final id = DriftElementId(state.ownUri, local);
-      try {
-        await deserializer.readDriftElement(id);
-      } on CouldNotDeserializeException catch (e, s) {
-        backend.log.fine('Could not deserialize $id', e, s);
-        allRecovered = false;
-      }
-    }
-
-    final cachedImports = cache.serializationCache[state.ownUri]?.cachedImports;
-    if (cachedImports != null && state.discovery == null) {
-      state.cachedDiscovery ??= CachedDiscoveryResults(true, cachedImports, []);
-
-      for (final import in cachedImports) {
-        final found = cache.stateForUri(import);
-
-        if (found.imports == null) {
-          // Attempt to recover this file as well to make sure we know the
-          // imports for every file transitively reachable from the sources
-          // analyzed.
-          await _recoverFromCache(found);
-        }
-      }
-    }
-
-    return allRecovered;
-  }
-
   Future<void> discoverIfNecessary(
     FileState file, {
     bool warnIfFileDoesntExist = true,
@@ -208,15 +172,10 @@ class DriftAnalysisDriver {
     return known;
   }
 
-  Future<void> _findLocalElementsInAllImports(FileState known) async {
-    // To analyze references in elements, we also need to know locally defined
-    // elements in all imports.
+  Future<void> _warnAboutUnresolvedImportsInDriftFile(FileState known) async {
     final state = known.discovery;
     if (state is DiscoveredDriftFile) {
       for (final import in state.imports) {
-        // todo: We shouldn't unconditionally crawl files like this. The build
-        // backend should emit prepared file results in a previous step which
-        // should be used here.
         final file = await findLocalElements(import.importedUri);
 
         if (file.isValidImport != true) {
@@ -227,37 +186,26 @@ class DriftAnalysisDriver {
               "can't be imported.",
             ),
           );
-        } else {
-          await _findLocalElementsInAllImports(file);
         }
       }
-    } else {
-      for (final import in known.imports ?? const <Uri>[]) {
-        await findLocalElements(
-          import,
-          // We might import a generated file that doesn't exist yet, that
-          // should not be a user-visible error. Users will notice because the
-          // import is reported as an error by the analyzer either way.
-          warnIfFileDoesntExist: false,
-        );
-      }
     }
   }
 
-  /// Runs the second analysis step (element analysis) on a file.
+  /// Analyzes elements known to be defined in [state], or restores them from
+  /// cache.
   ///
-  /// The file, as well as all imports, should have undergone the first analysis
-  /// step (discovery) at this point, so that the resolver is able to
-  /// recognize dependencies between different elements.
-  Future<void> _analyzePrepared(FileState state) async {
-    assert(state.discovery != null);
+  /// Elements in the file must be known at this point - either because the file
+  /// was discovered or because discovered elements have been imported from
+  /// cache.
+  Future<void> _analyzeLocalElements(FileState state) async {
+    assert(state.discovery != null || state.cachedDiscovery != null);
 
-    for (final discovered in state.discovery!.locallyDefinedElements) {
+    for (final discovered in state.definedElements) {
       if (!state.elementIsAnalyzed(discovered.ownId)) {
         final resolver = DriftResolver(this);
 
         try {
-          await resolver.resolveDiscovered(discovered);
+          await resolver.resolveEntrypoint(discovered.ownId);
         } catch (e, s) {
           if (e is! CouldNotResolveElementException) {
             backend.log.warning('Could not analyze ${discovered.ownId}', e, s);
@@ -278,19 +226,38 @@ class DriftAnalysisDriver {
       return known;
     }
 
-    final allRecoveredFromCache = await _recoverFromCache(known);
-    if (allRecoveredFromCache) {
-      // We were able to read all elements from cache, so we don't have to
-      // run any analysis now.
-      return known;
-    }
-
     // We couldn't recover all analyzed elements. Let's run an analysis run
     // then.
-    await discoverIfNecessary(known);
-    await _findLocalElementsInAllImports(known);
+    await findLocalElements(uri);
+    await _warnAboutUnresolvedImportsInDriftFile(known);
+
+    // Also make sure elements in transitive imports have been resolved.
+    final seen = cache.knownFiles.keys.toSet();
+    final pending = <Uri>[known.ownUri];
+
+    while (pending.isNotEmpty) {
+      final file = pending.removeLast();
+      seen.add(file);
+
+      final fileState = await findLocalElements(
+        file,
+        // We might import a generated file that doesn't exist yet, that
+        // should not be a user-visible error. Users will notice because the
+        // import is reported as an error by the analyzer either way.
+        warnIfFileDoesntExist: true,
+      );
+
+      // Drift file imports are transitive, Dart imports aren't.
+      if (file == known.ownUri || fileState.extension == '.drift') {
+        for (final dependency in fileState.imports ?? const <Uri>[]) {
+          if (!seen.contains(dependency)) {
+            pending.add(dependency);
+          }
+        }
+      }
+    }
 
-    await _analyzePrepared(known);
+    await _analyzeLocalElements(known);
     return known;
   }
 

drift_dev/lib/src/analysis/resolver/resolver.dart

@@ -6,6 +6,7 @@ import '../driver/error.dart';
 import '../driver/state.dart';
 import '../results/element.dart';
 
+import '../serializer.dart';
 import 'dart/accessor.dart' as dart_accessor;
 import 'dart/table.dart' as dart_table;
 import 'dart/view.dart' as dart_view;
@@ -25,10 +26,38 @@ class DriftResolver {
   /// This path is used to detect and prevent circular references.
   final List<DriftElementId> _currentDependencyPath = [];
 
+  late final ElementDeserializer _deserializer =
+      ElementDeserializer(driver, _currentDependencyPath);
+
   DriftResolver(this.driver);
 
+  Future<DriftElement> resolveEntrypoint(DriftElementId element) async {
+    assert(_currentDependencyPath.isEmpty);
+    _currentDependencyPath.add(element);
+
+    return await _restoreOrResolve(element);
+  }
+
+  Future<DriftElement> _restoreOrResolve(DriftElementId element) async {
+    try {
+      if (await driver.readStoredAnalysisResult(element.libraryUri) != null) {
+        return await _deserializer.readDriftElement(element);
+      }
+    } on CouldNotDeserializeException catch (e, s) {
+      driver.backend.log.fine('Could not deserialize $element', e, s);
+    }
+
+    // We can't resolve the element from cache, so we need to resolve it.
+    final owningFile = driver.cache.stateForUri(element.libraryUri);
+    await driver.discoverIfNecessary(owningFile);
+    final discovered = owningFile.discovery!.locallyDefinedElements
+        .firstWhere((e) => e.ownId == element);
+
+    return _resolveDiscovered(discovered);
+  }
+
   /// Resolves a discovered element by analyzing it and its dependencies.
-  Future<DriftElement> resolveDiscovered(DiscoveredElement discovered) async {
+  Future<DriftElement> _resolveDiscovered(DiscoveredElement discovered) async {
     LocalElementResolver resolver;
 
     final fileState = driver.cache.knownFiles[discovered.ownId.libraryUri]!;
@@ -111,18 +140,14 @@ class DriftResolver {
 
     final pending = driver.cache.discoveredElements[reference];
     if (pending != null) {
+      // We know the element exists, but we haven't resolved it yet.
       _currentDependencyPath.add(reference);
 
       try {
-        final owningFile = driver.cache.stateForUri(reference.libraryUri);
-        await driver.discoverIfNecessary(owningFile);
-        final discovered = owningFile.discovery!.locallyDefinedElements
-            .firstWhere((e) => e.ownId == reference);
-
-        final resolved = await resolveDiscovered(discovered);
+        final resolved = await _restoreOrResolve(reference);
         return ResolvedReferenceFound(resolved);
       } catch (e, s) {
-        driver.backend.log.warning('Could not analze $reference', e, s);
+        driver.backend.log.warning('Could not analyze $reference', e, s);
         return ReferencedElementCouldNotBeResolved();
       } finally {
         final removed = _currentDependencyPath.removeLast();

drift_dev/lib/src/analysis/serializer.dart

@@ -358,20 +358,21 @@ class ElementSerializer {
 
 /// Deserializes the element structure emitted by [ElementSerializer].
 class ElementDeserializer {
-  final Map<Uri, LibraryElement?> _typeHelperLibraries = {};
-  final List<DriftElementId> _currentlyReading = [];
+  final List<DriftElementId> _currentlyReading;
 
   final DriftAnalysisDriver driver;
 
-  ElementDeserializer(this.driver);
+  ElementDeserializer(this.driver, this._currentlyReading);
 
   Future<DartType> _readDartType(Uri import, int typeId) async {
     LibraryElement? element;
-    if (_typeHelperLibraries.containsKey(import)) {
-      element = _typeHelperLibraries[import];
+    final helpers = driver.cache.typeHelperLibraries;
+
+    if (helpers.containsKey(import)) {
+      element = helpers[import];
     } else {
-      element = _typeHelperLibraries[import] =
-          await driver.cacheReader!.readTypeHelperFor(import);
+      element =
+          helpers[import] = await driver.cacheReader!.readTypeHelperFor(import);
     }
 
     if (element == null) {
@@ -383,11 +384,28 @@ class ElementDeserializer {
     return typedef.aliasedType;
   }
 
-  Future<DriftElement> _readElementReference(Map json) {
-    return readDriftElement(DriftElementId.fromJson(json));
+  Future<DriftElement> _readElementReference(Map json) async {
+    final id = DriftElementId.fromJson(json);
+
+    if (_currentlyReading.contains(id)) {
+      throw StateError(
+          'Circular error when deserializing drift modules. This is a '
+          'bug in drift_dev!');
+    }
+
+    _currentlyReading.add(id);
+
+    try {
+      return await readDriftElement(DriftElementId.fromJson(json));
+    } finally {
+      final lastId = _currentlyReading.removeLast();
+      assert(lastId == id);
+    }
   }
 
   Future<DriftElement> readDriftElement(DriftElementId id) async {
+    assert(_currentlyReading.last == id);
+
     final state = driver.cache.stateForUri(id.libraryUri).analysis[id] ??=
         ElementAnalysisState(id);
     if (state.result != null && state.isUpToDate) {
@@ -397,18 +415,10 @@ class ElementDeserializer {
     final data = await driver.readStoredAnalysisResult(id.libraryUri);
     if (data == null) {
       throw CouldNotDeserializeException(
-          'Analysis data for ${id..libraryUri} not found');
-    }
-
-    if (_currentlyReading.contains(id)) {
-      throw StateError(
-          'Circular error when deserializing drift modules. This is a '
-          'bug in drift_dev!');
+          'Analysis data for ${id.libraryUri} not found');
     }
 
     try {
-      _currentlyReading.add(id);
-
       final result = await _readDriftElement(data[id.name] as Map);
       state
         ..result = result
@@ -419,9 +429,6 @@ class ElementDeserializer {
 
       throw CouldNotDeserializeException(
           'Internal error while deserializing $id: $e at \n$s');
-    } finally {
-      final lastId = _currentlyReading.removeLast();
-      assert(lastId == id);
     }
   }
 
@@ -665,7 +672,7 @@ class ElementDeserializer {
             schemaVersion: json['schema_version'] as int?,
             accessors: [
               for (final dao in json.list('daos'))
-                await readDriftElement(DriftElementId.fromJson(dao as Map))
+                await _readElementReference(dao as Map<String, Object?>)
                     as DatabaseAccessor,
             ],
           );