feat: Move factions to the database

This removes the FactionCache.
I also converted varchar to text and json types in the db.
Renamed toSubsitutionValues to toColumnMap to be shorter and easier to spell.
removed factionCache from ui for now.

README.md

@@ -641,4 +641,9 @@ Invalid argument (marketSymbol): ESEIDEL-6F is not at X1-YA22-87615D, X1-YA22-92
 
 ### Types
 Move Transaction and other custom model objects down to that layer.
-Remove api.dart re-export of types.
+Remove api.dart re-export of types.
+
+### Surveys
+Miners will likely fight over surveys, probably all grabbing the "best"
+survey and possibly exhausting it at the same time and having to restart
+the mining operation?

packages/cli/bin/cli.dart

@@ -93,7 +93,7 @@ Future<void> cliMain(List<String> args) async {
   final api =
       getApi(token, db, useOutOfProcessNetwork: !(results['local'] as bool));
 
-  final caches = await Caches.load(fs, api);
+  final caches = await Caches.load(fs, api, db);
   logger.info(
     'Loaded ${caches.marketPrices.count} prices from '
     '${caches.marketPrices.waypointCount} markets and '

packages/cli/bin/route_perf.dart

@@ -1,10 +1,10 @@
 import 'package:cli/api.dart';
-import 'package:cli/cache/faction_cache.dart';
 import 'package:cli/cache/systems_cache.dart';
 import 'package:cli/cli.dart';
 import 'package:cli/nav/jump_cache.dart';
 import 'package:cli/nav/route.dart';
 import 'package:cli/nav/system_connectivity.dart';
+import 'package:db/db.dart';
 
 void main(List<String> args) async {
   await runOffline(args, command);
@@ -25,12 +25,13 @@ class Result {
 Future<void> command(FileSystem fs, List<String> args) async {
   const count = 5000;
 
+  final db = await defaultDatabase();
   final systemsCache = SystemsCache.loadCached(fs)!;
   final systemConnectivity = SystemConnectivity.fromSystemsCache(systemsCache);
   final jumpCache = JumpCache();
-  final factionCache = FactionCache.loadFromCache(fs)!;
   // COSMIC has always been on the main jumpgate network.
-  final factionHq = factionCache.headquartersFor(FactionSymbols.COSMIC);
+  final factionHq =
+      (await db.factionBySymbol(FactionSymbols.COSMIC)).headquartersSymbol;
   final systemSymbol = factionHq.systemSymbol;
   final clusterId = systemConnectivity.clusterIdForSystem(systemSymbol);
   final allSystemSymbols =

packages/cli/bin/systems_reachable_from_factions.dart

@@ -1,14 +1,11 @@
-import 'package:cli/api.dart';
-import 'package:cli/cache/faction_cache.dart';
-import 'package:cli/cache/systems_cache.dart';
+import 'package:cli/cache/caches.dart';
 import 'package:cli/cli.dart';
-import 'package:cli/nav/system_connectivity.dart';
+import 'package:db/db.dart';
 
 Future<void> command(FileSystem fs, List<String> args) async {
+  final db = await defaultDatabase();
   final systemsCache = await SystemsCache.load(fs);
-  final factionCache = await FactionCache.loadUnauthenticated(fs);
-
-  final factions = factionCache.factions;
+  final factions = await loadFactions(db);
 
   final clusterCache = SystemConnectivity.fromSystemsCache(systemsCache);
   for (final faction in factions) {

packages/cli/lib/behavior/explorer.dart

@@ -185,17 +185,17 @@ Future<WaypointSymbol?> findNewWaypointSymbolToExplore(
   return null;
 }
 
-WaypointSymbol _nearestHeadquarters(
+Future<WaypointSymbol> _nearestHeadquarters(
+  Database db,
   SystemConnectivity systemConnectivity,
   SystemsCache systemsCache,
   AgentCache agentCache,
-  FactionCache factionCache,
   Ship ship,
-) {
-  final factionsHqs =
-      factionCache.factions.map((e) => e.headquartersSymbol).toList();
+) async {
+  final factionHqs =
+      (await loadFactions(db)).map((e) => e.headquartersSymbol).toList();
   final startSystem = systemsCache.systemBySymbol(ship.systemSymbol);
-  final reachableHqs = factionsHqs
+  final reachableHqs = factionHqs
       .where(
         (hq) => systemConnectivity.canJumpBetweenSystemSymbols(
           ship.systemSymbol,
@@ -217,6 +217,7 @@ WaypointSymbol _nearestHeadquarters(
 /// If we're low on fuel, route to the nearest market which trades fuel.
 Future<DateTime?> routeForEmergencyFuelingIfNeeded(
   Api api,
+  Database db,
   Caches caches,
   CentralCommand centralCommand,
   Waypoint waypoint,
@@ -237,11 +238,11 @@ Future<DateTime?> routeForEmergencyFuelingIfNeeded(
       ?.waypointSymbol;
   if (destination == null) {
     shipErr(ship, 'No nearby market trades fuel, routing to nearest hq.');
-    destination = _nearestHeadquarters(
+    destination = await _nearestHeadquarters(
+      db,
       caches.systemConnectivity,
       caches.systems,
       caches.agent,
-      caches.factions,
       ship,
     );
   }
@@ -307,6 +308,7 @@ Future<DateTime?> advanceExplorer(
   // which do not have markets.  Other behaviors always stick to markets.
   final refuelWaitTime = await routeForEmergencyFuelingIfNeeded(
     api,
+    db,
     caches,
     centralCommand,
     waypoint,

packages/cli/lib/cache/caches.dart

@@ -4,7 +4,6 @@ import 'package:cli/cache/behavior_cache.dart';
 import 'package:cli/cache/charting_cache.dart';
 import 'package:cli/cache/contract_cache.dart';
 import 'package:cli/cache/extraction_log.dart';
-import 'package:cli/cache/faction_cache.dart';
 import 'package:cli/cache/market_prices.dart';
 import 'package:cli/cache/ship_cache.dart';
 import 'package:cli/cache/shipyard_prices.dart';
@@ -13,6 +12,8 @@ import 'package:cli/cache/waypoint_cache.dart';
 import 'package:cli/nav/jump_cache.dart';
 import 'package:cli/nav/route.dart';
 import 'package:cli/nav/system_connectivity.dart';
+import 'package:cli/net/queries.dart';
+import 'package:db/db.dart';
 import 'package:file/file.dart';
 import 'package:http/http.dart' as http;
 
@@ -22,7 +23,6 @@ export 'package:cli/cache/behavior_cache.dart';
 export 'package:cli/cache/charting_cache.dart';
 export 'package:cli/cache/contract_cache.dart';
 export 'package:cli/cache/extraction_log.dart';
-export 'package:cli/cache/faction_cache.dart';
 export 'package:cli/cache/market_prices.dart';
 export 'package:cli/cache/ship_cache.dart';
 export 'package:cli/cache/shipyard_prices.dart';
@@ -47,7 +47,6 @@ class Caches {
     required this.markets,
     required this.contracts,
     required this.behaviors,
-    required this.factions,
     required this.charting,
     required this.routePlanner,
   });
@@ -85,9 +84,6 @@ class Caches {
   /// The cache of behaviors.
   final BehaviorCache behaviors;
 
-  /// The cache of factions.
-  final FactionCache factions;
-
   /// The cache of charting data.
   final ChartingCache charting;
 
@@ -97,7 +93,8 @@ class Caches {
   /// Load the cache from disk and network.
   static Future<Caches> load(
     FileSystem fs,
-    Api api, {
+    Api api,
+    Database db, {
     Future<http.Response> Function(Uri uri) httpGet = defaultHttpGet,
   }) async {
     final agent = await AgentCache.load(api, fs: fs);
@@ -113,8 +110,6 @@ class Caches {
     // Intentionally force refresh contracts in case we've been offline.
     final contracts = await ContractCache.load(api, fs: fs, forceRefresh: true);
     final behaviors = BehaviorCache.load(fs);
-    // Intentionally load factions from disk (they never change).
-    final factions = await FactionCache.load(api, fs: fs);
 
     final systemConnectivity = SystemConnectivity.fromSystemsCache(systems);
     final jumps = JumpCache();
@@ -124,8 +119,8 @@ class Caches {
       systemConnectivity: systemConnectivity,
     );
 
-    // Save out the caches we never modify so we don't have to load them again.
-    factions.save();
+    // Make sure factions are loaded.
+    await loadFactions(db);
 
     // We rarely modify contracts, so save them out here too.
     contracts.save();
@@ -142,7 +137,6 @@ class Caches {
       markets: markets,
       contracts: contracts,
       behaviors: behaviors,
-      factions: factions,
       charting: charting,
       routePlanner: routePlanner,
     );
@@ -169,3 +163,27 @@ class Caches {
     ships.save();
   }
 }
+
+/// Load all factions from the API.
+// With our out-of-process rate limiting, this won't matter that it uses
+// a separate API client.
+Future<List<Faction>> allFactionsUnauthenticated() async {
+  final factionsApi = FactionsApi();
+  final factions = await fetchAllPages(factionsApi, (factionsApi, page) async {
+    final response = await factionsApi.getFactions(page: page);
+    return (response!.data, response.meta);
+  }).toList();
+  return factions;
+}
+
+/// Loads the factions from the database, or fetches them from the API if
+/// they're not cached.
+Future<List<Faction>> loadFactions(Database db) async {
+  final cachedFactions = await db.allFactions();
+  if (cachedFactions.isNotEmpty) {
+    return Future.value(cachedFactions);
+  }
+  final factions = await allFactionsUnauthenticated();
+  await db.cacheFactions(factions);
+  return factions;
+}

packages/cli/lib/net/queue.dart

@@ -271,7 +271,7 @@ class NetQueue {
         continue;
       }
       final queued = QueuedResponse.fromJson(
-        jsonDecode(result[0][0] as String) as Map<String, dynamic>,
+        result[0][0] as Map<String, dynamic>,
       );
       return queued.toResponse();
     }
@@ -299,7 +299,7 @@ class NetQueue {
       id: row[0] as int,
       priority: row[1] as int,
       request: QueuedRequest.fromJson(
-        jsonDecode(row[2] as String) as Map<String, dynamic>,
+        row[2] as Map<String, dynamic>,
       ),
     );
   }

packages/cli/test/cache/caches_test.dart

@@ -1,26 +1,30 @@
 import 'package:cli/cache/caches.dart';
 import 'package:cli/logger.dart';
+import 'package:db/db.dart';
 import 'package:file/memory.dart';
 import 'package:mocktail/mocktail.dart';
 import 'package:test/test.dart';
 
-class _MockApi extends Mock implements Api {}
-
-class _MockAgentsApi extends Mock implements AgentsApi {}
-
 class _MockAgent extends Mock implements Agent {}
 
-class _MockFleetApi extends Mock implements FleetApi {}
+class _MockAgentsApi extends Mock implements AgentsApi {}
 
-class _MockLogger extends Mock implements Logger {}
+class _MockApi extends Mock implements Api {}
 
 class _MockContractsApi extends Mock implements ContractsApi {}
 
+class _MockDatabase extends Mock implements Database {}
+
 class _MockFactionsApi extends Mock implements FactionsApi {}
 
+class _MockFleetApi extends Mock implements FleetApi {}
+
+class _MockLogger extends Mock implements Logger {}
+
 void main() {
   test('Caches load test', () async {
     final api = _MockApi();
+    final db = _MockDatabase();
     final agentsApi = _MockAgentsApi();
     when(() => api.agents).thenReturn(agentsApi);
     final agent = _MockAgent();
@@ -69,7 +73,7 @@ void main() {
     Never httpGet(f) => throw UnimplementedError();
     final caches = await runWithLogger(
       logger,
-      () async => Caches.load(fs, api, httpGet: httpGet),
+      () async => Caches.load(fs, api, db, httpGet: httpGet),
     );
     expect(caches.agent, isNotNull);
   });