Attempt to fix networking deadlocks?

README.md

@@ -638,12 +638,26 @@ Invalid argument (marketSymbol): ESEIDEL-6F is not at X1-YA22-87615D, X1-YA22-92
 
 ## Teach late-start clients how to purchase traders instead of miners?
 
-
-### Types
-Move Transaction and other custom model objects down to that layer.
-Remove api.dart re-export of types.
-
 ### Surveys
 Miners will likely fight over surveys, probably all grabbing the "best"
 survey and possibly exhausting it at the same time and having to restart
-the mining operation?
+the mining operation?
+
+### Add a /healthz cli and monitor it.
+
+### Teach net_execute to recover from IO errors:
+
+Unhandled exception: (ClientException)
+Connection reset by peer
+#0      IOClient.send (package:http/src/io_client.dart:96:7)
+<asynchronous suspension>
+#1      BaseClient._sendUnstreamed (package:http/src/base_client.dart:93:32)
+<asynchronous suspension>
+#2      NetExecutor.sendRequest (file:///Users/eseidel/Documents/GitHub/space_traders/packages/cli/bin/network_execute.dart:55:9)
+<asynchronous suspension>
+#3      NetExecutor.run (file:///Users/eseidel/Documents/GitHub/space_traders/packages/cli/bin/network_execute.dart:92:24)
+<asynchronous suspension>
+#4      command (file:///Users/eseidel/Documents/GitHub/space_traders/packages/cli/bin/network_execute.dart:128:3)
+<asynchronous suspension>
+#5      main (file:///Users/eseidel/Documents/GitHub/space_traders/packages/cli/bin/network_execute.dart:132:3)
+<asynchronous suspension>

packages/cli/lib/net/queue.dart

@@ -1,6 +1,7 @@
 import 'dart:async';
 import 'dart:convert';
 
+import 'package:cli/logger.dart';
 import 'package:db/db.dart';
 import 'package:http/http.dart';
 
@@ -266,9 +267,13 @@ class NetQueue {
       );
       // If we don't yet have a response, wait for one.
       if (result.isEmpty) {
-        final _ = await _db.connection.notifications.firstWhere(
-          (notification) => notification.channel == 'response_',
-        );
+        try {
+          await _db.connection.notifications
+              .firstWhere((notification) => notification.channel == 'response_')
+              .timeout(const Duration(minutes: 1));
+        } on TimeoutException {
+          logger.err('Timed out waiting for response?');
+        }
         continue;
       }
       final queued = QueuedResponse.fromJson(
@@ -339,8 +344,12 @@ class NetQueue {
   Future<void> waitForRequest() async {
     assert(role == QueueRole.responder, 'Only responders can wait.');
     await _listenIfNeeded();
-    await _db.connection.notifications.firstWhere(
-      (notification) => notification.channel == 'request_',
-    );
+    try {
+      await _db.connection.notifications
+          .firstWhere((notification) => notification.channel == 'request_')
+          .timeout(const Duration(minutes: 1));
+    } on TimeoutException {
+      logger.err('Timed out waiting for request?');
+    }
   }
 }