JS API should attempt to reconnect to the gRPC server

proto/proto-backplane-grpc/src/main/proto/deephaven/proto/console.proto

@@ -81,9 +81,14 @@ message GetHeapInfoResponse {
     int64 free_memory = 3 [jstype=JS_STRING];
 }
 
+// Presently you get _all_ logs, not just your console. A future version might take a specific console_id to
+// restrict this to a single console.
 message LogSubscriptionRequest {
-    // presently you get _all_ logs, not just your console
 //    Ticket console_id = 1;
+    // If a non-zero value is specified, represents the timestamp in microseconds since the unix epoch when
+    // the client last saw a message. Technically this might skip messages if more than one message was
+    // logged at the same microsecond that connection was lost - to avoid this, subtract one from the last
+    // seen message's micros, and expect to receive some messages that have already been seen.
     int64 last_seen_log_timestamp = 1 [jstype=JS_STRING];
     repeated string levels = 2;
 }

server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java

@@ -364,7 +364,7 @@ public void record(LogBufferRecord record) {
             }
             // since the subscribe() method auto-replays all existing logs, filter to just once this consumer probably
             // hasn't seen
-            if (record.getTimestampMicros() < request.getLastSeenLogTimestamp()) {
+            if (record.getTimestampMicros() <= request.getLastSeenLogTimestamp()) {
                 return;
             }
             // Note: we can't send record off-thread without doing a deepCopy.

server/src/main/java/io/deephaven/server/table/ExportedTableUpdateListener.java

@@ -169,7 +169,7 @@ private synchronized void sendUpdateMessage(final Ticket ticket, final long size
         try {
             responseObserver.onNext(update.build());
         } catch (final RuntimeException err) {
-            log.error().append(logPrefix).append("failed to notify listener of state change: ").append(err).endl();
+            log.debug().append(logPrefix).append("failed to notify listener of state change: ").append(err).endl();
             session.removeExportListener(this);
         }
     }

web/client-api/src/main/java/io/deephaven/web/client/api/QueryConnectable.java

@@ -8,6 +8,7 @@
 import elemental2.dom.CustomEventInit;
 import elemental2.dom.DomGlobal;
 import elemental2.promise.Promise;
+import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.session_pb.TerminationNotificationResponse;
 import io.deephaven.web.client.ide.IdeSession;
 import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.console_pb.*;
 import io.deephaven.javascript.proto.dhinternal.io.deephaven.proto.ticket_pb.Ticket;
@@ -45,7 +46,12 @@ public abstract class QueryConnectable<Self extends QueryConnectable<Self>> exte
     @JsProperty(namespace = "dh.QueryInfo")
     public static final String EVENT_CONNECT = "connect";
 
+    /**
+     * Removed in favor of a proper disconnect/reconnect. Event listeners should switch to the "disconnect" and
+     * "reconnect" events instead.
+     */
     @JsProperty(namespace = "dh.IdeConnection")
+    @Deprecated
     public static final String HACK_CONNECTION_FAILURE = "hack-connection-failure";
 
     private final List<IdeSession> sessions = new ArrayList<>();
@@ -66,8 +72,9 @@ public QueryConnectable() {
 
     public abstract Promise<ConnectOptions> getConnectOptions();
 
+    @Deprecated
     public void notifyConnectionError(ResponseStreamWrapper.Status status) {
-        if (notifiedConnectionError) {
+        if (notifiedConnectionError || !hasListeners(HACK_CONNECTION_FAILURE)) {
             return;
         }
         notifiedConnectionError = true;
@@ -78,6 +85,8 @@ public void notifyConnectionError(ResponseStreamWrapper.Status status) {
                 "details", status.getDetails(),
                 "metadata", status.getMetadata()));
         fireEvent(HACK_CONNECTION_FAILURE, event);
+        JsLog.warn(
+                "The event dh.IdeConnection.HACK_CONNECTION_FAILURE is deprecated and will be removed in a later release");
     }
 
     @Override
@@ -104,6 +113,20 @@ public String getServerUrl() {
         throw new UnsupportedOperationException();
     }
 
+    /**
+     * Internal method to permit delegating to some orchestration tool to see if this worker can be connected to yet.
+     */
+    @JsIgnore
+    public Promise<Self> onReady() {
+        // noinspection unchecked
+        return Promise.resolve((Self) this);
+    }
+
+    /**
+     * Promise that resolves when this worker instance can be connected to, or rejects if it can't be used.
+     * 
+     * @return A promise that resolves with this instance.
+     */
     public abstract Promise<Self> running();
 
     @JsMethod
@@ -203,6 +226,7 @@ public void connected() {
         JsLog.debug(getClass(), " connected");
 
         connected = true;
+        notifiedConnectionError = false;
 
         fireEvent(EVENT_CONNECT);
 
@@ -243,4 +267,6 @@ public void disconnected() {
                     + "Query disconnected (to prevent this log message, handle the EVENT_DISCONNECT event)");
         }
     }
+
+    public abstract void notifyServerShutdown(TerminationNotificationResponse success);
 }