Skip to content

Commit

Permalink
fix: restart AppMap CLI processes when they terminate unexpectedly
Browse files Browse the repository at this point in the history
  • Loading branch information
@jansorg
jansorg committed Nov 22, 2023
1 parent 8a2cd1f commit 94c7727
Show file tree
Hide file tree
Showing 2 changed files with 248 additions and 66 deletions.
271 changes: 205 additions & 66 deletions plugin-core/src/main/java/appland/cli/DefaultCommandLineService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@
import appland.config.AppMapConfigFile;
import appland.config.AppMapConfigFileListener;
import appland.files.AppMapVfsUtils;
import com.intellij.execution.CantRunException;
import com.intellij.execution.ExecutionException;
import com.intellij.execution.configurations.GeneralCommandLine;
import com.intellij.execution.configurations.PtyCommandLine;
import com.intellij.execution.process.KillableProcessHandler;
import com.intellij.execution.process.ProcessAdapter;
import com.intellij.execution.process.ProcessEvent;
import com.intellij.execution.process.ProcessOutputType;
import com.intellij.openapi.Disposable;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.project.ProjectManager;
Expand All @@ -21,12 +21,16 @@
import com.intellij.openapi.vfs.VfsUtilCore;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.openapi.vfs.ex.temp.TempFileSystem;
import com.intellij.util.Alarm;
import com.intellij.util.AlarmFactory;
import com.intellij.util.io.BaseOutputReader;
import com.intellij.util.system.CpuArch;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.TestOnly;

import javax.annotation.concurrent.GuardedBy;
import java.nio.file.Files;
Expand All @@ -39,6 +43,11 @@

public class DefaultCommandLineService implements AppLandCommandLineService {
private static final Logger LOG = Logger.getInstance(DefaultCommandLineService.class);
private static final long INITIAL_RESTART_DELAY_MILLIS = 5_000;
private static final double NEXT_RESTART_FACTOR = 1.5;
// allow for two restarts
private static final long MAX_RESTART_DELAY_MILLIS = INITIAL_RESTART_DELAY_MILLIS * 3;
private static final Key<Long> NEXT_RESTART_DELAY = Key.create("appmap.processRestartDelay");

@GuardedBy("this")
protected final Map<VirtualFile, CliProcesses> processes = new HashMap<>();
Expand Down Expand Up @@ -88,10 +97,17 @@ public synchronized void start(@NotNull VirtualFile directory, boolean waitForPr

// start new services
var newProcesses = startProcesses(directory);
if (newProcesses != null) {
processes.put(directory, newProcesses);
if (newProcesses != null && !newProcesses.isEmpty()) {
var scanner = newProcesses.scanner;
var indexer = newProcesses.indexer;

newProcesses.indexer.addProcessListener(new IndexEventsProcessListener(), this);
processes.put(directory, newProcesses);
if (scanner != null) {
scanner.startNotify();
}
if (indexer != null) {
indexer.startNotify();
}
}
}

Expand Down Expand Up @@ -196,18 +212,9 @@ public synchronized String toString() {
'}';
}

private void stopLocked(@NotNull CliProcesses value, boolean waitForTermination) {
try {
shutdownInBackground(value.indexer, waitForTermination);
} catch (Exception e) {
LOG.warn("Error shutting down indexer", e);
}

try {
shutdownInBackground(value.scanner, waitForTermination);
} catch (Exception e) {
LOG.warn("Error shutting down scanner", e);
}
private void stopLocked(@NotNull DefaultCommandLineService.CliProcesses processes, boolean waitForTermination) {
stopProcess(processes.indexer, waitForTermination);
stopProcess(processes.scanner, waitForTermination);
}

@Override
Expand All @@ -220,7 +227,26 @@ protected void requestVirtualFileRefresh(@NotNull Path path) {
VfsUtil.markDirtyAndRefresh(true, false, false, path.toFile());
}

private static @Nullable CliProcesses startProcesses(@NotNull VirtualFile directory) throws ExecutionException {
@TestOnly
synchronized @Nullable DefaultCommandLineService.CliProcesses getProcesses(@NotNull VirtualFile directory) {
return processes.get(directory);
}

private @Nullable DefaultCommandLineService.CliProcesses startProcesses(@NotNull VirtualFile directory) throws ExecutionException {
var indexer = startIndexerProcess(directory);
try {
var scanner = startScannerProcesses(directory);
return new CliProcesses(indexer, scanner);
} catch (ExecutionException e) {
LOG.debug("Error starting scanner process", e);
if (indexer != null) {
stopProcess(indexer, false);
}
return null;
}
}

private @Nullable KillableProcessHandler startIndexerProcess(@NotNull VirtualFile directory) throws ExecutionException {
if (!isSupported()) {
return null;
}
Expand All @@ -235,37 +261,40 @@ protected void requestVirtualFileRefresh(@NotNull Path path) {
return null;
}

var workingDir = AppMapVfsUtils.asNativePath(directory);
var watchedDir = findWatchedAppMapDirectory(workingDir);
prepareWatchedDirectory(watchedDir);

var process = startProcess(workingDir, indexerPath.toString(), "index", "--verbose", "--watch", "--appmap-dir", watchedDir.toString());
process.addProcessListener(LoggingProcessAdapter.INSTANCE);
process.addProcessListener(new RestartProcessListener(directory, process, ProcessType.Indexer, this), this);
process.addProcessListener(new IndexEventsProcessListener(), this);
return process;
}

private @Nullable KillableProcessHandler startScannerProcesses(@NotNull VirtualFile directory) throws ExecutionException {
if (!isSupported()) {
return null;
}

// don't launch for in-memory directories in unit test mode
if (ApplicationManager.getApplication().isUnitTestMode() && directory.getFileSystem() instanceof TempFileSystem) {
return null;
}

var scannerPath = AppLandDownloadService.getInstance().getDownloadFilePath(CliTool.Scanner);
if (scannerPath == null || Files.notExists(scannerPath)) {
return null;
}

var workingDir = AppMapVfsUtils.asNativePath(directory);
var watchedDir = findWatchedAppMapDirectory(workingDir);
prepareWatchedDirectory(watchedDir);

// create AppMap directory if it does not exist yet
if (Files.notExists(watchedDir)) {
try {
Files.createDirectories(watchedDir);
} catch (Exception e) {
LOG.debug("Failed to create AppMap directory: " + watchedDir, e);
}
}

var indexer = startProcess(workingDir, indexerPath.toString(), "index", "--verbose", "--watch", "--appmap-dir", watchedDir.toString());

try {
var scanner = startProcess(workingDir, scannerPath.toString(), "scan", "--watch", "--appmap-dir", watchedDir.toString());
return new CliProcesses(indexer, scanner);
} catch (ExecutionException e) {
LOG.debug("Error executing scanner process. Attempting to terminate indexer process.");
try {
indexer.killProcess();
} catch (Exception ex) {
LOG.debug("Error terminating scanner process", ex);
}
throw new CantRunException("Failed to execute AppMap scanner process", e);
}
var process = startProcess(workingDir, scannerPath.toString(), "scan", "--watch", "--appmap-dir", watchedDir.toString());
process.addProcessListener(LoggingProcessAdapter.INSTANCE);
process.addProcessListener(new RestartProcessListener(directory, process, ProcessType.Scanner, this), this);
return process;
}

private void doRefreshForOpenProjectsLocked() {
Expand Down Expand Up @@ -324,6 +353,19 @@ private void doRefreshForOpenProjectsLocked() {
}
}

/**
* Create AppMap directory if it does not exist yet.
*/
private static void prepareWatchedDirectory(Path watchedDir) {
if (Files.notExists(watchedDir)) {
try {
Files.createDirectories(watchedDir);
} catch (Exception e) {
LOG.debug("Failed to create AppMap directory: " + watchedDir, e);
}
}
}

private static boolean isSupported() {
return SystemInfo.isMac && (CpuArch.isIntel64() || CpuArch.isArm64())
|| SystemInfo.isLinux && CpuArch.isIntel64()
Expand All @@ -341,34 +383,43 @@ private static boolean isSupported() {
command.withWorkDirectory(workingDir.toString());
command.withParentEnvironmentType(GeneralCommandLine.ParentEnvironmentType.SYSTEM);

var processHandler = new KillableProcessHandler(command) {
return new KillableProcessHandler(command) {
@Override
protected BaseOutputReader.@NotNull Options readerOptions() {
return BaseOutputReader.Options.BLOCKING;
}
};
processHandler.addProcessListener(LoggingProcessAdapter.INSTANCE);
processHandler.startNotify();

return processHandler;
}

private static void shutdownInBackground(@NotNull KillableProcessHandler process, boolean waitForTermination) throws Exception {
var future = ApplicationManager.getApplication().executeOnPooledThread(() -> {
process.destroyProcess();
process.waitFor(500);
private static void stopProcess(@Nullable KillableProcessHandler process, boolean waitForTermination) {
if (process != null) {
NEXT_RESTART_DELAY.set(process, 0L);

if (!process.isProcessTerminated()) {
process.killProcess();
}
try {
var shutdownRunnable = new Runnable() {
@Override
public void run() {
process.destroyProcess();
process.waitFor(500);

if (!process.isProcessTerminated()) {
process.killProcess();
}

if (waitForTermination) {
process.waitFor();
}
});
if (waitForTermination) {
process.waitFor(1_000);
}
}
};

if (waitForTermination) {
future.get();
if (waitForTermination) {
ApplicationManager.getApplication().executeOnPooledThread(shutdownRunnable).get();
} else {
shutdownRunnable.run();
}
} catch (Exception e) {
LOG.warn("Error shutting down process: " + process, e);
}
}
}

Expand Down Expand Up @@ -403,15 +454,17 @@ private static void shutdownInBackground(@NotNull KillableProcessHandler process
return cmd;
}

@ToString
@EqualsAndHashCode
/**
* Processes active for a single base directory.
*/
@Data
@AllArgsConstructor
protected static final class CliProcesses {
private final @NotNull KillableProcessHandler indexer;
private final @NotNull KillableProcessHandler scanner;
private volatile @Nullable KillableProcessHandler indexer;
private volatile @Nullable KillableProcessHandler scanner;

CliProcesses(@NotNull KillableProcessHandler indexer, @NotNull KillableProcessHandler scanner) {
this.indexer = indexer;
this.scanner = scanner;
boolean isEmpty() {
return indexer == null && scanner == null;
}
}

Expand Down Expand Up @@ -445,4 +498,90 @@ public void onTextAvailable(@NotNull ProcessEvent event, @NotNull Key outputType
}
}
}

/**
* Process listener to handle restarts when the given process terminates unexpectedly.
*/
@Data
@EqualsAndHashCode(callSuper = true)
private class RestartProcessListener extends ProcessAdapter {
private final @NotNull VirtualFile directory;
private final @NotNull KillableProcessHandler process;
private final @NotNull ProcessType type;
private final @NotNull Disposable parentDisposable;
private final Alarm alarm;

public RestartProcessListener(@NotNull VirtualFile directory,
@NotNull KillableProcessHandler process,
@NotNull ProcessType type,
@NotNull Disposable parentDisposable) {
this.directory = directory;
this.process = process;
this.type = type;
this.parentDisposable = parentDisposable;
this.alarm = AlarmFactory.getInstance().create(Alarm.ThreadToUse.POOLED_THREAD, this.parentDisposable);
}

@Override
public void processTerminated(@NotNull ProcessEvent event) {
long nextRestartDelay = NEXT_RESTART_DELAY.get(process, INITIAL_RESTART_DELAY_MILLIS);

// don't restart if max attempts were reached or if restarts were disabled
if (nextRestartDelay <= 0 || nextRestartDelay > MAX_RESTART_DELAY_MILLIS) {
synchronized (DefaultCommandLineService.this) {
var entry = processes.get(directory);
if (entry != null) {
switch (type) {
case Indexer:
entry.indexer = null;
break;
case Scanner:
entry.scanner = null;
break;
}

if (entry.isEmpty()) {
processes.remove(directory);
}
}
}
return;
}

// schedule next restart
NEXT_RESTART_DELAY.set(process, (long) ((double) nextRestartDelay * NEXT_RESTART_FACTOR));
alarm.cancelAllRequests();
alarm.addRequest(() -> {
try {
synchronized (DefaultCommandLineService.this) {
var directoryProcesses = processes.get(directory);
if (directoryProcesses != null) {
switch (type) {
case Indexer:
var indexer = startIndexerProcess(directory);
directoryProcesses.indexer = indexer;
if (indexer != null) {
indexer.startNotify();
}
break;
case Scanner:
var scanner = startScannerProcesses(directory);
directoryProcesses.scanner = scanner;
if (scanner != null) {
scanner.startNotify();
}
break;
}
}
}
} catch (ExecutionException e) {
LOG.debug("Error restarting process. Type: " + type + ", command line: " + process.getCommandLine(), e);
}
}, nextRestartDelay);
}
}

private enum ProcessType {
Indexer, Scanner
}
}
Loading

0 comments on commit 94c7727

Please sign in to comment.